Spaces:
Running
on
Zero
Running
on
Zero
1st draft release
Browse files- .gitignore +177 -0
- app.py +25 -62
- docs/leaderboard_header.md +16 -0
- docs/model_registration_guide.md +74 -0
- indiebot_arena/__init__.py +0 -0
- indiebot_arena/config.py +15 -0
- indiebot_arena/dao/__init__.py +0 -0
- indiebot_arena/dao/mongo_dao.py +156 -0
- indiebot_arena/model/__init__.py +0 -0
- indiebot_arena/model/domain_model.py +41 -0
- indiebot_arena/service/__init__.py +0 -0
- indiebot_arena/service/arena_service.py +232 -0
- indiebot_arena/service/bootstrap_service.py +32 -0
- indiebot_arena/ui/__init__.py +0 -0
- indiebot_arena/ui/battle.py +234 -0
- indiebot_arena/ui/leaderboard.py +47 -0
- indiebot_arena/ui/registration.py +196 -0
- requirements.txt +11 -1
- style.css +19 -0
.gitignore
ADDED
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Byte-compiled / optimized / DLL files
|
2 |
+
__pycache__/
|
3 |
+
*.py[cod]
|
4 |
+
*$py.class
|
5 |
+
|
6 |
+
# C extensions
|
7 |
+
*.so
|
8 |
+
|
9 |
+
# Distribution / packaging
|
10 |
+
.Python
|
11 |
+
build/
|
12 |
+
develop-eggs/
|
13 |
+
dist/
|
14 |
+
downloads/
|
15 |
+
eggs/
|
16 |
+
.eggs/
|
17 |
+
lib/
|
18 |
+
lib64/
|
19 |
+
parts/
|
20 |
+
sdist/
|
21 |
+
var/
|
22 |
+
wheels/
|
23 |
+
share/python-wheels/
|
24 |
+
*.egg-info/
|
25 |
+
.installed.cfg
|
26 |
+
*.egg
|
27 |
+
MANIFEST
|
28 |
+
|
29 |
+
# PyInstaller
|
30 |
+
# Usually these files are written by a python script from a template
|
31 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
32 |
+
*.manifest
|
33 |
+
*.spec
|
34 |
+
|
35 |
+
# Installer logs
|
36 |
+
pip-log.txt
|
37 |
+
pip-delete-this-directory.txt
|
38 |
+
|
39 |
+
# Unit test / coverage reports
|
40 |
+
htmlcov/
|
41 |
+
.tox/
|
42 |
+
.nox/
|
43 |
+
.coverage
|
44 |
+
.coverage.*
|
45 |
+
.cache
|
46 |
+
nosetests.xml
|
47 |
+
coverage.xml
|
48 |
+
*.cover
|
49 |
+
*.py,cover
|
50 |
+
.hypothesis/
|
51 |
+
.pytest_cache/
|
52 |
+
cover/
|
53 |
+
|
54 |
+
# Translations
|
55 |
+
*.mo
|
56 |
+
*.pot
|
57 |
+
|
58 |
+
# Django stuff:
|
59 |
+
*.log
|
60 |
+
local_settings.py
|
61 |
+
db.sqlite3
|
62 |
+
db.sqlite3-journal
|
63 |
+
|
64 |
+
# Flask stuff:
|
65 |
+
instance/
|
66 |
+
.webassets-cache
|
67 |
+
|
68 |
+
# Scrapy stuff:
|
69 |
+
.scrapy
|
70 |
+
|
71 |
+
# Sphinx documentation
|
72 |
+
docs/_build/
|
73 |
+
|
74 |
+
# PyBuilder
|
75 |
+
.pybuilder/
|
76 |
+
target/
|
77 |
+
|
78 |
+
# Jupyter Notebook
|
79 |
+
.ipynb_checkpoints
|
80 |
+
|
81 |
+
# IPython
|
82 |
+
profile_default/
|
83 |
+
ipython_config.py
|
84 |
+
|
85 |
+
# pyenv
|
86 |
+
# For a library or package, you might want to ignore these files since the code is
|
87 |
+
# intended to run in multiple environments; otherwise, check them in:
|
88 |
+
# .python-version
|
89 |
+
|
90 |
+
# pipenv
|
91 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
92 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
93 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
94 |
+
# install all needed dependencies.
|
95 |
+
#Pipfile.lock
|
96 |
+
|
97 |
+
# UV
|
98 |
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
99 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
100 |
+
# commonly ignored for libraries.
|
101 |
+
#uv.lock
|
102 |
+
|
103 |
+
# poetry
|
104 |
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
105 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
106 |
+
# commonly ignored for libraries.
|
107 |
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
108 |
+
#poetry.lock
|
109 |
+
|
110 |
+
# pdm
|
111 |
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
112 |
+
#pdm.lock
|
113 |
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
114 |
+
# in version control.
|
115 |
+
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
116 |
+
.pdm.toml
|
117 |
+
.pdm-python
|
118 |
+
.pdm-build/
|
119 |
+
|
120 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
121 |
+
__pypackages__/
|
122 |
+
|
123 |
+
# Celery stuff
|
124 |
+
celerybeat-schedule
|
125 |
+
celerybeat.pid
|
126 |
+
|
127 |
+
# SageMath parsed files
|
128 |
+
*.sage.py
|
129 |
+
|
130 |
+
# Environments
|
131 |
+
.env
|
132 |
+
.venv
|
133 |
+
env/
|
134 |
+
venv/
|
135 |
+
ENV/
|
136 |
+
env.bak/
|
137 |
+
venv.bak/
|
138 |
+
|
139 |
+
# Spyder project settings
|
140 |
+
.spyderproject
|
141 |
+
.spyproject
|
142 |
+
|
143 |
+
# Rope project settings
|
144 |
+
.ropeproject
|
145 |
+
|
146 |
+
# mkdocs documentation
|
147 |
+
/site
|
148 |
+
|
149 |
+
# mypy
|
150 |
+
.mypy_cache/
|
151 |
+
.dmypy.json
|
152 |
+
dmypy.json
|
153 |
+
|
154 |
+
# Pyre type checker
|
155 |
+
.pyre/
|
156 |
+
|
157 |
+
# pytype static type analyzer
|
158 |
+
.pytype/
|
159 |
+
|
160 |
+
# Cython debug symbols
|
161 |
+
cython_debug/
|
162 |
+
|
163 |
+
# PyCharm
|
164 |
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
165 |
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
166 |
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
167 |
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
168 |
+
#.idea/
|
169 |
+
|
170 |
+
# Ruff stuff:
|
171 |
+
.ruff_cache/
|
172 |
+
|
173 |
+
# PyPI configuration file
|
174 |
+
.pypirc
|
175 |
+
|
176 |
+
.idea
|
177 |
+
.DS_Store
|
app.py
CHANGED
@@ -1,64 +1,27 @@
|
|
1 |
import gradio as gr
|
2 |
-
from huggingface_hub import InferenceClient
|
3 |
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
for message in client.chat_completion(
|
31 |
-
messages,
|
32 |
-
max_tokens=max_tokens,
|
33 |
-
stream=True,
|
34 |
-
temperature=temperature,
|
35 |
-
top_p=top_p,
|
36 |
-
):
|
37 |
-
token = message.choices[0].delta.content
|
38 |
-
|
39 |
-
response += token
|
40 |
-
yield response
|
41 |
-
|
42 |
-
|
43 |
-
"""
|
44 |
-
For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
|
45 |
-
"""
|
46 |
-
demo = gr.ChatInterface(
|
47 |
-
respond,
|
48 |
-
additional_inputs=[
|
49 |
-
gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
|
50 |
-
gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
|
51 |
-
gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
|
52 |
-
gr.Slider(
|
53 |
-
minimum=0.1,
|
54 |
-
maximum=1.0,
|
55 |
-
value=0.95,
|
56 |
-
step=0.05,
|
57 |
-
label="Top-p (nucleus sampling)",
|
58 |
-
),
|
59 |
-
],
|
60 |
-
)
|
61 |
-
|
62 |
-
|
63 |
-
if __name__ == "__main__":
|
64 |
-
demo.launch()
|
|
|
1 |
import gradio as gr
|
|
|
2 |
|
3 |
+
from indiebot_arena.config import MONGO_DB_URI, MONGO_DB_NAME, LANGUAGE
|
4 |
+
from indiebot_arena.dao.mongo_dao import MongoDAO
|
5 |
+
from indiebot_arena.service.bootstrap_service import BootstrapService
|
6 |
+
from indiebot_arena.ui.battle import battle_content
|
7 |
+
from indiebot_arena.ui.leaderboard import leaderboard_content
|
8 |
+
from indiebot_arena.ui.registration import registration_content
|
9 |
+
|
10 |
+
dao = MongoDAO(MONGO_DB_URI, MONGO_DB_NAME)
|
11 |
+
bootstrap_service = BootstrapService(dao)
|
12 |
+
bootstrap_service.provision_database()
|
13 |
+
|
14 |
+
with open("style.css", "r") as f:
|
15 |
+
custom_css = f.read()
|
16 |
+
|
17 |
+
with gr.Blocks(css=custom_css) as demo:
|
18 |
+
with gr.Tabs():
|
19 |
+
with gr.TabItem("🏆 リーダーボード"):
|
20 |
+
leaderboard_content(dao, LANGUAGE)
|
21 |
+
with gr.TabItem("⚔️ モデルに投票"):
|
22 |
+
battle_content(dao, LANGUAGE)
|
23 |
+
with gr.TabItem("📚️ モデルの登録"):
|
24 |
+
registration_content(dao, LANGUAGE)
|
25 |
+
|
26 |
+
if __name__=="__main__":
|
27 |
+
demo.queue(max_size=20).launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
docs/leaderboard_header.md
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# 🚀 IndieBot Arena
|
2 |
+
|
3 |
+
インディーボットアリーナはChatbot Arenaのインディーズ版のようなWebアプリです。
|
4 |
+
Chatbot Arenaにインスパイアされて開発されましが、以下のような違いがあります。
|
5 |
+
|
6 |
+
- 誰でもモデルを登録してコンペに参加可能
|
7 |
+
- 階級別にチャットバトルとリーダーボードがある
|
8 |
+
|
9 |
+
初期データとして、Google公式のInstruction-Tuning済みモデルを登録しています。
|
10 |
+
【モデルに投票】から匿名のチャットバトルで投票することが出来ます。
|
11 |
+
【モデルの登録】から自分の事後学習済みモデルを登録することも出来ます。
|
12 |
+
|
13 |
+
Google公式モデルを上回る日本語性能スコアのモデルをぜひ皆で作りましょう!
|
14 |
+
|
15 |
+
|
16 |
+
|
docs/model_registration_guide.md
ADDED
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
## 登録可能なモデルの要件
|
2 |
+
|
3 |
+
登録可能なモデルは、Hugging Faceのモデルハブで公開されているPyTorchベースのモデルで、transformersライブラリのgenerate APIに対応している必要があります。
|
4 |
+
重みのファイル形式はsafetensorsのみで、量子化する場合はBitsAndBytesのバージョン0.44.1以降で量子化されている必要があります。
|
5 |
+
tokenizer_config.jsonで正しくchat_templateが設定されている必要があります。
|
6 |
+
パラメータ数ではなく、重みのファイルのサイズが5GB未満または10GB未満で階級ごとに分かれてリーダーボードが作れます。
|
7 |
+
|
8 |
+
- **GPU環境**: HF SpacesのZeroGPU A100(40GB)
|
9 |
+
- **LLM実行環境**: transformersライブラリ(4.50.0)
|
10 |
+
- **量子化**: BitsAndBytesのみ対応(0.44.1)
|
11 |
+
- **ファイル形式**: safetensorsのみ
|
12 |
+
- **ファイルサイズ**: 5GB又は10GB未満
|
13 |
+
- **チャットテンプレート**: chat_template設定が必要(tokenizer_config.json)
|
14 |
+
|
15 |
+
非量子化モデルでもファイルサイズ制限をクリアすれば登録可能ですが、サーバーの負荷低減のためにBitsAndBytesによる量子化を推奨します。
|
16 |
+
将来的には、llama.cppの実行環境でGGUF形式に対応する予定です。
|
17 |
+
|
18 |
+
## モデルの登録方法
|
19 |
+
|
20 |
+
このページの一番下にある、モデルの新規登録フォームから登録できます。
|
21 |
+
モデルIDとエントリーしたいファイルサイズ区分を選択してロードテストボタンを押して下さい。
|
22 |
+
|
23 |
+
### ① ロードテスト
|
24 |
+
ロードテストではファイル形式やサイズなどのチェックが行われます。
|
25 |
+
|
26 |
+
### ② チャットテスト
|
27 |
+
チャットテストでは簡単な日本語応答が出来るかチェックが行われます。
|
28 |
+
|
29 |
+
### ③ 登録
|
30 |
+
ロードテストとチャットテストをクリアしたモデルだけ登録可能です。
|
31 |
+
登録が完了すると、登録済みモデル一覧にあなたのモデルが表示されます。
|
32 |
+
|
33 |
+
## 量子化サンプルコード
|
34 |
+
|
35 |
+
以下はBitsAndBytesで4bit量子化して自分のリポジトリにPushするまでのサンプルコードです。
|
36 |
+
|
37 |
+
```
|
38 |
+
# python 3.10
|
39 |
+
pip install bitsandbytes==0.44.1
|
40 |
+
pip install accelerate==1.2.1
|
41 |
+
pip install transformers==4.50.0
|
42 |
+
pip install huggingface_hub[cli]
|
43 |
+
```
|
44 |
+
```
|
45 |
+
# アクセストークンを入力してログイン
|
46 |
+
huggingface-cli login
|
47 |
+
```
|
48 |
+
```python
|
49 |
+
import torch
|
50 |
+
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
|
51 |
+
|
52 |
+
model_id = "google/gemma-2-2b-it"
|
53 |
+
repo_id = "xxxxx/gemma-2-2b-it-bnb-4bit"
|
54 |
+
|
55 |
+
bnb_config = BitsAndBytesConfig(
|
56 |
+
load_in_4bit=True,
|
57 |
+
bnb_4bit_quant_type="nf4",
|
58 |
+
bnb_4bit_compute_dtype=torch.bfloat16
|
59 |
+
)
|
60 |
+
|
61 |
+
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
62 |
+
model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map="auto")
|
63 |
+
|
64 |
+
tokenizer.push_to_hub(repo_id)
|
65 |
+
model.push_to_hub(repo_id)
|
66 |
+
|
67 |
+
```
|
68 |
+
|
69 |
+
量子化後のモデルIDは任意の名前が可能ですが、以下の形式を推奨します。
|
70 |
+
|
71 |
+
* BitsAndBytesの4bit量子化の場合
|
72 |
+
```
|
73 |
+
[量子化前のモデルID]-bnb-4bit
|
74 |
+
```
|
indiebot_arena/__init__.py
ADDED
File without changes
|
indiebot_arena/config.py
ADDED
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
|
3 |
+
MONGO_DB_URI = os.environ.get("MONGO_DB_URI", "mongodb://localhost:27017")
|
4 |
+
MONGO_DB_NAME = os.environ.get("MONGO_DB_NAME", "test_db")
|
5 |
+
LANGUAGE = "ja"
|
6 |
+
|
7 |
+
DEBUG = os.getenv("DEBUG", "False").lower() in ["true", "1", "yes"]
|
8 |
+
LOCAL_TESTING = os.getenv("LOCAL_TESTING", "False").lower() in ["true", "1", "yes"]
|
9 |
+
MODEL_SELECTION_MODE = os.getenv("MODEL_SELECTION_MODE", "random")
|
10 |
+
MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
|
11 |
+
|
12 |
+
if LOCAL_TESTING:
|
13 |
+
MAX_NEW_TOKENS = 20
|
14 |
+
else:
|
15 |
+
MAX_NEW_TOKENS = 512
|
indiebot_arena/dao/__init__.py
ADDED
File without changes
|
indiebot_arena/dao/mongo_dao.py
ADDED
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from dataclasses import asdict
|
2 |
+
from typing import List, Optional
|
3 |
+
|
4 |
+
from bson import ObjectId
|
5 |
+
from pymongo import MongoClient
|
6 |
+
|
7 |
+
from indiebot_arena.model.domain_model import Model, Battle, LeaderboardEntry
|
8 |
+
|
9 |
+
|
10 |
+
class MongoDAO:
|
11 |
+
def __init__(self, uri: str, db_name: str):
|
12 |
+
self.client = MongoClient(uri)
|
13 |
+
self.db = self.client[db_name]
|
14 |
+
self.models_collection = self.db["models"]
|
15 |
+
self.battles_collection = self.db["battles"]
|
16 |
+
self.leaderboard_collection = self.db["leaderboard"]
|
17 |
+
|
18 |
+
self.models_collection.create_index(
|
19 |
+
[("language", 1), ("weight_class", 1), ("model_name", 1)], unique=True
|
20 |
+
)
|
21 |
+
self.leaderboard_collection.create_index(
|
22 |
+
[("language", 1), ("weight_class", 1), ("model_id", 1)], unique=True
|
23 |
+
)
|
24 |
+
self.battles_collection.create_index(
|
25 |
+
[("language", 1), ("weight_class", 1), ("vote_timestamp", 1)]
|
26 |
+
)
|
27 |
+
|
28 |
+
# ---------- Model ----------
|
29 |
+
|
30 |
+
def insert_model(self, model: Model) -> ObjectId:
|
31 |
+
data = asdict(model)
|
32 |
+
if data.get("_id") is None:
|
33 |
+
data.pop("_id")
|
34 |
+
result = self.models_collection.insert_one(data)
|
35 |
+
return result.inserted_id
|
36 |
+
|
37 |
+
def get_model(self, model_id: ObjectId) -> Optional[Model]:
|
38 |
+
data = self.models_collection.find_one({"_id": model_id})
|
39 |
+
if data:
|
40 |
+
return Model(**data)
|
41 |
+
return None
|
42 |
+
|
43 |
+
def update_model(self, model: Model) -> bool:
|
44 |
+
data = asdict(model)
|
45 |
+
if data.get("_id") is None:
|
46 |
+
raise ValueError("model _id is required for updating.")
|
47 |
+
result = self.models_collection.replace_one({"_id": data["_id"]}, data)
|
48 |
+
return result.modified_count > 0
|
49 |
+
|
50 |
+
def delete_model(self, model_id: ObjectId) -> bool:
|
51 |
+
result = self.models_collection.delete_one({"_id": model_id})
|
52 |
+
return result.deleted_count > 0
|
53 |
+
|
54 |
+
def find_models(self, language: str, weight_class: str) -> List[Model]:
|
55 |
+
query = {
|
56 |
+
"language": language,
|
57 |
+
"weight_class": weight_class
|
58 |
+
}
|
59 |
+
cursor = self.models_collection.find(query).sort("_id", 1)
|
60 |
+
return [Model(**doc) for doc in cursor]
|
61 |
+
|
62 |
+
def find_one_model(self, language: str, weight_class: str, model_name: str) -> Optional[Model]:
|
63 |
+
query = {
|
64 |
+
"language": language,
|
65 |
+
"weight_class": weight_class,
|
66 |
+
"model_name": model_name
|
67 |
+
}
|
68 |
+
data = self.models_collection.find_one(query)
|
69 |
+
if data:
|
70 |
+
return Model(**data)
|
71 |
+
return None
|
72 |
+
|
73 |
+
# ---------- Battle ----------
|
74 |
+
|
75 |
+
def insert_battle(self, battle: Battle) -> ObjectId:
|
76 |
+
data = asdict(battle)
|
77 |
+
if data.get("_id") is None:
|
78 |
+
data.pop("_id")
|
79 |
+
result = self.battles_collection.insert_one(data)
|
80 |
+
return result.inserted_id
|
81 |
+
|
82 |
+
def get_battle(self, battle_id: ObjectId) -> Optional[Battle]:
|
83 |
+
data = self.battles_collection.find_one({"_id": battle_id})
|
84 |
+
if data:
|
85 |
+
return Battle(**data)
|
86 |
+
return None
|
87 |
+
|
88 |
+
def update_battle(self, battle: Battle) -> bool:
|
89 |
+
data = asdict(battle)
|
90 |
+
if data.get("_id") is None:
|
91 |
+
raise ValueError("battle _id is required for updating.")
|
92 |
+
result = self.battles_collection.replace_one({"_id": data["_id"]}, data)
|
93 |
+
return result.modified_count > 0
|
94 |
+
|
95 |
+
def delete_battle(self, battle_id: ObjectId) -> bool:
|
96 |
+
result = self.battles_collection.delete_one({"_id": battle_id})
|
97 |
+
return result.deleted_count > 0
|
98 |
+
|
99 |
+
def find_battles(self, language: str, weight_class: str) -> List[Battle]:
|
100 |
+
query = {
|
101 |
+
"language": language,
|
102 |
+
"weight_class": weight_class
|
103 |
+
}
|
104 |
+
cursor = self.battles_collection.find(query)
|
105 |
+
return [Battle(**doc) for doc in cursor]
|
106 |
+
|
107 |
+
def find_last_battle(self) -> Optional[Battle]:
|
108 |
+
battle_doc = self.battles_collection.find_one(sort=[("_id", -1)])
|
109 |
+
if battle_doc:
|
110 |
+
return Battle(**battle_doc)
|
111 |
+
return None
|
112 |
+
|
113 |
+
# ---------- LeaderboardEntry ----------
|
114 |
+
|
115 |
+
def insert_leaderboard_entry(self, entry: LeaderboardEntry) -> ObjectId:
|
116 |
+
data = asdict(entry)
|
117 |
+
if data.get("_id") is None:
|
118 |
+
data.pop("_id")
|
119 |
+
result = self.leaderboard_collection.insert_one(data)
|
120 |
+
return result.inserted_id
|
121 |
+
|
122 |
+
def get_leaderboard_entry(self, entry_id: ObjectId) -> Optional[LeaderboardEntry]:
|
123 |
+
data = self.leaderboard_collection.find_one({"_id": entry_id})
|
124 |
+
if data:
|
125 |
+
return LeaderboardEntry(**data)
|
126 |
+
return None
|
127 |
+
|
128 |
+
def update_leaderboard_entry(self, entry: LeaderboardEntry) -> bool:
|
129 |
+
data = asdict(entry)
|
130 |
+
if data.get("_id") is None:
|
131 |
+
raise ValueError("leaderboard _id is required for updating.")
|
132 |
+
result = self.leaderboard_collection.replace_one({"_id": data["_id"]}, data)
|
133 |
+
return result.modified_count > 0
|
134 |
+
|
135 |
+
def delete_leaderboard_entry(self, entry_id: ObjectId) -> bool:
|
136 |
+
result = self.leaderboard_collection.delete_one({"_id": entry_id})
|
137 |
+
return result.deleted_count > 0
|
138 |
+
|
139 |
+
def find_leaderboard_entries(self, language: str, weight_class: str) -> List[LeaderboardEntry]:
|
140 |
+
query = {
|
141 |
+
"language": language,
|
142 |
+
"weight_class": weight_class
|
143 |
+
}
|
144 |
+
cursor = self.leaderboard_collection.find(query).sort("elo_score", -1)
|
145 |
+
return [LeaderboardEntry(**doc) for doc in cursor]
|
146 |
+
|
147 |
+
def find_one_leaderboard_entry(self, language: str, weight_class: str, model_id: ObjectId) -> Optional[
|
148 |
+
LeaderboardEntry]:
|
149 |
+
data = self.leaderboard_collection.find_one({
|
150 |
+
"language": language,
|
151 |
+
"weight_class": weight_class,
|
152 |
+
"model_id": model_id
|
153 |
+
})
|
154 |
+
if data:
|
155 |
+
return LeaderboardEntry(**data)
|
156 |
+
return None
|
indiebot_arena/model/__init__.py
ADDED
File without changes
|
indiebot_arena/model/domain_model.py
ADDED
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from dataclasses import dataclass, field
|
2 |
+
from datetime import datetime
|
3 |
+
from typing import Optional
|
4 |
+
|
5 |
+
from bson import ObjectId
|
6 |
+
|
7 |
+
|
8 |
+
@dataclass
|
9 |
+
class Model:
|
10 |
+
language: str # 言語区分 (例: "ja", "en")
|
11 |
+
weight_class: str # サイズ区分 (例: "U-5GB", "U-10GB")
|
12 |
+
model_name: str # 例: "fukugawa/gemma-2-9b-finetuned-bnb-4bit"
|
13 |
+
runtime: str # 実行環境 (例: "transformers", "llama.cpp")
|
14 |
+
quantization: str # 量子化方式 (例: "none", "bnb", "gptq")
|
15 |
+
file_format: str # 保存形式 (例: "safetensors", "gguf")
|
16 |
+
file_size_gb: float # ファイルサイズ(単位: GB)
|
17 |
+
description: Optional[str] = None
|
18 |
+
created_at: datetime = field(default_factory=datetime.utcnow)
|
19 |
+
_id: Optional[ObjectId] = None
|
20 |
+
|
21 |
+
|
22 |
+
@dataclass
|
23 |
+
class Battle:
|
24 |
+
language: str
|
25 |
+
weight_class: str
|
26 |
+
model_a_id: ObjectId
|
27 |
+
model_b_id: ObjectId
|
28 |
+
winner_model_id: ObjectId
|
29 |
+
user_id: str
|
30 |
+
vote_timestamp: datetime = field(default_factory=datetime.utcnow)
|
31 |
+
_id: Optional[ObjectId] = None
|
32 |
+
|
33 |
+
|
34 |
+
@dataclass
|
35 |
+
class LeaderboardEntry:
|
36 |
+
language: str
|
37 |
+
weight_class: str
|
38 |
+
model_id: ObjectId
|
39 |
+
elo_score: int
|
40 |
+
last_updated: datetime = field(default_factory=datetime.utcnow)
|
41 |
+
_id: Optional[ObjectId] = None
|
indiebot_arena/service/__init__.py
ADDED
File without changes
|
indiebot_arena/service/arena_service.py
ADDED
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import random
|
2 |
+
import re
|
3 |
+
from datetime import datetime
|
4 |
+
from typing import List, Optional, Tuple, Dict
|
5 |
+
|
6 |
+
from bson import ObjectId
|
7 |
+
|
8 |
+
from indiebot_arena.dao.mongo_dao import MongoDAO
|
9 |
+
from indiebot_arena.model.domain_model import Model, Battle, LeaderboardEntry
|
10 |
+
|
11 |
+
INITIAL_RATING = 1000
|
12 |
+
K_FACTOR = 32
|
13 |
+
|
14 |
+
|
15 |
+
class ArenaService:
|
16 |
+
def __init__(self, dao: MongoDAO):
|
17 |
+
self.dao = dao
|
18 |
+
|
19 |
+
# ---------- Model ----------
|
20 |
+
|
21 |
+
def register_model(
|
22 |
+
self,
|
23 |
+
language: str,
|
24 |
+
weight_class: str,
|
25 |
+
model_name: str,
|
26 |
+
runtime: str,
|
27 |
+
quantization: str,
|
28 |
+
file_format: str,
|
29 |
+
file_size_gb: float,
|
30 |
+
description: Optional[str] = ""
|
31 |
+
) -> ObjectId:
|
32 |
+
# Non-empty checks
|
33 |
+
if not model_name.strip():
|
34 |
+
raise ValueError("Model name cannot be empty.")
|
35 |
+
if not runtime.strip():
|
36 |
+
raise ValueError("Runtime cannot be empty.")
|
37 |
+
if not quantization.strip():
|
38 |
+
raise ValueError("Quantization cannot be empty.")
|
39 |
+
if not file_format.strip():
|
40 |
+
raise ValueError("File format cannot be empty.")
|
41 |
+
|
42 |
+
# Validation
|
43 |
+
if runtime!="transformers":
|
44 |
+
raise ValueError("Runtime must be 'transformers'.")
|
45 |
+
if quantization not in ("bnb", "none"):
|
46 |
+
raise ValueError("Quantization must be 'bnb' or 'none'.")
|
47 |
+
if file_format!="safetensors":
|
48 |
+
raise ValueError("File format must be 'safetensors'.")
|
49 |
+
if language not in ("ja", "en"):
|
50 |
+
raise ValueError("Language must be 'ja' or 'en'.")
|
51 |
+
if weight_class not in ("U-5GB", "U-10GB"):
|
52 |
+
raise ValueError("Weight class must be 'U-5GB' or 'U-10GB'.")
|
53 |
+
if file_size_gb <= 0:
|
54 |
+
raise ValueError("File size must be greater than 0.")
|
55 |
+
if weight_class=="U-5GB" and file_size_gb >= 5.0:
|
56 |
+
raise ValueError("For U-5GB, file size must be less than 5.0.")
|
57 |
+
if weight_class=="U-10GB" and file_size_gb >= 10.0:
|
58 |
+
raise ValueError("For U-10GB, file size must be less than 10.0.")
|
59 |
+
if not re.match(r"^[^/]+/[^/]+$", model_name):
|
60 |
+
raise ValueError("Model name must be in 'xxxxx/xxxxxx' format.")
|
61 |
+
|
62 |
+
# Description length check
|
63 |
+
if description is not None and len(description) > 1024:
|
64 |
+
raise ValueError("Description must be 1024 characters or less.")
|
65 |
+
|
66 |
+
# Duplicate check
|
67 |
+
if self.dao.find_one_model(language, weight_class, model_name) is not None:
|
68 |
+
raise ValueError("A model with this language, weight class and model name already exists.")
|
69 |
+
|
70 |
+
model = Model(
|
71 |
+
model_name=model_name,
|
72 |
+
runtime=runtime,
|
73 |
+
quantization=quantization,
|
74 |
+
file_format=file_format,
|
75 |
+
file_size_gb=file_size_gb,
|
76 |
+
language=language,
|
77 |
+
weight_class=weight_class,
|
78 |
+
description=description
|
79 |
+
)
|
80 |
+
return self.dao.insert_model(model)
|
81 |
+
|
82 |
+
def get_one_model(
|
83 |
+
self,
|
84 |
+
language: str,
|
85 |
+
weight_class: str,
|
86 |
+
model_name: str
|
87 |
+
) -> Optional[Model]:
|
88 |
+
return self.dao.find_one_model(language, weight_class, model_name)
|
89 |
+
|
90 |
+
def get_two_random_models(
|
91 |
+
self,
|
92 |
+
language: str,
|
93 |
+
weight_class: str
|
94 |
+
) -> Tuple[Model, Model]:
|
95 |
+
models = self.dao.find_models(language, weight_class)
|
96 |
+
if len(models) < 2:
|
97 |
+
raise ValueError("Need at least 2 models for a battle.")
|
98 |
+
return tuple(random.sample(models, 2))
|
99 |
+
|
100 |
+
def get_model_dropdown_list(
|
101 |
+
self,
|
102 |
+
language: str,
|
103 |
+
weight_class: str
|
104 |
+
) -> List[Dict[str, str]]:
|
105 |
+
models = self.dao.find_models(language, weight_class)
|
106 |
+
return [
|
107 |
+
{"label": model.model_name, "value": model.model_name}
|
108 |
+
for model in models if model.model_name
|
109 |
+
]
|
110 |
+
|
111 |
+
# ---------- Battle ----------
|
112 |
+
|
113 |
+
def record_battle(
|
114 |
+
self,
|
115 |
+
language: str,
|
116 |
+
weight_class: str,
|
117 |
+
model_a_id: ObjectId,
|
118 |
+
model_b_id: ObjectId,
|
119 |
+
winner_model_id: ObjectId,
|
120 |
+
user_id: str
|
121 |
+
) -> ObjectId:
|
122 |
+
# Validation
|
123 |
+
if language not in ("ja", "en"):
|
124 |
+
raise ValueError("Language must be 'ja' or 'en'.")
|
125 |
+
if weight_class not in ("U-5GB", "U-10GB"):
|
126 |
+
raise ValueError("Weight class must be 'U-5GB' or 'U-10GB'.")
|
127 |
+
|
128 |
+
if model_a_id is None or model_b_id is None or winner_model_id is None:
|
129 |
+
raise ValueError("All model IDs must be provided.")
|
130 |
+
|
131 |
+
if model_a_id==model_b_id:
|
132 |
+
raise ValueError("Model A and Model B must be different.")
|
133 |
+
|
134 |
+
if winner_model_id!=model_a_id and winner_model_id!=model_b_id:
|
135 |
+
raise ValueError("Winner model ID must be either model_a_id or model_b_id.")
|
136 |
+
|
137 |
+
# Check duplicate
|
138 |
+
last_battle = self.dao.find_last_battle()
|
139 |
+
if last_battle is not None:
|
140 |
+
if (last_battle.language==language and
|
141 |
+
last_battle.weight_class==weight_class and
|
142 |
+
last_battle.model_a_id==model_a_id and
|
143 |
+
last_battle.model_b_id==model_b_id and
|
144 |
+
last_battle.winner_model_id==winner_model_id and
|
145 |
+
last_battle.user_id==user_id):
|
146 |
+
raise ValueError("Duplicate battle record detected: the latest record already matches the provided battle details.")
|
147 |
+
|
148 |
+
# Check model exists
|
149 |
+
model_a = self.dao.get_model(model_a_id)
|
150 |
+
model_b = self.dao.get_model(model_b_id)
|
151 |
+
if model_a is None or model_b is None:
|
152 |
+
raise ValueError("Both Model A and Model B must exist in the database.")
|
153 |
+
|
154 |
+
battle = Battle(
|
155 |
+
model_a_id=model_a_id,
|
156 |
+
model_b_id=model_b_id,
|
157 |
+
language=language,
|
158 |
+
weight_class=weight_class,
|
159 |
+
winner_model_id=winner_model_id,
|
160 |
+
user_id=user_id,
|
161 |
+
)
|
162 |
+
battle_id = self.dao.insert_battle(battle)
|
163 |
+
return battle_id
|
164 |
+
|
165 |
+
# ---------- LeaderboardEntry ----------
|
166 |
+
|
167 |
+
def get_leaderboard(
|
168 |
+
self,
|
169 |
+
language: str,
|
170 |
+
weight_class: str
|
171 |
+
) -> List[LeaderboardEntry]:
|
172 |
+
return self.dao.find_leaderboard_entries(language, weight_class)
|
173 |
+
|
174 |
+
def update_leaderboard(
|
175 |
+
self,
|
176 |
+
language: str,
|
177 |
+
weight_class: str
|
178 |
+
) -> bool:
|
179 |
+
"""
|
180 |
+
Update the leaderboard using battle history.
|
181 |
+
Each model starts at INITIAL_RATING and K is K_FACTOR.
|
182 |
+
"""
|
183 |
+
models = self.dao.find_models(language, weight_class)
|
184 |
+
if not models:
|
185 |
+
return False
|
186 |
+
|
187 |
+
ratings = {
|
188 |
+
str(model._id): INITIAL_RATING for model in models if model._id is not None
|
189 |
+
}
|
190 |
+
|
191 |
+
battles = self.dao.find_battles(language, weight_class)
|
192 |
+
battles.sort(key=lambda b: b.vote_timestamp)
|
193 |
+
|
194 |
+
for battle in battles:
|
195 |
+
model_a_id_str = str(battle.model_a_id)
|
196 |
+
model_b_id_str = str(battle.model_b_id)
|
197 |
+
|
198 |
+
if model_a_id_str not in ratings:
|
199 |
+
ratings[model_a_id_str] = INITIAL_RATING
|
200 |
+
if model_b_id_str not in ratings:
|
201 |
+
ratings[model_b_id_str] = INITIAL_RATING
|
202 |
+
|
203 |
+
if str(battle.winner_model_id)==model_a_id_str:
|
204 |
+
score_a, score_b = 1, 0
|
205 |
+
else:
|
206 |
+
score_a, score_b = 0, 1
|
207 |
+
|
208 |
+
exp_a = 1 / (1 + 10 ** ((ratings[model_b_id_str] - ratings[model_a_id_str]) / 400))
|
209 |
+
exp_b = 1 / (1 + 10 ** ((ratings[model_a_id_str] - ratings[model_b_id_str]) / 400))
|
210 |
+
|
211 |
+
ratings[model_a_id_str] += K_FACTOR * (score_a - exp_a)
|
212 |
+
ratings[model_b_id_str] += K_FACTOR * (score_b - exp_b)
|
213 |
+
|
214 |
+
for model in models:
|
215 |
+
if model._id is None:
|
216 |
+
continue
|
217 |
+
model_id_str = str(model._id)
|
218 |
+
new_rating = round(ratings.get(model_id_str, INITIAL_RATING))
|
219 |
+
entry = self.dao.find_one_leaderboard_entry(language, weight_class, model._id)
|
220 |
+
if entry:
|
221 |
+
entry.elo_score = new_rating
|
222 |
+
entry.last_updated = datetime.utcnow()
|
223 |
+
self.dao.update_leaderboard_entry(entry)
|
224 |
+
else:
|
225 |
+
new_entry = LeaderboardEntry(
|
226 |
+
model_id=model._id,
|
227 |
+
language=language,
|
228 |
+
weight_class=weight_class,
|
229 |
+
elo_score=new_rating,
|
230 |
+
)
|
231 |
+
self.dao.insert_leaderboard_entry(new_entry)
|
232 |
+
return True
|
indiebot_arena/service/bootstrap_service.py
ADDED
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import logging
|
2 |
+
|
3 |
+
from indiebot_arena.dao.mongo_dao import MongoDAO
|
4 |
+
from indiebot_arena.service.arena_service import ArenaService
|
5 |
+
|
6 |
+
|
7 |
+
class BootstrapService:
|
8 |
+
def __init__(self, dao: MongoDAO):
|
9 |
+
self.dao = dao
|
10 |
+
self.arena_service = ArenaService(dao);
|
11 |
+
|
12 |
+
def provision_database(self):
|
13 |
+
if self.dao.models_collection.count_documents({}) > 0:
|
14 |
+
logging.info("Database already provisioned. Skipping initial data insertion.")
|
15 |
+
return
|
16 |
+
|
17 |
+
try:
|
18 |
+
self.arena_service.register_model("ja", "U-5GB", "google/gemma-3-1b-it", "transformers", "none", "safetensors", 1.86)
|
19 |
+
self.arena_service.register_model("ja", "U-5GB", "google/gemma-2-2b-it", "transformers", "none", "safetensors", 4.87)
|
20 |
+
self.arena_service.register_model("ja", "U-5GB", "google/gemma-2-2b-jpn-it", "transformers", "none", "safetensors", 4.87)
|
21 |
+
self.arena_service.register_model("ja", "U-5GB", "indiebot-community/gemma-3-1b-it-bnb-4bit", "transformers", "bnb", "safetensors", 0.93)
|
22 |
+
self.arena_service.register_model("ja", "U-5GB", "indiebot-community/gemma-2-2b-it-bnb-4bit", "transformers", "bnb", "safetensors", 2.16)
|
23 |
+
self.arena_service.register_model("ja", "U-5GB", "indiebot-community/gemma-2-2b-jpn-it-bnb-4bit", "transformers", "bnb", "safetensors", 2.16)
|
24 |
+
self.arena_service.register_model("ja", "U-10GB", "google/gemma-3-4b-it", "transformers", "none", "safetensors", 8.01)
|
25 |
+
self.arena_service.register_model("ja", "U-10GB", "indiebot-community/gemma-3-4b-it-bnb-4bit", "transformers", "bnb", "safetensors", 2.93)
|
26 |
+
self.arena_service.register_model("ja", "U-10GB", "indiebot-community/gemma-3-12b-it-bnb-4bit", "transformers", "bnb", "safetensors", 7.51)
|
27 |
+
self.arena_service.register_model("ja", "U-10GB", "indiebot-community/gemma-2-9b-it-bnb-4bit", "transformers", "bnb", "safetensors", 6.07)
|
28 |
+
|
29 |
+
self.arena_service.update_leaderboard("ja", "U-5GB")
|
30 |
+
self.arena_service.update_leaderboard("ja", "U-10GB")
|
31 |
+
except Exception as e:
|
32 |
+
logging.error(f"Error register_model: {e}")
|
indiebot_arena/ui/__init__.py
ADDED
File without changes
|
indiebot_arena/ui/battle.py
ADDED
@@ -0,0 +1,234 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import concurrent.futures
|
2 |
+
import hashlib
|
3 |
+
import random
|
4 |
+
import threading
|
5 |
+
|
6 |
+
import gradio as gr
|
7 |
+
import spaces
|
8 |
+
import torch
|
9 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
10 |
+
|
11 |
+
from indiebot_arena.config import LOCAL_TESTING, MODEL_SELECTION_MODE, MAX_INPUT_TOKEN_LENGTH, MAX_NEW_TOKENS
|
12 |
+
from indiebot_arena.service.arena_service import ArenaService
|
13 |
+
|
14 |
+
DESCRIPTION = "### 💬 チャットバトル"
|
15 |
+
|
16 |
+
_model_cache = {}
|
17 |
+
_model_lock = threading.Lock()
|
18 |
+
|
19 |
+
|
20 |
+
# Avoid “cannot copy out of meta tensor” error
|
21 |
+
def get_cached_model_and_tokenizer(model_id: str):
|
22 |
+
with _model_lock:
|
23 |
+
if model_id not in _model_cache:
|
24 |
+
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
25 |
+
model = AutoModelForCausalLM.from_pretrained(
|
26 |
+
model_id,
|
27 |
+
device_map="auto",
|
28 |
+
torch_dtype=torch.bfloat16,
|
29 |
+
use_safetensors=True
|
30 |
+
)
|
31 |
+
model.eval()
|
32 |
+
_model_cache[model_id] = (model, tokenizer)
|
33 |
+
return _model_cache[model_id]
|
34 |
+
|
35 |
+
|
36 |
+
@spaces.GPU(duration=30)
|
37 |
+
def generate(message: str, chat_history: list, model_id: str, max_new_tokens: int = MAX_NEW_TOKENS,
|
38 |
+
temperature: float = 0.6, top_p: float = 0.9, top_k: int = 50, repetition_penalty: float = 1.2) -> str:
|
39 |
+
if LOCAL_TESTING:
|
40 |
+
model, tokenizer = get_cached_model_and_tokenizer(model_id)
|
41 |
+
else:
|
42 |
+
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
43 |
+
model = AutoModelForCausalLM.from_pretrained(
|
44 |
+
model_id,
|
45 |
+
device_map="auto",
|
46 |
+
torch_dtype=torch.bfloat16,
|
47 |
+
use_safetensors=True
|
48 |
+
)
|
49 |
+
|
50 |
+
conversation = chat_history.copy()
|
51 |
+
conversation.append({"role": "user", "content": message})
|
52 |
+
input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
|
53 |
+
if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
|
54 |
+
input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
|
55 |
+
gr.Warning(f"Trimmed input as it exceeded {MAX_INPUT_TOKEN_LENGTH} tokens.")
|
56 |
+
input_ids = input_ids.to(model.device)
|
57 |
+
outputs = model.generate(
|
58 |
+
input_ids=input_ids,
|
59 |
+
max_new_tokens=max_new_tokens,
|
60 |
+
do_sample=True,
|
61 |
+
top_p=top_p,
|
62 |
+
top_k=top_k,
|
63 |
+
temperature=temperature,
|
64 |
+
repetition_penalty=repetition_penalty,
|
65 |
+
)
|
66 |
+
response = tokenizer.decode(outputs[0][input_ids.shape[1]:], skip_special_tokens=True)
|
67 |
+
return response
|
68 |
+
|
69 |
+
|
70 |
+
def format_chat_history(history):
|
71 |
+
conversation = []
|
72 |
+
for user_msg, assistant_msg in history:
|
73 |
+
conversation.append({"role": "user", "content": user_msg})
|
74 |
+
if assistant_msg:
|
75 |
+
conversation.append({"role": "assistant", "content": assistant_msg})
|
76 |
+
return conversation
|
77 |
+
|
78 |
+
|
79 |
+
def submit_message(message, history_a, history_b, model_a, model_b):
|
80 |
+
history_a.append((message, ""))
|
81 |
+
history_b.append((message, ""))
|
82 |
+
conv_history_a = format_chat_history(history_a[:-1])
|
83 |
+
conv_history_b = format_chat_history(history_b[:-1])
|
84 |
+
with concurrent.futures.ThreadPoolExecutor() as executor:
|
85 |
+
future_a = executor.submit(generate, message, conv_history_a, model_a)
|
86 |
+
future_b = executor.submit(generate, message, conv_history_b, model_b)
|
87 |
+
response_a = future_a.result()
|
88 |
+
response_b = future_b.result()
|
89 |
+
history_a[-1] = (message, response_a)
|
90 |
+
history_b[-1] = (message, response_b)
|
91 |
+
return history_a, history_b, "", gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=False)
|
92 |
+
|
93 |
+
|
94 |
+
def get_random_values(model_labels):
|
95 |
+
if MODEL_SELECTION_MODE=="random":
|
96 |
+
return random.sample(model_labels, 2)
|
97 |
+
if MODEL_SELECTION_MODE=="manual":
|
98 |
+
return model_labels[0], model_labels[0]
|
99 |
+
|
100 |
+
def battle_content(dao, language):
|
101 |
+
arena_service = ArenaService(dao)
|
102 |
+
default_weight = "U-5GB"
|
103 |
+
initial_models = arena_service.get_model_dropdown_list(language, default_weight)
|
104 |
+
initial_choices = [m["label"] for m in initial_models]
|
105 |
+
initial_value_a, initial_value_b = get_random_values(initial_choices)
|
106 |
+
dropdown_visible = False if MODEL_SELECTION_MODE=="random" else True
|
107 |
+
|
108 |
+
def fetch_model_dropdown(weight_class):
|
109 |
+
models = arena_service.get_model_dropdown_list(language, weight_class)
|
110 |
+
model_labels = [m["label"] for m in models]
|
111 |
+
value_a, value_b = get_random_values(model_labels)
|
112 |
+
update_obj_a = gr.update(choices=model_labels, value=value_a)
|
113 |
+
update_obj_b = gr.update(choices=model_labels, value=value_b)
|
114 |
+
return update_obj_a, update_obj_b, model_labels
|
115 |
+
|
116 |
+
def submit_vote(vote_choice, weight_class, model_a_name, model_b_name, request: gr.Request):
|
117 |
+
user_id = generate_anonymous_user_id(request)
|
118 |
+
model_a = arena_service.get_one_model(language, weight_class, model_a_name)
|
119 |
+
model_b = arena_service.get_one_model(language, weight_class, model_b_name)
|
120 |
+
winner = model_a if vote_choice=="Chatbot A" else model_b
|
121 |
+
try:
|
122 |
+
arena_service.record_battle(language, weight_class, model_a._id, model_b._id, winner._id, user_id)
|
123 |
+
arena_service.update_leaderboard(language, weight_class)
|
124 |
+
return "投票が完了しました"
|
125 |
+
except Exception as e:
|
126 |
+
return f"エラー: {e}"
|
127 |
+
|
128 |
+
def handle_vote(vote_choice, weight_class, model_a_name, model_b_name, request: gr.Request):
|
129 |
+
msg = submit_vote(vote_choice, weight_class, model_a_name, model_b_name, request)
|
130 |
+
return (
|
131 |
+
gr.update(value=msg, visible=True),
|
132 |
+
gr.update(interactive=False, value=model_a_name),
|
133 |
+
gr.update(interactive=False, value=model_b_name),
|
134 |
+
gr.update(visible=False),
|
135 |
+
gr.update(visible=True, interactive=True)
|
136 |
+
)
|
137 |
+
|
138 |
+
def generate_anonymous_user_id(request: gr.Request):
|
139 |
+
user_ip = request.headers.get('x-forwarded-for')
|
140 |
+
if user_ip:
|
141 |
+
user_id = "indiebot:" + user_ip
|
142 |
+
hashed_user_id = hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:16]
|
143 |
+
return hashed_user_id
|
144 |
+
else:
|
145 |
+
return "anonymous"
|
146 |
+
|
147 |
+
def on_vote_a_click(weight, a, b, request: gr.Request):
|
148 |
+
return handle_vote("Chatbot A", weight, a, b, request)
|
149 |
+
|
150 |
+
def on_vote_b_click(weight, a, b, request: gr.Request):
|
151 |
+
return handle_vote("Chatbot B", weight, a, b, request)
|
152 |
+
|
153 |
+
def reset_battle(dropdown_options):
|
154 |
+
value_a, value_b = get_random_values(dropdown_options)
|
155 |
+
return (
|
156 |
+
[], # chatbot_aのリセット
|
157 |
+
[], # chatbot_bのリセット
|
158 |
+
gr.update(value="", visible=True), # user_inputのクリア&表示
|
159 |
+
gr.update(interactive=False, value="A is better"), # vote_a_btnのリセット
|
160 |
+
gr.update(interactive=False, value="B is better"), # vote_b_btnのリセット
|
161 |
+
gr.update(visible=False), # vote_messageの非表示
|
162 |
+
gr.update(visible=False, interactive=False), # next_battle_btnの非表示
|
163 |
+
gr.update(choices=dropdown_options, value=value_a), # model_dropdown_a更新
|
164 |
+
gr.update(choices=dropdown_options, value=value_b), # model_dropdown_b更新
|
165 |
+
gr.update(interactive=True) # weight_class_radio を有効化
|
166 |
+
)
|
167 |
+
|
168 |
+
with gr.Blocks(css="style.css") as battle_ui:
|
169 |
+
gr.Markdown(DESCRIPTION)
|
170 |
+
weight_class_radio = gr.Radio(
|
171 |
+
choices=["U-5GB", "U-10GB"],
|
172 |
+
label="階級",
|
173 |
+
value=default_weight
|
174 |
+
)
|
175 |
+
dropdown_options_state = gr.State(initial_choices)
|
176 |
+
with gr.Row():
|
177 |
+
model_dropdown_a = gr.Dropdown(
|
178 |
+
choices=initial_choices,
|
179 |
+
label="モデルAを選択",
|
180 |
+
value=initial_value_a,
|
181 |
+
visible=dropdown_visible
|
182 |
+
)
|
183 |
+
model_dropdown_b = gr.Dropdown(
|
184 |
+
choices=initial_choices,
|
185 |
+
label="モデルBを選択",
|
186 |
+
value=initial_value_b,
|
187 |
+
visible=dropdown_visible
|
188 |
+
)
|
189 |
+
weight_class_radio.change(
|
190 |
+
fn=fetch_model_dropdown,
|
191 |
+
inputs=weight_class_radio,
|
192 |
+
outputs=[model_dropdown_a, model_dropdown_b, dropdown_options_state]
|
193 |
+
)
|
194 |
+
with gr.Row():
|
195 |
+
chatbot_a = gr.Chatbot(label="Chatbot A")
|
196 |
+
chatbot_b = gr.Chatbot(label="Chatbot B")
|
197 |
+
with gr.Row():
|
198 |
+
vote_a_btn = gr.Button("A is better", interactive=False)
|
199 |
+
vote_b_btn = gr.Button("B is better", interactive=False)
|
200 |
+
user_input = gr.Textbox(
|
201 |
+
placeholder="日本語でメッセージを入力...",
|
202 |
+
submit_btn=True,
|
203 |
+
show_label=False
|
204 |
+
)
|
205 |
+
with gr.Row():
|
206 |
+
with gr.Column(scale=3):
|
207 |
+
vote_message = gr.Textbox(show_label=False, interactive=False, visible=False)
|
208 |
+
with gr.Column(scale=1):
|
209 |
+
next_battle_btn = gr.Button("次のバトルへ", interactive=False, visible=False, elem_id="next_battle_btn")
|
210 |
+
user_input.submit(
|
211 |
+
fn=submit_message,
|
212 |
+
inputs=[user_input, chatbot_a, chatbot_b, model_dropdown_a, model_dropdown_b],
|
213 |
+
outputs=[chatbot_a, chatbot_b, user_input, vote_a_btn, vote_b_btn, weight_class_radio]
|
214 |
+
)
|
215 |
+
vote_a_btn.click(
|
216 |
+
fn=on_vote_a_click,
|
217 |
+
inputs=[weight_class_radio, model_dropdown_a, model_dropdown_b],
|
218 |
+
outputs=[vote_message, vote_a_btn, vote_b_btn, user_input, next_battle_btn]
|
219 |
+
)
|
220 |
+
vote_b_btn.click(
|
221 |
+
fn=on_vote_b_click,
|
222 |
+
inputs=[weight_class_radio, model_dropdown_a, model_dropdown_b],
|
223 |
+
outputs=[vote_message, vote_a_btn, vote_b_btn, user_input, next_battle_btn]
|
224 |
+
)
|
225 |
+
next_battle_btn.click(
|
226 |
+
fn=reset_battle,
|
227 |
+
inputs=[dropdown_options_state],
|
228 |
+
outputs=[
|
229 |
+
chatbot_a, chatbot_b, user_input, vote_a_btn,
|
230 |
+
vote_b_btn, vote_message, next_battle_btn,
|
231 |
+
model_dropdown_a, model_dropdown_b, weight_class_radio
|
232 |
+
]
|
233 |
+
)
|
234 |
+
return battle_ui
|
indiebot_arena/ui/leaderboard.py
ADDED
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
|
3 |
+
import gradio as gr
|
4 |
+
import pandas as pd
|
5 |
+
|
6 |
+
from indiebot_arena.service.arena_service import ArenaService
|
7 |
+
|
8 |
+
DESCRIPTION = "### 🏆️ リーダーボード"
|
9 |
+
|
10 |
+
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
11 |
+
docs_path = os.path.join(base_dir, "docs", "leaderboard_header.md")
|
12 |
+
|
13 |
+
def leaderboard_content(dao, language):
|
14 |
+
arena_service = ArenaService(dao)
|
15 |
+
|
16 |
+
def fetch_leaderboard_data(weight_class):
|
17 |
+
entries = arena_service.get_leaderboard(language, weight_class)
|
18 |
+
data = []
|
19 |
+
for entry in entries:
|
20 |
+
model_obj = arena_service.dao.get_model(entry.model_id)
|
21 |
+
model_name = model_obj.model_name if model_obj else "Unknown"
|
22 |
+
file_size = model_obj.file_size_gb if model_obj else "N/A"
|
23 |
+
desc = model_obj.description if model_obj else ""
|
24 |
+
last_updated = entry.last_updated.strftime("%Y-%m-%d %H:%M:%S")
|
25 |
+
data.append([model_name, entry.elo_score, file_size, desc, last_updated])
|
26 |
+
if not data:
|
27 |
+
data = [["No data available", "", "", "", ""]]
|
28 |
+
df = pd.DataFrame(data, columns=["Model Name", "Elo Score", "File Size (GB)", "Description", "Last Updated"])
|
29 |
+
df.insert(0, "Rank", range(1, len(df) + 1))
|
30 |
+
return df
|
31 |
+
|
32 |
+
initial_weight_class = "U-5GB"
|
33 |
+
with gr.Blocks(css="style.css") as leaderboard_ui:
|
34 |
+
with open(docs_path, "r", encoding="utf-8") as f:
|
35 |
+
markdown_content = f.read()
|
36 |
+
gr.Markdown(markdown_content)
|
37 |
+
gr.Markdown(DESCRIPTION)
|
38 |
+
weight_class_radio = gr.Radio(choices=["U-5GB", "U-10GB"], label="階級", value=initial_weight_class)
|
39 |
+
leaderboard_table = gr.Dataframe(
|
40 |
+
headers=["Rank", "Model Name", "Elo Score", "File Size (GB)", "Description", "Last Updated"],
|
41 |
+
value=fetch_leaderboard_data(initial_weight_class),
|
42 |
+
interactive=False
|
43 |
+
)
|
44 |
+
refresh_btn = gr.Button("更新")
|
45 |
+
weight_class_radio.change(fn=fetch_leaderboard_data, inputs=weight_class_radio, outputs=leaderboard_table)
|
46 |
+
refresh_btn.click(fn=fetch_leaderboard_data, inputs=weight_class_radio, outputs=leaderboard_table)
|
47 |
+
return leaderboard_ui
|
indiebot_arena/ui/registration.py
ADDED
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import re
|
3 |
+
from dataclasses import dataclass
|
4 |
+
|
5 |
+
import gradio as gr
|
6 |
+
import spaces
|
7 |
+
import torch
|
8 |
+
from huggingface_hub import hf_hub_url, get_hf_file_metadata, model_info
|
9 |
+
from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
|
10 |
+
|
11 |
+
from indiebot_arena.service.arena_service import ArenaService
|
12 |
+
from indiebot_arena.ui.battle import generate
|
13 |
+
|
14 |
+
DESCRIPTION = "### 📚️ 登録済みモデル"
|
15 |
+
|
16 |
+
base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
17 |
+
docs_path = os.path.join(base_dir, "docs", "model_registration_guide.md")
|
18 |
+
|
19 |
+
|
20 |
+
@dataclass
|
21 |
+
class ModelMeta:
|
22 |
+
model_id: str
|
23 |
+
architecture: list
|
24 |
+
parameters: str
|
25 |
+
model_type: str
|
26 |
+
weights_file_size: float
|
27 |
+
weights_format: str
|
28 |
+
quantization: str
|
29 |
+
|
30 |
+
|
31 |
+
def format_model_meta(meta: ModelMeta) -> str:
|
32 |
+
return (
|
33 |
+
f"Model ID: {meta.model_id}\n"
|
34 |
+
f"Architecture: {meta.architecture}\n"
|
35 |
+
f"Parameters: {meta.parameters}\n"
|
36 |
+
f"Model Type: {meta.model_type}\n"
|
37 |
+
f"Weights File Size: {meta.weights_file_size} GB\n"
|
38 |
+
f"Weights Format: {meta.weights_format}\n"
|
39 |
+
f"Quantization: {meta.quantization}"
|
40 |
+
)
|
41 |
+
|
42 |
+
|
43 |
+
@spaces.GPU(duration=30)
|
44 |
+
def get_model_meta(model_id: str):
|
45 |
+
try:
|
46 |
+
config = AutoConfig.from_pretrained(model_id)
|
47 |
+
model = AutoModelForCausalLM.from_pretrained(
|
48 |
+
model_id,
|
49 |
+
device_map="auto",
|
50 |
+
torch_dtype=torch.bfloat16,
|
51 |
+
use_safetensors=True
|
52 |
+
)
|
53 |
+
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
54 |
+
repo_info = model_info(model_id)
|
55 |
+
weights_files = [file.rfilename for file in repo_info.siblings if
|
56 |
+
file.rfilename.endswith('.bin') or file.rfilename.endswith('.safetensors')]
|
57 |
+
total_size = 0
|
58 |
+
file_formats = set()
|
59 |
+
for file_name in weights_files:
|
60 |
+
file_formats.add(file_name.split('.')[-1])
|
61 |
+
file_url = hf_hub_url(model_id, filename=file_name)
|
62 |
+
metadata = get_hf_file_metadata(file_url)
|
63 |
+
total_size += metadata.size or 0
|
64 |
+
total_size_gb = round(total_size / (1024 ** 3), 2)
|
65 |
+
model_type = type(model).__name__
|
66 |
+
quant_config = getattr(config, 'quantization_config', {})
|
67 |
+
quant_method = quant_config.get('quant_method', 'none')
|
68 |
+
meta = ModelMeta(
|
69 |
+
model_id=model_id,
|
70 |
+
architecture=config.architectures if hasattr(config, 'architectures') else [],
|
71 |
+
parameters=f"{round(sum(p.numel() for p in model.parameters()) / 1e9, 2)} Billion",
|
72 |
+
model_type=model_type,
|
73 |
+
weights_file_size=total_size_gb,
|
74 |
+
weights_format=", ".join(file_formats),
|
75 |
+
quantization=quant_method,
|
76 |
+
)
|
77 |
+
return meta
|
78 |
+
except Exception as e:
|
79 |
+
return f"Error: {str(e)}"
|
80 |
+
|
81 |
+
|
82 |
+
def registration_content(dao, language):
|
83 |
+
arena_service = ArenaService(dao)
|
84 |
+
|
85 |
+
def fetch_models(weight_class):
|
86 |
+
models = arena_service.dao.find_models(language, weight_class)
|
87 |
+
data = []
|
88 |
+
for m in models:
|
89 |
+
created_at_str = m.created_at.strftime("%Y-%m-%d %H:%M:%S") if m.created_at else ""
|
90 |
+
data.append([m.weight_class, m.model_name, m.runtime, m.quantization, m.file_format, m.file_size_gb,
|
91 |
+
m.description or "", created_at_str])
|
92 |
+
return data
|
93 |
+
|
94 |
+
def load_test(model_id, weight_class, current_output):
|
95 |
+
if not re.match(r'^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$', model_id):
|
96 |
+
err = "Error: Invalid model_id format. It must be in the format 'owner/model'."
|
97 |
+
return (current_output + "\n" + err, gr.update(interactive=False), None)
|
98 |
+
meta = get_model_meta(model_id)
|
99 |
+
if isinstance(meta, str) and meta.startswith("Error:"):
|
100 |
+
return (current_output + "\n" + meta, gr.update(interactive=False), None)
|
101 |
+
if weight_class=="U-5GB" and meta.weights_file_size >= 5.0:
|
102 |
+
err = f"Error: File size exceeds U-5GB limit. {meta.weights_file_size} GB"
|
103 |
+
return (current_output + "\n" + err, gr.update(interactive=False), None)
|
104 |
+
if weight_class=="U-10GB" and meta.weights_file_size >= 10.0:
|
105 |
+
err = f"Error: File size exceeds U-10GB limit. {meta.weights_file_size} GB"
|
106 |
+
return (current_output + "\n" + err, gr.update(interactive=False), None)
|
107 |
+
if "safetensors" not in meta.weights_format.lower():
|
108 |
+
err = "Error: Weights Format must include safetensors."
|
109 |
+
return (current_output + "\n" + err, gr.update(interactive=False), None)
|
110 |
+
if meta.quantization.lower() not in ["bitsandbytes", "none"]:
|
111 |
+
err = "Error: Quantization must be bitsandbytes or none."
|
112 |
+
return (current_output + "\n" + err, gr.update(interactive=False), None)
|
113 |
+
display_str = format_model_meta(meta) + "\n\nロードテストが成功しました。\nチャットテストを実施して下さい。\n"
|
114 |
+
return (current_output + "\n" + display_str, gr.update(interactive=True), meta)
|
115 |
+
|
116 |
+
def chat_test(model_id, current_output):
|
117 |
+
question = "日本の首都は?"
|
118 |
+
expected_word = "東京"
|
119 |
+
try:
|
120 |
+
response = generate(question, [], model_id)
|
121 |
+
except Exception as e:
|
122 |
+
error_msg = f"エラー: {str(e)}"
|
123 |
+
return (current_output + "\n" + error_msg, gr.update(interactive=False))
|
124 |
+
|
125 |
+
result_text = f"質問: {question}\n応答: {response}\n"
|
126 |
+
if expected_word in response:
|
127 |
+
result_text += "成功: 応答に期待するワードが含まれています。\nモデル登録を実施して下さい。\n"
|
128 |
+
register_enabled = True
|
129 |
+
else:
|
130 |
+
result_text += "失敗: 応答に期待するワードが含まれていません。\n"
|
131 |
+
register_enabled = False
|
132 |
+
return (current_output + "\n" + result_text, gr.update(interactive=register_enabled))
|
133 |
+
|
134 |
+
def register_model(meta, weight_class, description, current_output):
|
135 |
+
try:
|
136 |
+
if meta is None:
|
137 |
+
raise ValueError("No meta info available.")
|
138 |
+
model_id_extracted = meta.model_id
|
139 |
+
file_size_gb = meta.weights_file_size
|
140 |
+
quantization = "bnb" if meta.quantization.lower()=="bitsandbytes" else "none"
|
141 |
+
weights_format = meta.weights_format
|
142 |
+
file_format = "safetensors" if "safetensors" in weights_format.lower() else weights_format
|
143 |
+
desc = description if description else ""
|
144 |
+
arena_service.register_model(language, weight_class, model_id_extracted, "transformers", quantization, file_format, file_size_gb, desc)
|
145 |
+
arena_service.update_leaderboard(language, weight_class)
|
146 |
+
result = "モデルの登録が完了しました。"
|
147 |
+
disable = True
|
148 |
+
except Exception as e:
|
149 |
+
result = f"エラー: {str(e)}"
|
150 |
+
disable = False
|
151 |
+
new_output = current_output + "\n" + result
|
152 |
+
if disable:
|
153 |
+
btn_update = gr.update(interactive=False)
|
154 |
+
else:
|
155 |
+
btn_update = gr.update()
|
156 |
+
return new_output, meta, btn_update, btn_update, btn_update
|
157 |
+
|
158 |
+
def clear_all():
|
159 |
+
initial_weight = "U-5GB"
|
160 |
+
return "", initial_weight, "", "", gr.update(interactive=True), gr.update(interactive=False), gr.update(interactive=False), None
|
161 |
+
|
162 |
+
with gr.Blocks(css="style.css") as ui:
|
163 |
+
gr.Markdown(DESCRIPTION)
|
164 |
+
weight_class_radio = gr.Radio(choices=["U-5GB", "U-10GB"], label="階級", value="U-5GB")
|
165 |
+
mdl_list = gr.Dataframe(
|
166 |
+
headers=["Weight Class", "Model Name", "Runtime", "Quantization", "Weights Format",
|
167 |
+
"Weights File Size", "Description", "Created At"],
|
168 |
+
value=fetch_models("U-5GB"),
|
169 |
+
interactive=False
|
170 |
+
)
|
171 |
+
weight_class_radio.change(fn=fetch_models, inputs=weight_class_radio, outputs=mdl_list)
|
172 |
+
with gr.Accordion("🔰 モデルの登録ガイド", open=False):
|
173 |
+
with open(docs_path, "r", encoding="utf-8") as f:
|
174 |
+
markdown_content = f.read()
|
175 |
+
gr.Markdown(markdown_content)
|
176 |
+
with gr.Accordion("🤖 モデルの新規登録", open=False):
|
177 |
+
model_id_input = gr.Textbox(label="モデルID", max_lines=1)
|
178 |
+
reg_weight_class_radio = gr.Radio(choices=["U-5GB", "U-10GB"], label="階級", value="U-5GB")
|
179 |
+
description_input = gr.Textbox(label="Description (オプション)", placeholder="任意の説明を入力", max_lines=1, visible=False)
|
180 |
+
output_box = gr.Textbox(label="結果出力", lines=10)
|
181 |
+
meta_state = gr.State(None)
|
182 |
+
with gr.Row():
|
183 |
+
test_btn = gr.Button("ロードテスト")
|
184 |
+
chat_test_btn = gr.Button("チャットテスト", interactive=False)
|
185 |
+
register_btn = gr.Button("モデル登録", interactive=False)
|
186 |
+
clear_btn = gr.Button("クリア")
|
187 |
+
test_btn.click(fn=load_test, inputs=[model_id_input, reg_weight_class_radio, output_box], outputs=[output_box,
|
188 |
+
chat_test_btn,
|
189 |
+
meta_state])
|
190 |
+
chat_test_btn.click(fn=chat_test, inputs=[model_id_input, output_box], outputs=[output_box, register_btn])
|
191 |
+
register_btn.click(fn=register_model, inputs=[meta_state, reg_weight_class_radio, description_input,
|
192 |
+
output_box], outputs=[output_box, meta_state, test_btn,
|
193 |
+
chat_test_btn, register_btn])
|
194 |
+
clear_btn.click(fn=clear_all, inputs=[], outputs=[model_id_input, reg_weight_class_radio, description_input,
|
195 |
+
output_box, test_btn, chat_test_btn, register_btn, meta_state])
|
196 |
+
return ui
|
requirements.txt
CHANGED
@@ -1 +1,11 @@
|
|
1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate==1.2.1
|
2 |
+
bitsandbytes==0.44.1
|
3 |
+
gradio==5.12.0
|
4 |
+
huggingface-hub==0.27.1
|
5 |
+
pandas==2.2.3
|
6 |
+
peft==0.13.2
|
7 |
+
pydantic==2.10.6
|
8 |
+
pymongo[srv]==4.11.3
|
9 |
+
spaces==0.32.0
|
10 |
+
torch==2.4.0
|
11 |
+
transformers==4.50.0
|
style.css
ADDED
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
h1 {
|
2 |
+
font-size: 2.5rem;
|
3 |
+
font-weight: 800;
|
4 |
+
text-align: left;
|
5 |
+
margin-bottom: 1rem;
|
6 |
+
line-height: 1.2;
|
7 |
+
}
|
8 |
+
|
9 |
+
#duplicate-button {
|
10 |
+
margin: auto;
|
11 |
+
color: #fff;
|
12 |
+
background: #1565c0;
|
13 |
+
border-radius: 100vh;
|
14 |
+
}
|
15 |
+
|
16 |
+
#next_battle_btn {
|
17 |
+
margin-top: auto;
|
18 |
+
height: 4rem;
|
19 |
+
}
|