fukugawa commited on
Commit
f0e4d3b
·
1 Parent(s): 8b3893f

1st draft release

Browse files
.gitignore ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ #uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ #poetry.lock
109
+
110
+ # pdm
111
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
112
+ #pdm.lock
113
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
114
+ # in version control.
115
+ # https://pdm.fming.dev/latest/usage/project/#working-with-version-control
116
+ .pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
121
+ __pypackages__/
122
+
123
+ # Celery stuff
124
+ celerybeat-schedule
125
+ celerybeat.pid
126
+
127
+ # SageMath parsed files
128
+ *.sage.py
129
+
130
+ # Environments
131
+ .env
132
+ .venv
133
+ env/
134
+ venv/
135
+ ENV/
136
+ env.bak/
137
+ venv.bak/
138
+
139
+ # Spyder project settings
140
+ .spyderproject
141
+ .spyproject
142
+
143
+ # Rope project settings
144
+ .ropeproject
145
+
146
+ # mkdocs documentation
147
+ /site
148
+
149
+ # mypy
150
+ .mypy_cache/
151
+ .dmypy.json
152
+ dmypy.json
153
+
154
+ # Pyre type checker
155
+ .pyre/
156
+
157
+ # pytype static type analyzer
158
+ .pytype/
159
+
160
+ # Cython debug symbols
161
+ cython_debug/
162
+
163
+ # PyCharm
164
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
165
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
166
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
167
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
168
+ #.idea/
169
+
170
+ # Ruff stuff:
171
+ .ruff_cache/
172
+
173
+ # PyPI configuration file
174
+ .pypirc
175
+
176
+ .idea
177
+ .DS_Store
app.py CHANGED
@@ -1,64 +1,27 @@
1
  import gradio as gr
2
- from huggingface_hub import InferenceClient
3
 
4
- """
5
- For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
6
- """
7
- client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
8
-
9
-
10
- def respond(
11
- message,
12
- history: list[tuple[str, str]],
13
- system_message,
14
- max_tokens,
15
- temperature,
16
- top_p,
17
- ):
18
- messages = [{"role": "system", "content": system_message}]
19
-
20
- for val in history:
21
- if val[0]:
22
- messages.append({"role": "user", "content": val[0]})
23
- if val[1]:
24
- messages.append({"role": "assistant", "content": val[1]})
25
-
26
- messages.append({"role": "user", "content": message})
27
-
28
- response = ""
29
-
30
- for message in client.chat_completion(
31
- messages,
32
- max_tokens=max_tokens,
33
- stream=True,
34
- temperature=temperature,
35
- top_p=top_p,
36
- ):
37
- token = message.choices[0].delta.content
38
-
39
- response += token
40
- yield response
41
-
42
-
43
- """
44
- For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
45
- """
46
- demo = gr.ChatInterface(
47
- respond,
48
- additional_inputs=[
49
- gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
50
- gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
51
- gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
52
- gr.Slider(
53
- minimum=0.1,
54
- maximum=1.0,
55
- value=0.95,
56
- step=0.05,
57
- label="Top-p (nucleus sampling)",
58
- ),
59
- ],
60
- )
61
-
62
-
63
- if __name__ == "__main__":
64
- demo.launch()
 
1
  import gradio as gr
 
2
 
3
+ from indiebot_arena.config import MONGO_DB_URI, MONGO_DB_NAME, LANGUAGE
4
+ from indiebot_arena.dao.mongo_dao import MongoDAO
5
+ from indiebot_arena.service.bootstrap_service import BootstrapService
6
+ from indiebot_arena.ui.battle import battle_content
7
+ from indiebot_arena.ui.leaderboard import leaderboard_content
8
+ from indiebot_arena.ui.registration import registration_content
9
+
10
+ dao = MongoDAO(MONGO_DB_URI, MONGO_DB_NAME)
11
+ bootstrap_service = BootstrapService(dao)
12
+ bootstrap_service.provision_database()
13
+
14
+ with open("style.css", "r") as f:
15
+ custom_css = f.read()
16
+
17
+ with gr.Blocks(css=custom_css) as demo:
18
+ with gr.Tabs():
19
+ with gr.TabItem("🏆 リーダーボード"):
20
+ leaderboard_content(dao, LANGUAGE)
21
+ with gr.TabItem("⚔️ モデルに投票"):
22
+ battle_content(dao, LANGUAGE)
23
+ with gr.TabItem("📚️ モデルの登録"):
24
+ registration_content(dao, LANGUAGE)
25
+
26
+ if __name__=="__main__":
27
+ demo.queue(max_size=20).launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docs/leaderboard_header.md ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚀 IndieBot Arena
2
+
3
+ インディーボットアリーナはChatbot Arenaのインディーズ版のようなWebアプリです。
4
+ Chatbot Arenaにインスパイアされて開発されましが、以下のような違いがあります。
5
+
6
+ - 誰でもモデルを登録してコンペに参加可能
7
+ - 階級別にチャットバトルとリーダーボードがある
8
+
9
+ 初期データとして、Google公式のInstruction-Tuning済みモデルを登録しています。
10
+ 【モデルに投票】から匿名のチャットバトルで投票することが出来ます。
11
+ 【モデルの登録】から自分の事後学習済みモデルを登録することも出来ます。
12
+
13
+ Google公式モデルを上回る日本語性能スコアのモデルをぜひ皆で作りましょう!
14
+
15
+
16
+
docs/model_registration_guide.md ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## 登録可能なモデルの要件
2
+
3
+ 登録可能なモデルは、Hugging Faceのモデルハブで公開されているPyTorchベースのモデルで、transformersライブラリのgenerate APIに対応している必要があります。
4
+ 重みのファイル形式はsafetensorsのみで、量子化する場合はBitsAndBytesのバージョン0.44.1以降で量子化されている必要があります。
5
+ tokenizer_config.jsonで正しくchat_templateが設定されている必要があります。
6
+ パラメータ数ではなく、重みのファイルのサイズが5GB未満または10GB未満で階級ごとに分かれてリーダーボードが作れます。
7
+
8
+ - **GPU環境**: HF SpacesのZeroGPU A100(40GB)
9
+ - **LLM実行環境**: transformersライブラリ(4.50.0)
10
+ - **量子化**: BitsAndBytesのみ対応(0.44.1)
11
+ - **ファイル形式**: safetensorsのみ
12
+ - **ファイルサイズ**: 5GB又は10GB未満
13
+ - **チャットテンプレート**: chat_template設定が必要(tokenizer_config.json)
14
+
15
+ 非量子化モデルでもファイルサイズ制限をクリアすれば登録可能ですが、サーバーの負荷低減のためにBitsAndBytesによる量子化を推奨します。
16
+ 将来的には、llama.cppの実行環境でGGUF形式に対応する予定です。
17
+
18
+ ## モデルの登録方法
19
+
20
+ このページの一番下にある、モデルの新規登録フォームから登録できます。
21
+ モデルIDとエントリーしたいファイルサイズ区分を選択してロードテストボタンを押して下さい。
22
+
23
+ ### ① ロードテスト
24
+ ロードテストではファイル形式やサイズなどのチェックが行われます。
25
+
26
+ ### ② チャットテスト
27
+ チャットテストでは簡単な日本語応答が出来るかチェックが行われます。
28
+
29
+ ### ③ 登録
30
+ ロードテストとチャットテストをクリアしたモデルだけ登録可能です。
31
+ 登録が完了すると、登録済みモデル一覧にあなたのモデルが表示されます。
32
+
33
+ ## 量子化サンプルコード
34
+
35
+ 以下はBitsAndBytesで4bit量子化して自分のリポジトリにPushするまでのサンプルコードです。
36
+
37
+ ```
38
+ # python 3.10
39
+ pip install bitsandbytes==0.44.1
40
+ pip install accelerate==1.2.1
41
+ pip install transformers==4.50.0
42
+ pip install huggingface_hub[cli]
43
+ ```
44
+ ```
45
+ # アクセストークンを入力してログイン
46
+ huggingface-cli login
47
+ ```
48
+ ```python
49
+ import torch
50
+ from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
51
+
52
+ model_id = "google/gemma-2-2b-it"
53
+ repo_id = "xxxxx/gemma-2-2b-it-bnb-4bit"
54
+
55
+ bnb_config = BitsAndBytesConfig(
56
+ load_in_4bit=True,
57
+ bnb_4bit_quant_type="nf4",
58
+ bnb_4bit_compute_dtype=torch.bfloat16
59
+ )
60
+
61
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
62
+ model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map="auto")
63
+
64
+ tokenizer.push_to_hub(repo_id)
65
+ model.push_to_hub(repo_id)
66
+
67
+ ```
68
+
69
+ 量子化後のモデルIDは任意の名前が可能ですが、以下の形式を推奨します。
70
+
71
+ * BitsAndBytesの4bit量子化の場合
72
+ ```
73
+ [量子化前のモデルID]-bnb-4bit
74
+ ```
indiebot_arena/__init__.py ADDED
File without changes
indiebot_arena/config.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ MONGO_DB_URI = os.environ.get("MONGO_DB_URI", "mongodb://localhost:27017")
4
+ MONGO_DB_NAME = os.environ.get("MONGO_DB_NAME", "test_db")
5
+ LANGUAGE = "ja"
6
+
7
+ DEBUG = os.getenv("DEBUG", "False").lower() in ["true", "1", "yes"]
8
+ LOCAL_TESTING = os.getenv("LOCAL_TESTING", "False").lower() in ["true", "1", "yes"]
9
+ MODEL_SELECTION_MODE = os.getenv("MODEL_SELECTION_MODE", "random")
10
+ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
11
+
12
+ if LOCAL_TESTING:
13
+ MAX_NEW_TOKENS = 20
14
+ else:
15
+ MAX_NEW_TOKENS = 512
indiebot_arena/dao/__init__.py ADDED
File without changes
indiebot_arena/dao/mongo_dao.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import asdict
2
+ from typing import List, Optional
3
+
4
+ from bson import ObjectId
5
+ from pymongo import MongoClient
6
+
7
+ from indiebot_arena.model.domain_model import Model, Battle, LeaderboardEntry
8
+
9
+
10
+ class MongoDAO:
11
+ def __init__(self, uri: str, db_name: str):
12
+ self.client = MongoClient(uri)
13
+ self.db = self.client[db_name]
14
+ self.models_collection = self.db["models"]
15
+ self.battles_collection = self.db["battles"]
16
+ self.leaderboard_collection = self.db["leaderboard"]
17
+
18
+ self.models_collection.create_index(
19
+ [("language", 1), ("weight_class", 1), ("model_name", 1)], unique=True
20
+ )
21
+ self.leaderboard_collection.create_index(
22
+ [("language", 1), ("weight_class", 1), ("model_id", 1)], unique=True
23
+ )
24
+ self.battles_collection.create_index(
25
+ [("language", 1), ("weight_class", 1), ("vote_timestamp", 1)]
26
+ )
27
+
28
+ # ---------- Model ----------
29
+
30
+ def insert_model(self, model: Model) -> ObjectId:
31
+ data = asdict(model)
32
+ if data.get("_id") is None:
33
+ data.pop("_id")
34
+ result = self.models_collection.insert_one(data)
35
+ return result.inserted_id
36
+
37
+ def get_model(self, model_id: ObjectId) -> Optional[Model]:
38
+ data = self.models_collection.find_one({"_id": model_id})
39
+ if data:
40
+ return Model(**data)
41
+ return None
42
+
43
+ def update_model(self, model: Model) -> bool:
44
+ data = asdict(model)
45
+ if data.get("_id") is None:
46
+ raise ValueError("model _id is required for updating.")
47
+ result = self.models_collection.replace_one({"_id": data["_id"]}, data)
48
+ return result.modified_count > 0
49
+
50
+ def delete_model(self, model_id: ObjectId) -> bool:
51
+ result = self.models_collection.delete_one({"_id": model_id})
52
+ return result.deleted_count > 0
53
+
54
+ def find_models(self, language: str, weight_class: str) -> List[Model]:
55
+ query = {
56
+ "language": language,
57
+ "weight_class": weight_class
58
+ }
59
+ cursor = self.models_collection.find(query).sort("_id", 1)
60
+ return [Model(**doc) for doc in cursor]
61
+
62
+ def find_one_model(self, language: str, weight_class: str, model_name: str) -> Optional[Model]:
63
+ query = {
64
+ "language": language,
65
+ "weight_class": weight_class,
66
+ "model_name": model_name
67
+ }
68
+ data = self.models_collection.find_one(query)
69
+ if data:
70
+ return Model(**data)
71
+ return None
72
+
73
+ # ---------- Battle ----------
74
+
75
+ def insert_battle(self, battle: Battle) -> ObjectId:
76
+ data = asdict(battle)
77
+ if data.get("_id") is None:
78
+ data.pop("_id")
79
+ result = self.battles_collection.insert_one(data)
80
+ return result.inserted_id
81
+
82
+ def get_battle(self, battle_id: ObjectId) -> Optional[Battle]:
83
+ data = self.battles_collection.find_one({"_id": battle_id})
84
+ if data:
85
+ return Battle(**data)
86
+ return None
87
+
88
+ def update_battle(self, battle: Battle) -> bool:
89
+ data = asdict(battle)
90
+ if data.get("_id") is None:
91
+ raise ValueError("battle _id is required for updating.")
92
+ result = self.battles_collection.replace_one({"_id": data["_id"]}, data)
93
+ return result.modified_count > 0
94
+
95
+ def delete_battle(self, battle_id: ObjectId) -> bool:
96
+ result = self.battles_collection.delete_one({"_id": battle_id})
97
+ return result.deleted_count > 0
98
+
99
+ def find_battles(self, language: str, weight_class: str) -> List[Battle]:
100
+ query = {
101
+ "language": language,
102
+ "weight_class": weight_class
103
+ }
104
+ cursor = self.battles_collection.find(query)
105
+ return [Battle(**doc) for doc in cursor]
106
+
107
+ def find_last_battle(self) -> Optional[Battle]:
108
+ battle_doc = self.battles_collection.find_one(sort=[("_id", -1)])
109
+ if battle_doc:
110
+ return Battle(**battle_doc)
111
+ return None
112
+
113
+ # ---------- LeaderboardEntry ----------
114
+
115
+ def insert_leaderboard_entry(self, entry: LeaderboardEntry) -> ObjectId:
116
+ data = asdict(entry)
117
+ if data.get("_id") is None:
118
+ data.pop("_id")
119
+ result = self.leaderboard_collection.insert_one(data)
120
+ return result.inserted_id
121
+
122
+ def get_leaderboard_entry(self, entry_id: ObjectId) -> Optional[LeaderboardEntry]:
123
+ data = self.leaderboard_collection.find_one({"_id": entry_id})
124
+ if data:
125
+ return LeaderboardEntry(**data)
126
+ return None
127
+
128
+ def update_leaderboard_entry(self, entry: LeaderboardEntry) -> bool:
129
+ data = asdict(entry)
130
+ if data.get("_id") is None:
131
+ raise ValueError("leaderboard _id is required for updating.")
132
+ result = self.leaderboard_collection.replace_one({"_id": data["_id"]}, data)
133
+ return result.modified_count > 0
134
+
135
+ def delete_leaderboard_entry(self, entry_id: ObjectId) -> bool:
136
+ result = self.leaderboard_collection.delete_one({"_id": entry_id})
137
+ return result.deleted_count > 0
138
+
139
+ def find_leaderboard_entries(self, language: str, weight_class: str) -> List[LeaderboardEntry]:
140
+ query = {
141
+ "language": language,
142
+ "weight_class": weight_class
143
+ }
144
+ cursor = self.leaderboard_collection.find(query).sort("elo_score", -1)
145
+ return [LeaderboardEntry(**doc) for doc in cursor]
146
+
147
+ def find_one_leaderboard_entry(self, language: str, weight_class: str, model_id: ObjectId) -> Optional[
148
+ LeaderboardEntry]:
149
+ data = self.leaderboard_collection.find_one({
150
+ "language": language,
151
+ "weight_class": weight_class,
152
+ "model_id": model_id
153
+ })
154
+ if data:
155
+ return LeaderboardEntry(**data)
156
+ return None
indiebot_arena/model/__init__.py ADDED
File without changes
indiebot_arena/model/domain_model.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+ from datetime import datetime
3
+ from typing import Optional
4
+
5
+ from bson import ObjectId
6
+
7
+
8
+ @dataclass
9
+ class Model:
10
+ language: str # 言語区分 (例: "ja", "en")
11
+ weight_class: str # サイズ区分 (例: "U-5GB", "U-10GB")
12
+ model_name: str # 例: "fukugawa/gemma-2-9b-finetuned-bnb-4bit"
13
+ runtime: str # 実行環境 (例: "transformers", "llama.cpp")
14
+ quantization: str # 量子化方式 (例: "none", "bnb", "gptq")
15
+ file_format: str # 保存形式 (例: "safetensors", "gguf")
16
+ file_size_gb: float # ファイルサイズ(単位: GB)
17
+ description: Optional[str] = None
18
+ created_at: datetime = field(default_factory=datetime.utcnow)
19
+ _id: Optional[ObjectId] = None
20
+
21
+
22
+ @dataclass
23
+ class Battle:
24
+ language: str
25
+ weight_class: str
26
+ model_a_id: ObjectId
27
+ model_b_id: ObjectId
28
+ winner_model_id: ObjectId
29
+ user_id: str
30
+ vote_timestamp: datetime = field(default_factory=datetime.utcnow)
31
+ _id: Optional[ObjectId] = None
32
+
33
+
34
+ @dataclass
35
+ class LeaderboardEntry:
36
+ language: str
37
+ weight_class: str
38
+ model_id: ObjectId
39
+ elo_score: int
40
+ last_updated: datetime = field(default_factory=datetime.utcnow)
41
+ _id: Optional[ObjectId] = None
indiebot_arena/service/__init__.py ADDED
File without changes
indiebot_arena/service/arena_service.py ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+ import re
3
+ from datetime import datetime
4
+ from typing import List, Optional, Tuple, Dict
5
+
6
+ from bson import ObjectId
7
+
8
+ from indiebot_arena.dao.mongo_dao import MongoDAO
9
+ from indiebot_arena.model.domain_model import Model, Battle, LeaderboardEntry
10
+
11
+ INITIAL_RATING = 1000
12
+ K_FACTOR = 32
13
+
14
+
15
+ class ArenaService:
16
+ def __init__(self, dao: MongoDAO):
17
+ self.dao = dao
18
+
19
+ # ---------- Model ----------
20
+
21
+ def register_model(
22
+ self,
23
+ language: str,
24
+ weight_class: str,
25
+ model_name: str,
26
+ runtime: str,
27
+ quantization: str,
28
+ file_format: str,
29
+ file_size_gb: float,
30
+ description: Optional[str] = ""
31
+ ) -> ObjectId:
32
+ # Non-empty checks
33
+ if not model_name.strip():
34
+ raise ValueError("Model name cannot be empty.")
35
+ if not runtime.strip():
36
+ raise ValueError("Runtime cannot be empty.")
37
+ if not quantization.strip():
38
+ raise ValueError("Quantization cannot be empty.")
39
+ if not file_format.strip():
40
+ raise ValueError("File format cannot be empty.")
41
+
42
+ # Validation
43
+ if runtime!="transformers":
44
+ raise ValueError("Runtime must be 'transformers'.")
45
+ if quantization not in ("bnb", "none"):
46
+ raise ValueError("Quantization must be 'bnb' or 'none'.")
47
+ if file_format!="safetensors":
48
+ raise ValueError("File format must be 'safetensors'.")
49
+ if language not in ("ja", "en"):
50
+ raise ValueError("Language must be 'ja' or 'en'.")
51
+ if weight_class not in ("U-5GB", "U-10GB"):
52
+ raise ValueError("Weight class must be 'U-5GB' or 'U-10GB'.")
53
+ if file_size_gb <= 0:
54
+ raise ValueError("File size must be greater than 0.")
55
+ if weight_class=="U-5GB" and file_size_gb >= 5.0:
56
+ raise ValueError("For U-5GB, file size must be less than 5.0.")
57
+ if weight_class=="U-10GB" and file_size_gb >= 10.0:
58
+ raise ValueError("For U-10GB, file size must be less than 10.0.")
59
+ if not re.match(r"^[^/]+/[^/]+$", model_name):
60
+ raise ValueError("Model name must be in 'xxxxx/xxxxxx' format.")
61
+
62
+ # Description length check
63
+ if description is not None and len(description) > 1024:
64
+ raise ValueError("Description must be 1024 characters or less.")
65
+
66
+ # Duplicate check
67
+ if self.dao.find_one_model(language, weight_class, model_name) is not None:
68
+ raise ValueError("A model with this language, weight class and model name already exists.")
69
+
70
+ model = Model(
71
+ model_name=model_name,
72
+ runtime=runtime,
73
+ quantization=quantization,
74
+ file_format=file_format,
75
+ file_size_gb=file_size_gb,
76
+ language=language,
77
+ weight_class=weight_class,
78
+ description=description
79
+ )
80
+ return self.dao.insert_model(model)
81
+
82
+ def get_one_model(
83
+ self,
84
+ language: str,
85
+ weight_class: str,
86
+ model_name: str
87
+ ) -> Optional[Model]:
88
+ return self.dao.find_one_model(language, weight_class, model_name)
89
+
90
+ def get_two_random_models(
91
+ self,
92
+ language: str,
93
+ weight_class: str
94
+ ) -> Tuple[Model, Model]:
95
+ models = self.dao.find_models(language, weight_class)
96
+ if len(models) < 2:
97
+ raise ValueError("Need at least 2 models for a battle.")
98
+ return tuple(random.sample(models, 2))
99
+
100
+ def get_model_dropdown_list(
101
+ self,
102
+ language: str,
103
+ weight_class: str
104
+ ) -> List[Dict[str, str]]:
105
+ models = self.dao.find_models(language, weight_class)
106
+ return [
107
+ {"label": model.model_name, "value": model.model_name}
108
+ for model in models if model.model_name
109
+ ]
110
+
111
+ # ---------- Battle ----------
112
+
113
+ def record_battle(
114
+ self,
115
+ language: str,
116
+ weight_class: str,
117
+ model_a_id: ObjectId,
118
+ model_b_id: ObjectId,
119
+ winner_model_id: ObjectId,
120
+ user_id: str
121
+ ) -> ObjectId:
122
+ # Validation
123
+ if language not in ("ja", "en"):
124
+ raise ValueError("Language must be 'ja' or 'en'.")
125
+ if weight_class not in ("U-5GB", "U-10GB"):
126
+ raise ValueError("Weight class must be 'U-5GB' or 'U-10GB'.")
127
+
128
+ if model_a_id is None or model_b_id is None or winner_model_id is None:
129
+ raise ValueError("All model IDs must be provided.")
130
+
131
+ if model_a_id==model_b_id:
132
+ raise ValueError("Model A and Model B must be different.")
133
+
134
+ if winner_model_id!=model_a_id and winner_model_id!=model_b_id:
135
+ raise ValueError("Winner model ID must be either model_a_id or model_b_id.")
136
+
137
+ # Check duplicate
138
+ last_battle = self.dao.find_last_battle()
139
+ if last_battle is not None:
140
+ if (last_battle.language==language and
141
+ last_battle.weight_class==weight_class and
142
+ last_battle.model_a_id==model_a_id and
143
+ last_battle.model_b_id==model_b_id and
144
+ last_battle.winner_model_id==winner_model_id and
145
+ last_battle.user_id==user_id):
146
+ raise ValueError("Duplicate battle record detected: the latest record already matches the provided battle details.")
147
+
148
+ # Check model exists
149
+ model_a = self.dao.get_model(model_a_id)
150
+ model_b = self.dao.get_model(model_b_id)
151
+ if model_a is None or model_b is None:
152
+ raise ValueError("Both Model A and Model B must exist in the database.")
153
+
154
+ battle = Battle(
155
+ model_a_id=model_a_id,
156
+ model_b_id=model_b_id,
157
+ language=language,
158
+ weight_class=weight_class,
159
+ winner_model_id=winner_model_id,
160
+ user_id=user_id,
161
+ )
162
+ battle_id = self.dao.insert_battle(battle)
163
+ return battle_id
164
+
165
+ # ---------- LeaderboardEntry ----------
166
+
167
+ def get_leaderboard(
168
+ self,
169
+ language: str,
170
+ weight_class: str
171
+ ) -> List[LeaderboardEntry]:
172
+ return self.dao.find_leaderboard_entries(language, weight_class)
173
+
174
+ def update_leaderboard(
175
+ self,
176
+ language: str,
177
+ weight_class: str
178
+ ) -> bool:
179
+ """
180
+ Update the leaderboard using battle history.
181
+ Each model starts at INITIAL_RATING and K is K_FACTOR.
182
+ """
183
+ models = self.dao.find_models(language, weight_class)
184
+ if not models:
185
+ return False
186
+
187
+ ratings = {
188
+ str(model._id): INITIAL_RATING for model in models if model._id is not None
189
+ }
190
+
191
+ battles = self.dao.find_battles(language, weight_class)
192
+ battles.sort(key=lambda b: b.vote_timestamp)
193
+
194
+ for battle in battles:
195
+ model_a_id_str = str(battle.model_a_id)
196
+ model_b_id_str = str(battle.model_b_id)
197
+
198
+ if model_a_id_str not in ratings:
199
+ ratings[model_a_id_str] = INITIAL_RATING
200
+ if model_b_id_str not in ratings:
201
+ ratings[model_b_id_str] = INITIAL_RATING
202
+
203
+ if str(battle.winner_model_id)==model_a_id_str:
204
+ score_a, score_b = 1, 0
205
+ else:
206
+ score_a, score_b = 0, 1
207
+
208
+ exp_a = 1 / (1 + 10 ** ((ratings[model_b_id_str] - ratings[model_a_id_str]) / 400))
209
+ exp_b = 1 / (1 + 10 ** ((ratings[model_a_id_str] - ratings[model_b_id_str]) / 400))
210
+
211
+ ratings[model_a_id_str] += K_FACTOR * (score_a - exp_a)
212
+ ratings[model_b_id_str] += K_FACTOR * (score_b - exp_b)
213
+
214
+ for model in models:
215
+ if model._id is None:
216
+ continue
217
+ model_id_str = str(model._id)
218
+ new_rating = round(ratings.get(model_id_str, INITIAL_RATING))
219
+ entry = self.dao.find_one_leaderboard_entry(language, weight_class, model._id)
220
+ if entry:
221
+ entry.elo_score = new_rating
222
+ entry.last_updated = datetime.utcnow()
223
+ self.dao.update_leaderboard_entry(entry)
224
+ else:
225
+ new_entry = LeaderboardEntry(
226
+ model_id=model._id,
227
+ language=language,
228
+ weight_class=weight_class,
229
+ elo_score=new_rating,
230
+ )
231
+ self.dao.insert_leaderboard_entry(new_entry)
232
+ return True
indiebot_arena/service/bootstrap_service.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import logging
2
+
3
+ from indiebot_arena.dao.mongo_dao import MongoDAO
4
+ from indiebot_arena.service.arena_service import ArenaService
5
+
6
+
7
+ class BootstrapService:
8
+ def __init__(self, dao: MongoDAO):
9
+ self.dao = dao
10
+ self.arena_service = ArenaService(dao);
11
+
12
+ def provision_database(self):
13
+ if self.dao.models_collection.count_documents({}) > 0:
14
+ logging.info("Database already provisioned. Skipping initial data insertion.")
15
+ return
16
+
17
+ try:
18
+ self.arena_service.register_model("ja", "U-5GB", "google/gemma-3-1b-it", "transformers", "none", "safetensors", 1.86)
19
+ self.arena_service.register_model("ja", "U-5GB", "google/gemma-2-2b-it", "transformers", "none", "safetensors", 4.87)
20
+ self.arena_service.register_model("ja", "U-5GB", "google/gemma-2-2b-jpn-it", "transformers", "none", "safetensors", 4.87)
21
+ self.arena_service.register_model("ja", "U-5GB", "indiebot-community/gemma-3-1b-it-bnb-4bit", "transformers", "bnb", "safetensors", 0.93)
22
+ self.arena_service.register_model("ja", "U-5GB", "indiebot-community/gemma-2-2b-it-bnb-4bit", "transformers", "bnb", "safetensors", 2.16)
23
+ self.arena_service.register_model("ja", "U-5GB", "indiebot-community/gemma-2-2b-jpn-it-bnb-4bit", "transformers", "bnb", "safetensors", 2.16)
24
+ self.arena_service.register_model("ja", "U-10GB", "google/gemma-3-4b-it", "transformers", "none", "safetensors", 8.01)
25
+ self.arena_service.register_model("ja", "U-10GB", "indiebot-community/gemma-3-4b-it-bnb-4bit", "transformers", "bnb", "safetensors", 2.93)
26
+ self.arena_service.register_model("ja", "U-10GB", "indiebot-community/gemma-3-12b-it-bnb-4bit", "transformers", "bnb", "safetensors", 7.51)
27
+ self.arena_service.register_model("ja", "U-10GB", "indiebot-community/gemma-2-9b-it-bnb-4bit", "transformers", "bnb", "safetensors", 6.07)
28
+
29
+ self.arena_service.update_leaderboard("ja", "U-5GB")
30
+ self.arena_service.update_leaderboard("ja", "U-10GB")
31
+ except Exception as e:
32
+ logging.error(f"Error register_model: {e}")
indiebot_arena/ui/__init__.py ADDED
File without changes
indiebot_arena/ui/battle.py ADDED
@@ -0,0 +1,234 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import concurrent.futures
2
+ import hashlib
3
+ import random
4
+ import threading
5
+
6
+ import gradio as gr
7
+ import spaces
8
+ import torch
9
+ from transformers import AutoModelForCausalLM, AutoTokenizer
10
+
11
+ from indiebot_arena.config import LOCAL_TESTING, MODEL_SELECTION_MODE, MAX_INPUT_TOKEN_LENGTH, MAX_NEW_TOKENS
12
+ from indiebot_arena.service.arena_service import ArenaService
13
+
14
+ DESCRIPTION = "### 💬 チャットバトル"
15
+
16
+ _model_cache = {}
17
+ _model_lock = threading.Lock()
18
+
19
+
20
+ # Avoid “cannot copy out of meta tensor” error
21
+ def get_cached_model_and_tokenizer(model_id: str):
22
+ with _model_lock:
23
+ if model_id not in _model_cache:
24
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
25
+ model = AutoModelForCausalLM.from_pretrained(
26
+ model_id,
27
+ device_map="auto",
28
+ torch_dtype=torch.bfloat16,
29
+ use_safetensors=True
30
+ )
31
+ model.eval()
32
+ _model_cache[model_id] = (model, tokenizer)
33
+ return _model_cache[model_id]
34
+
35
+
36
+ @spaces.GPU(duration=30)
37
+ def generate(message: str, chat_history: list, model_id: str, max_new_tokens: int = MAX_NEW_TOKENS,
38
+ temperature: float = 0.6, top_p: float = 0.9, top_k: int = 50, repetition_penalty: float = 1.2) -> str:
39
+ if LOCAL_TESTING:
40
+ model, tokenizer = get_cached_model_and_tokenizer(model_id)
41
+ else:
42
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
43
+ model = AutoModelForCausalLM.from_pretrained(
44
+ model_id,
45
+ device_map="auto",
46
+ torch_dtype=torch.bfloat16,
47
+ use_safetensors=True
48
+ )
49
+
50
+ conversation = chat_history.copy()
51
+ conversation.append({"role": "user", "content": message})
52
+ input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
53
+ if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
54
+ input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
55
+ gr.Warning(f"Trimmed input as it exceeded {MAX_INPUT_TOKEN_LENGTH} tokens.")
56
+ input_ids = input_ids.to(model.device)
57
+ outputs = model.generate(
58
+ input_ids=input_ids,
59
+ max_new_tokens=max_new_tokens,
60
+ do_sample=True,
61
+ top_p=top_p,
62
+ top_k=top_k,
63
+ temperature=temperature,
64
+ repetition_penalty=repetition_penalty,
65
+ )
66
+ response = tokenizer.decode(outputs[0][input_ids.shape[1]:], skip_special_tokens=True)
67
+ return response
68
+
69
+
70
+ def format_chat_history(history):
71
+ conversation = []
72
+ for user_msg, assistant_msg in history:
73
+ conversation.append({"role": "user", "content": user_msg})
74
+ if assistant_msg:
75
+ conversation.append({"role": "assistant", "content": assistant_msg})
76
+ return conversation
77
+
78
+
79
+ def submit_message(message, history_a, history_b, model_a, model_b):
80
+ history_a.append((message, ""))
81
+ history_b.append((message, ""))
82
+ conv_history_a = format_chat_history(history_a[:-1])
83
+ conv_history_b = format_chat_history(history_b[:-1])
84
+ with concurrent.futures.ThreadPoolExecutor() as executor:
85
+ future_a = executor.submit(generate, message, conv_history_a, model_a)
86
+ future_b = executor.submit(generate, message, conv_history_b, model_b)
87
+ response_a = future_a.result()
88
+ response_b = future_b.result()
89
+ history_a[-1] = (message, response_a)
90
+ history_b[-1] = (message, response_b)
91
+ return history_a, history_b, "", gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=False)
92
+
93
+
94
+ def get_random_values(model_labels):
95
+ if MODEL_SELECTION_MODE=="random":
96
+ return random.sample(model_labels, 2)
97
+ if MODEL_SELECTION_MODE=="manual":
98
+ return model_labels[0], model_labels[0]
99
+
100
+ def battle_content(dao, language):
101
+ arena_service = ArenaService(dao)
102
+ default_weight = "U-5GB"
103
+ initial_models = arena_service.get_model_dropdown_list(language, default_weight)
104
+ initial_choices = [m["label"] for m in initial_models]
105
+ initial_value_a, initial_value_b = get_random_values(initial_choices)
106
+ dropdown_visible = False if MODEL_SELECTION_MODE=="random" else True
107
+
108
+ def fetch_model_dropdown(weight_class):
109
+ models = arena_service.get_model_dropdown_list(language, weight_class)
110
+ model_labels = [m["label"] for m in models]
111
+ value_a, value_b = get_random_values(model_labels)
112
+ update_obj_a = gr.update(choices=model_labels, value=value_a)
113
+ update_obj_b = gr.update(choices=model_labels, value=value_b)
114
+ return update_obj_a, update_obj_b, model_labels
115
+
116
+ def submit_vote(vote_choice, weight_class, model_a_name, model_b_name, request: gr.Request):
117
+ user_id = generate_anonymous_user_id(request)
118
+ model_a = arena_service.get_one_model(language, weight_class, model_a_name)
119
+ model_b = arena_service.get_one_model(language, weight_class, model_b_name)
120
+ winner = model_a if vote_choice=="Chatbot A" else model_b
121
+ try:
122
+ arena_service.record_battle(language, weight_class, model_a._id, model_b._id, winner._id, user_id)
123
+ arena_service.update_leaderboard(language, weight_class)
124
+ return "投票が完了しました"
125
+ except Exception as e:
126
+ return f"エラー: {e}"
127
+
128
+ def handle_vote(vote_choice, weight_class, model_a_name, model_b_name, request: gr.Request):
129
+ msg = submit_vote(vote_choice, weight_class, model_a_name, model_b_name, request)
130
+ return (
131
+ gr.update(value=msg, visible=True),
132
+ gr.update(interactive=False, value=model_a_name),
133
+ gr.update(interactive=False, value=model_b_name),
134
+ gr.update(visible=False),
135
+ gr.update(visible=True, interactive=True)
136
+ )
137
+
138
+ def generate_anonymous_user_id(request: gr.Request):
139
+ user_ip = request.headers.get('x-forwarded-for')
140
+ if user_ip:
141
+ user_id = "indiebot:" + user_ip
142
+ hashed_user_id = hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:16]
143
+ return hashed_user_id
144
+ else:
145
+ return "anonymous"
146
+
147
+ def on_vote_a_click(weight, a, b, request: gr.Request):
148
+ return handle_vote("Chatbot A", weight, a, b, request)
149
+
150
+ def on_vote_b_click(weight, a, b, request: gr.Request):
151
+ return handle_vote("Chatbot B", weight, a, b, request)
152
+
153
+ def reset_battle(dropdown_options):
154
+ value_a, value_b = get_random_values(dropdown_options)
155
+ return (
156
+ [], # chatbot_aのリセット
157
+ [], # chatbot_bのリセット
158
+ gr.update(value="", visible=True), # user_inputのクリア&表示
159
+ gr.update(interactive=False, value="A is better"), # vote_a_btnのリセット
160
+ gr.update(interactive=False, value="B is better"), # vote_b_btnのリセット
161
+ gr.update(visible=False), # vote_messageの非表示
162
+ gr.update(visible=False, interactive=False), # next_battle_btnの非表示
163
+ gr.update(choices=dropdown_options, value=value_a), # model_dropdown_a更新
164
+ gr.update(choices=dropdown_options, value=value_b), # model_dropdown_b更新
165
+ gr.update(interactive=True) # weight_class_radio を有効化
166
+ )
167
+
168
+ with gr.Blocks(css="style.css") as battle_ui:
169
+ gr.Markdown(DESCRIPTION)
170
+ weight_class_radio = gr.Radio(
171
+ choices=["U-5GB", "U-10GB"],
172
+ label="階級",
173
+ value=default_weight
174
+ )
175
+ dropdown_options_state = gr.State(initial_choices)
176
+ with gr.Row():
177
+ model_dropdown_a = gr.Dropdown(
178
+ choices=initial_choices,
179
+ label="モデルAを選択",
180
+ value=initial_value_a,
181
+ visible=dropdown_visible
182
+ )
183
+ model_dropdown_b = gr.Dropdown(
184
+ choices=initial_choices,
185
+ label="モデルBを選択",
186
+ value=initial_value_b,
187
+ visible=dropdown_visible
188
+ )
189
+ weight_class_radio.change(
190
+ fn=fetch_model_dropdown,
191
+ inputs=weight_class_radio,
192
+ outputs=[model_dropdown_a, model_dropdown_b, dropdown_options_state]
193
+ )
194
+ with gr.Row():
195
+ chatbot_a = gr.Chatbot(label="Chatbot A")
196
+ chatbot_b = gr.Chatbot(label="Chatbot B")
197
+ with gr.Row():
198
+ vote_a_btn = gr.Button("A is better", interactive=False)
199
+ vote_b_btn = gr.Button("B is better", interactive=False)
200
+ user_input = gr.Textbox(
201
+ placeholder="日本語でメッセージを入力...",
202
+ submit_btn=True,
203
+ show_label=False
204
+ )
205
+ with gr.Row():
206
+ with gr.Column(scale=3):
207
+ vote_message = gr.Textbox(show_label=False, interactive=False, visible=False)
208
+ with gr.Column(scale=1):
209
+ next_battle_btn = gr.Button("次のバトルへ", interactive=False, visible=False, elem_id="next_battle_btn")
210
+ user_input.submit(
211
+ fn=submit_message,
212
+ inputs=[user_input, chatbot_a, chatbot_b, model_dropdown_a, model_dropdown_b],
213
+ outputs=[chatbot_a, chatbot_b, user_input, vote_a_btn, vote_b_btn, weight_class_radio]
214
+ )
215
+ vote_a_btn.click(
216
+ fn=on_vote_a_click,
217
+ inputs=[weight_class_radio, model_dropdown_a, model_dropdown_b],
218
+ outputs=[vote_message, vote_a_btn, vote_b_btn, user_input, next_battle_btn]
219
+ )
220
+ vote_b_btn.click(
221
+ fn=on_vote_b_click,
222
+ inputs=[weight_class_radio, model_dropdown_a, model_dropdown_b],
223
+ outputs=[vote_message, vote_a_btn, vote_b_btn, user_input, next_battle_btn]
224
+ )
225
+ next_battle_btn.click(
226
+ fn=reset_battle,
227
+ inputs=[dropdown_options_state],
228
+ outputs=[
229
+ chatbot_a, chatbot_b, user_input, vote_a_btn,
230
+ vote_b_btn, vote_message, next_battle_btn,
231
+ model_dropdown_a, model_dropdown_b, weight_class_radio
232
+ ]
233
+ )
234
+ return battle_ui
indiebot_arena/ui/leaderboard.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ import gradio as gr
4
+ import pandas as pd
5
+
6
+ from indiebot_arena.service.arena_service import ArenaService
7
+
8
+ DESCRIPTION = "### 🏆️ リーダーボード"
9
+
10
+ base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
11
+ docs_path = os.path.join(base_dir, "docs", "leaderboard_header.md")
12
+
13
+ def leaderboard_content(dao, language):
14
+ arena_service = ArenaService(dao)
15
+
16
+ def fetch_leaderboard_data(weight_class):
17
+ entries = arena_service.get_leaderboard(language, weight_class)
18
+ data = []
19
+ for entry in entries:
20
+ model_obj = arena_service.dao.get_model(entry.model_id)
21
+ model_name = model_obj.model_name if model_obj else "Unknown"
22
+ file_size = model_obj.file_size_gb if model_obj else "N/A"
23
+ desc = model_obj.description if model_obj else ""
24
+ last_updated = entry.last_updated.strftime("%Y-%m-%d %H:%M:%S")
25
+ data.append([model_name, entry.elo_score, file_size, desc, last_updated])
26
+ if not data:
27
+ data = [["No data available", "", "", "", ""]]
28
+ df = pd.DataFrame(data, columns=["Model Name", "Elo Score", "File Size (GB)", "Description", "Last Updated"])
29
+ df.insert(0, "Rank", range(1, len(df) + 1))
30
+ return df
31
+
32
+ initial_weight_class = "U-5GB"
33
+ with gr.Blocks(css="style.css") as leaderboard_ui:
34
+ with open(docs_path, "r", encoding="utf-8") as f:
35
+ markdown_content = f.read()
36
+ gr.Markdown(markdown_content)
37
+ gr.Markdown(DESCRIPTION)
38
+ weight_class_radio = gr.Radio(choices=["U-5GB", "U-10GB"], label="階級", value=initial_weight_class)
39
+ leaderboard_table = gr.Dataframe(
40
+ headers=["Rank", "Model Name", "Elo Score", "File Size (GB)", "Description", "Last Updated"],
41
+ value=fetch_leaderboard_data(initial_weight_class),
42
+ interactive=False
43
+ )
44
+ refresh_btn = gr.Button("更新")
45
+ weight_class_radio.change(fn=fetch_leaderboard_data, inputs=weight_class_radio, outputs=leaderboard_table)
46
+ refresh_btn.click(fn=fetch_leaderboard_data, inputs=weight_class_radio, outputs=leaderboard_table)
47
+ return leaderboard_ui
indiebot_arena/ui/registration.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ from dataclasses import dataclass
4
+
5
+ import gradio as gr
6
+ import spaces
7
+ import torch
8
+ from huggingface_hub import hf_hub_url, get_hf_file_metadata, model_info
9
+ from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
10
+
11
+ from indiebot_arena.service.arena_service import ArenaService
12
+ from indiebot_arena.ui.battle import generate
13
+
14
+ DESCRIPTION = "### 📚️ 登録済みモデル"
15
+
16
+ base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
17
+ docs_path = os.path.join(base_dir, "docs", "model_registration_guide.md")
18
+
19
+
20
+ @dataclass
21
+ class ModelMeta:
22
+ model_id: str
23
+ architecture: list
24
+ parameters: str
25
+ model_type: str
26
+ weights_file_size: float
27
+ weights_format: str
28
+ quantization: str
29
+
30
+
31
+ def format_model_meta(meta: ModelMeta) -> str:
32
+ return (
33
+ f"Model ID: {meta.model_id}\n"
34
+ f"Architecture: {meta.architecture}\n"
35
+ f"Parameters: {meta.parameters}\n"
36
+ f"Model Type: {meta.model_type}\n"
37
+ f"Weights File Size: {meta.weights_file_size} GB\n"
38
+ f"Weights Format: {meta.weights_format}\n"
39
+ f"Quantization: {meta.quantization}"
40
+ )
41
+
42
+
43
+ @spaces.GPU(duration=30)
44
+ def get_model_meta(model_id: str):
45
+ try:
46
+ config = AutoConfig.from_pretrained(model_id)
47
+ model = AutoModelForCausalLM.from_pretrained(
48
+ model_id,
49
+ device_map="auto",
50
+ torch_dtype=torch.bfloat16,
51
+ use_safetensors=True
52
+ )
53
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
54
+ repo_info = model_info(model_id)
55
+ weights_files = [file.rfilename for file in repo_info.siblings if
56
+ file.rfilename.endswith('.bin') or file.rfilename.endswith('.safetensors')]
57
+ total_size = 0
58
+ file_formats = set()
59
+ for file_name in weights_files:
60
+ file_formats.add(file_name.split('.')[-1])
61
+ file_url = hf_hub_url(model_id, filename=file_name)
62
+ metadata = get_hf_file_metadata(file_url)
63
+ total_size += metadata.size or 0
64
+ total_size_gb = round(total_size / (1024 ** 3), 2)
65
+ model_type = type(model).__name__
66
+ quant_config = getattr(config, 'quantization_config', {})
67
+ quant_method = quant_config.get('quant_method', 'none')
68
+ meta = ModelMeta(
69
+ model_id=model_id,
70
+ architecture=config.architectures if hasattr(config, 'architectures') else [],
71
+ parameters=f"{round(sum(p.numel() for p in model.parameters()) / 1e9, 2)} Billion",
72
+ model_type=model_type,
73
+ weights_file_size=total_size_gb,
74
+ weights_format=", ".join(file_formats),
75
+ quantization=quant_method,
76
+ )
77
+ return meta
78
+ except Exception as e:
79
+ return f"Error: {str(e)}"
80
+
81
+
82
+ def registration_content(dao, language):
83
+ arena_service = ArenaService(dao)
84
+
85
+ def fetch_models(weight_class):
86
+ models = arena_service.dao.find_models(language, weight_class)
87
+ data = []
88
+ for m in models:
89
+ created_at_str = m.created_at.strftime("%Y-%m-%d %H:%M:%S") if m.created_at else ""
90
+ data.append([m.weight_class, m.model_name, m.runtime, m.quantization, m.file_format, m.file_size_gb,
91
+ m.description or "", created_at_str])
92
+ return data
93
+
94
+ def load_test(model_id, weight_class, current_output):
95
+ if not re.match(r'^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$', model_id):
96
+ err = "Error: Invalid model_id format. It must be in the format 'owner/model'."
97
+ return (current_output + "\n" + err, gr.update(interactive=False), None)
98
+ meta = get_model_meta(model_id)
99
+ if isinstance(meta, str) and meta.startswith("Error:"):
100
+ return (current_output + "\n" + meta, gr.update(interactive=False), None)
101
+ if weight_class=="U-5GB" and meta.weights_file_size >= 5.0:
102
+ err = f"Error: File size exceeds U-5GB limit. {meta.weights_file_size} GB"
103
+ return (current_output + "\n" + err, gr.update(interactive=False), None)
104
+ if weight_class=="U-10GB" and meta.weights_file_size >= 10.0:
105
+ err = f"Error: File size exceeds U-10GB limit. {meta.weights_file_size} GB"
106
+ return (current_output + "\n" + err, gr.update(interactive=False), None)
107
+ if "safetensors" not in meta.weights_format.lower():
108
+ err = "Error: Weights Format must include safetensors."
109
+ return (current_output + "\n" + err, gr.update(interactive=False), None)
110
+ if meta.quantization.lower() not in ["bitsandbytes", "none"]:
111
+ err = "Error: Quantization must be bitsandbytes or none."
112
+ return (current_output + "\n" + err, gr.update(interactive=False), None)
113
+ display_str = format_model_meta(meta) + "\n\nロードテストが成功しました。\nチャットテストを実施して下さい。\n"
114
+ return (current_output + "\n" + display_str, gr.update(interactive=True), meta)
115
+
116
+ def chat_test(model_id, current_output):
117
+ question = "日本の首都は?"
118
+ expected_word = "東京"
119
+ try:
120
+ response = generate(question, [], model_id)
121
+ except Exception as e:
122
+ error_msg = f"エラー: {str(e)}"
123
+ return (current_output + "\n" + error_msg, gr.update(interactive=False))
124
+
125
+ result_text = f"質問: {question}\n応答: {response}\n"
126
+ if expected_word in response:
127
+ result_text += "成功: 応答に期待するワードが含まれています。\nモデル登録を実施して下さい。\n"
128
+ register_enabled = True
129
+ else:
130
+ result_text += "失敗: 応答に期待するワードが含まれていません。\n"
131
+ register_enabled = False
132
+ return (current_output + "\n" + result_text, gr.update(interactive=register_enabled))
133
+
134
+ def register_model(meta, weight_class, description, current_output):
135
+ try:
136
+ if meta is None:
137
+ raise ValueError("No meta info available.")
138
+ model_id_extracted = meta.model_id
139
+ file_size_gb = meta.weights_file_size
140
+ quantization = "bnb" if meta.quantization.lower()=="bitsandbytes" else "none"
141
+ weights_format = meta.weights_format
142
+ file_format = "safetensors" if "safetensors" in weights_format.lower() else weights_format
143
+ desc = description if description else ""
144
+ arena_service.register_model(language, weight_class, model_id_extracted, "transformers", quantization, file_format, file_size_gb, desc)
145
+ arena_service.update_leaderboard(language, weight_class)
146
+ result = "モデルの登録が完了しました。"
147
+ disable = True
148
+ except Exception as e:
149
+ result = f"エラー: {str(e)}"
150
+ disable = False
151
+ new_output = current_output + "\n" + result
152
+ if disable:
153
+ btn_update = gr.update(interactive=False)
154
+ else:
155
+ btn_update = gr.update()
156
+ return new_output, meta, btn_update, btn_update, btn_update
157
+
158
+ def clear_all():
159
+ initial_weight = "U-5GB"
160
+ return "", initial_weight, "", "", gr.update(interactive=True), gr.update(interactive=False), gr.update(interactive=False), None
161
+
162
+ with gr.Blocks(css="style.css") as ui:
163
+ gr.Markdown(DESCRIPTION)
164
+ weight_class_radio = gr.Radio(choices=["U-5GB", "U-10GB"], label="階級", value="U-5GB")
165
+ mdl_list = gr.Dataframe(
166
+ headers=["Weight Class", "Model Name", "Runtime", "Quantization", "Weights Format",
167
+ "Weights File Size", "Description", "Created At"],
168
+ value=fetch_models("U-5GB"),
169
+ interactive=False
170
+ )
171
+ weight_class_radio.change(fn=fetch_models, inputs=weight_class_radio, outputs=mdl_list)
172
+ with gr.Accordion("🔰 モデルの登録ガイド", open=False):
173
+ with open(docs_path, "r", encoding="utf-8") as f:
174
+ markdown_content = f.read()
175
+ gr.Markdown(markdown_content)
176
+ with gr.Accordion("🤖 モデルの新規登録", open=False):
177
+ model_id_input = gr.Textbox(label="モデルID", max_lines=1)
178
+ reg_weight_class_radio = gr.Radio(choices=["U-5GB", "U-10GB"], label="階級", value="U-5GB")
179
+ description_input = gr.Textbox(label="Description (オプション)", placeholder="任意の説明を入力", max_lines=1, visible=False)
180
+ output_box = gr.Textbox(label="結果出力", lines=10)
181
+ meta_state = gr.State(None)
182
+ with gr.Row():
183
+ test_btn = gr.Button("ロードテスト")
184
+ chat_test_btn = gr.Button("チャットテスト", interactive=False)
185
+ register_btn = gr.Button("モデル登録", interactive=False)
186
+ clear_btn = gr.Button("クリア")
187
+ test_btn.click(fn=load_test, inputs=[model_id_input, reg_weight_class_radio, output_box], outputs=[output_box,
188
+ chat_test_btn,
189
+ meta_state])
190
+ chat_test_btn.click(fn=chat_test, inputs=[model_id_input, output_box], outputs=[output_box, register_btn])
191
+ register_btn.click(fn=register_model, inputs=[meta_state, reg_weight_class_radio, description_input,
192
+ output_box], outputs=[output_box, meta_state, test_btn,
193
+ chat_test_btn, register_btn])
194
+ clear_btn.click(fn=clear_all, inputs=[], outputs=[model_id_input, reg_weight_class_radio, description_input,
195
+ output_box, test_btn, chat_test_btn, register_btn, meta_state])
196
+ return ui
requirements.txt CHANGED
@@ -1 +1,11 @@
1
- huggingface_hub==0.25.2
 
 
 
 
 
 
 
 
 
 
 
1
+ accelerate==1.2.1
2
+ bitsandbytes==0.44.1
3
+ gradio==5.12.0
4
+ huggingface-hub==0.27.1
5
+ pandas==2.2.3
6
+ peft==0.13.2
7
+ pydantic==2.10.6
8
+ pymongo[srv]==4.11.3
9
+ spaces==0.32.0
10
+ torch==2.4.0
11
+ transformers==4.50.0
style.css ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ h1 {
2
+ font-size: 2.5rem;
3
+ font-weight: 800;
4
+ text-align: left;
5
+ margin-bottom: 1rem;
6
+ line-height: 1.2;
7
+ }
8
+
9
+ #duplicate-button {
10
+ margin: auto;
11
+ color: #fff;
12
+ background: #1565c0;
13
+ border-radius: 100vh;
14
+ }
15
+
16
+ #next_battle_btn {
17
+ margin-top: auto;
18
+ height: 4rem;
19
+ }