Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+license: other
+base_model: Qwen/Qwen2.5-Coder-7B-Instruct
+tags:
+- llama-factory
+- freeze
+- generated_from_trainer
+model-index:
+- name: qwen_nsx
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# qwen_nsx
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) on the codes_330k_nsx dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 512
+- total_eval_batch_size: 32
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.48.2
+- Pytorch 2.5.1+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.992108229988726,
+    "num_input_tokens_seen": 230686720,
+    "total_flos": 9.786587384895242e+18,
+    "train_loss": 0.6728460962122137,
+    "train_runtime": 17352.9545,
+    "train_samples_per_second": 3.271,
+    "train_steps_per_second": 0.006
+}

config.json

@@ -0,0 +1,35 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 1.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 32768,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.48.2"
+}

llamaboard_config.yaml

@@ -0,0 +1,77 @@
+top.booster: liger_kernel
+top.checkpoint_path: null
+top.finetuning_type: freeze
+top.model_name: Qwen2.5-Coder-7B-Instruct
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: llama3
+top.template: qwen
+train.additional_target: ''
+train.apollo_rank: 256
+train.apollo_scale: 1
+train.apollo_target: all
+train.apollo_update_interval: 200
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 16
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 4096
+train.dataset:
+- codes_330k_nsx
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.extra_args: '{}'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 2
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-5
+train.logging_steps: 1
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '1.0'
+train.max_samples: '50000000'
+train.neat_packing: true
+train.neftune_alpha: 0
+train.num_train_epochs: '1'
+train.packing: true
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to:
+- none
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 500
+train.swanlab_api_key: ''
+train.swanlab_mode: cloud
+train.swanlab_project: llamafactory
+train.swanlab_run_name: ''
+train.swanlab_workspace: ''
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_apollo: true
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: true
+train.use_pissa: false
+train.use_rslora: false
+train.use_swanlab: false
+train.val_size: 0
+train.warmup_steps: 0

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b6f069918b07c064cbba8ae4f00f529aa9bbf84b7cdfcb7fc2694a40f6aa8ef
+size 4877660776

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76ab08070b2c4bf68ff363501c0dbb68275bf5b2d3e523f66cfe7a5be6bf0726
+size 4932749648

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9514261e4d84024f94aa180f5b6a62ce2dea738fe4bf398543d741a9f62c86e6
+size 4991480296

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:453b5ca1c31fa12264a22233413bfd1d61e7e29414a505a1146a8e1f9a9c1e22
+size 1361612328

model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 16163464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

running_log.txt

@@ -0,0 +1,418 @@
+[INFO|2025-04-28 17:07:05] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/config.json
+
+[INFO|2025-04-28 17:07:05] configuration_utils.py:768 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}
+
+
+[INFO|2025-04-28 17:07:05] tokenization_utils_base.py:2034 >> loading file vocab.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/vocab.json
+
+[INFO|2025-04-28 17:07:05] tokenization_utils_base.py:2034 >> loading file merges.txt from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/merges.txt
+
+[INFO|2025-04-28 17:07:05] tokenization_utils_base.py:2034 >> loading file tokenizer.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/tokenizer.json
+
+[INFO|2025-04-28 17:07:05] tokenization_utils_base.py:2034 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-04-28 17:07:05] tokenization_utils_base.py:2034 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-04-28 17:07:05] tokenization_utils_base.py:2034 >> loading file tokenizer_config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/tokenizer_config.json
+
+[INFO|2025-04-28 17:07:05] tokenization_utils_base.py:2034 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-04-28 17:07:06] tokenization_utils_base.py:2304 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-04-28 17:07:06] logging.py:157 >> Add <|im_end|> to stop words.
+
+[INFO|2025-04-28 17:07:06] logging.py:157 >> Loading dataset Codes_query_filtered_330k_ns.json...
+
+[INFO|2025-04-28 17:08:07] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/config.json
+
+[INFO|2025-04-28 17:08:07] configuration_utils.py:768 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}
+
+
+[WARNING|2025-04-28 17:08:07] logging.py:162 >> Input length is smaller than max length. Consider increase input length.
+
+[INFO|2025-04-28 17:08:07] logging.py:157 >> Using llama3 scaling strategy and setting scaling factor to 1.0.
+
+[INFO|2025-04-28 17:08:07] logging.py:157 >> Using block diagonal attention for sequence packing without cross-attention.
+
+[INFO|2025-04-28 17:08:07] logging.py:157 >> Liger kernel has been applied to the model.
+
+[INFO|2025-04-28 17:08:07] modeling_utils.py:3904 >> loading weights file model.safetensors from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/model.safetensors.index.json
+
+[INFO|2025-04-28 17:08:07] modeling_utils.py:1582 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|2025-04-28 17:08:07] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 151643,
+  "eos_token_id": 151645
+}
+
+
+[INFO|2025-04-28 17:08:12] modeling_utils.py:4888 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.
+
+
+[INFO|2025-04-28 17:08:12] modeling_utils.py:4896 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2.5-Coder-7B-Instruct.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.
+
+[INFO|2025-04-28 17:08:12] configuration_utils.py:1095 >> loading configuration file generation_config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/generation_config.json
+
+[INFO|2025-04-28 17:08:12] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8
+}
+
+
+[INFO|2025-04-28 17:08:13] logging.py:157 >> Gradient checkpointing enabled.
+
+[INFO|2025-04-28 17:08:13] logging.py:157 >> Using torch SDPA for faster training and inference.
+
+[INFO|2025-04-28 17:08:13] logging.py:157 >> Upcasting trainable params to float32.
+
+[INFO|2025-04-28 17:08:13] logging.py:157 >> Fine-tuning method: Freeze
+
+[INFO|2025-04-28 17:08:13] logging.py:157 >> Set trainable layers: .13.,.27.
+
+[INFO|2025-04-28 17:08:13] logging.py:157 >> trainable params: 466,115,584 || all params: 7,615,616,512 || trainable%: 6.1205
+
+[INFO|2025-04-28 17:08:13] trainer.py:741 >> Using auto half precision backend
+
+[INFO|2025-04-28 17:08:13] logging.py:157 >> Found linear modules: k_proj,up_proj,gate_proj,o_proj,down_proj,q_proj,v_proj
+
+[INFO|2025-04-28 17:08:13] logging.py:157 >> Using APOLLO optimizer with args: {'rank': 256, 'proj': 'random', 'proj_type': 'std', 'update_proj_gap': 200, 'scale': 1, 'scale_type': 'channel', 'scale_front': False}.
+
+[INFO|2025-04-28 17:08:14] trainer.py:2369 >> ***** Running training *****
+
+[INFO|2025-04-28 17:08:14] trainer.py:2370 >>   Num examples = 56,766
+
+[INFO|2025-04-28 17:08:14] trainer.py:2371 >>   Num Epochs = 1
+
+[INFO|2025-04-28 17:08:14] trainer.py:2372 >>   Instantaneous batch size per device = 16
+
+[INFO|2025-04-28 17:08:14] trainer.py:2375 >>   Total train batch size (w. parallel, distributed & accumulation) = 512
+
+[INFO|2025-04-28 17:08:14] trainer.py:2376 >>   Gradient Accumulation steps = 8
+
+[INFO|2025-04-28 17:08:14] trainer.py:2377 >>   Total optimization steps = 110
+
+[INFO|2025-04-28 17:08:14] trainer.py:2378 >>   Number of trainable parameters = 466,115,584
+
+[INFO|2025-04-28 17:11:00] logging.py:157 >> {'loss': 0.8968, 'learning_rate': 4.9990e-05, 'epoch': 0.01, 'throughput': 12763.83}
+
+[INFO|2025-04-28 17:13:36] logging.py:157 >> {'loss': 0.8202, 'learning_rate': 4.9959e-05, 'epoch': 0.02, 'throughput': 13091.94}
+
+[INFO|2025-04-28 17:16:11] logging.py:157 >> {'loss': 0.7783, 'learning_rate': 4.9908e-05, 'epoch': 0.03, 'throughput': 13222.77}
+
+[INFO|2025-04-28 17:18:47] logging.py:157 >> {'loss': 0.7642, 'learning_rate': 4.9837e-05, 'epoch': 0.04, 'throughput': 13287.21}
+
+[INFO|2025-04-28 17:21:22] logging.py:157 >> {'loss': 0.7475, 'learning_rate': 4.9746e-05, 'epoch': 0.05, 'throughput': 13330.27}
+
+[INFO|2025-04-28 17:23:57] logging.py:157 >> {'loss': 0.7319, 'learning_rate': 4.9634e-05, 'epoch': 0.05, 'throughput': 13369.02}
+
+[INFO|2025-04-28 17:26:31] logging.py:157 >> {'loss': 0.7125, 'learning_rate': 4.9502e-05, 'epoch': 0.06, 'throughput': 13397.69}
+
+[INFO|2025-04-28 17:29:06] logging.py:157 >> {'loss': 0.7117, 'learning_rate': 4.9350e-05, 'epoch': 0.07, 'throughput': 13417.34}
+
+[INFO|2025-04-28 17:31:41] logging.py:157 >> {'loss': 0.7240, 'learning_rate': 4.9179e-05, 'epoch': 0.08, 'throughput': 13429.19}
+
+[INFO|2025-04-28 17:34:16] logging.py:157 >> {'loss': 0.7144, 'learning_rate': 4.8987e-05, 'epoch': 0.09, 'throughput': 13440.96}
+
+[INFO|2025-04-28 17:36:51] logging.py:157 >> {'loss': 0.6970, 'learning_rate': 4.8776e-05, 'epoch': 0.10, 'throughput': 13448.05}
+
+[INFO|2025-04-28 17:39:26] logging.py:157 >> {'loss': 0.7117, 'learning_rate': 4.8546e-05, 'epoch': 0.11, 'throughput': 13452.88}
+
+[INFO|2025-04-28 17:42:01] logging.py:157 >> {'loss': 0.6996, 'learning_rate': 4.8297e-05, 'epoch': 0.12, 'throughput': 13458.10}
+
+[INFO|2025-04-28 17:44:36] logging.py:157 >> {'loss': 0.7172, 'learning_rate': 4.8028e-05, 'epoch': 0.13, 'throughput': 13461.87}
+
+[INFO|2025-04-28 17:47:12] logging.py:157 >> {'loss': 0.7250, 'learning_rate': 4.7741e-05, 'epoch': 0.14, 'throughput': 13465.50}
+
+[INFO|2025-04-28 17:49:47] logging.py:157 >> {'loss': 0.7073, 'learning_rate': 4.7435e-05, 'epoch': 0.14, 'throughput': 13469.44}
+
+[INFO|2025-04-28 17:52:22] logging.py:157 >> {'loss': 0.7021, 'learning_rate': 4.7111e-05, 'epoch': 0.15, 'throughput': 13473.17}
+
+[INFO|2025-04-28 17:54:57] logging.py:157 >> {'loss': 0.6894, 'learning_rate': 4.6769e-05, 'epoch': 0.16, 'throughput': 13473.99}
+
+[INFO|2025-04-28 17:57:33] logging.py:157 >> {'loss': 0.7031, 'learning_rate': 4.6409e-05, 'epoch': 0.17, 'throughput': 13471.10}
+
+[INFO|2025-04-28 18:00:08] logging.py:157 >> {'loss': 0.6629, 'learning_rate': 4.6031e-05, 'epoch': 0.18, 'throughput': 13474.21}
+
+[INFO|2025-04-28 18:02:43] logging.py:157 >> {'loss': 0.6774, 'learning_rate': 4.5637e-05, 'epoch': 0.19, 'throughput': 13476.95}
+
+[INFO|2025-04-28 18:05:18] logging.py:157 >> {'loss': 0.7041, 'learning_rate': 4.5225e-05, 'epoch': 0.20, 'throughput': 13481.25}
+
+[INFO|2025-04-28 18:07:52] logging.py:157 >> {'loss': 0.6894, 'learning_rate': 4.4798e-05, 'epoch': 0.21, 'throughput': 13484.61}
+
+[INFO|2025-04-28 18:10:27] logging.py:157 >> {'loss': 0.6730, 'learning_rate': 4.4354e-05, 'epoch': 0.22, 'throughput': 13488.41}
+
+[INFO|2025-04-28 18:13:02] logging.py:157 >> {'loss': 0.6691, 'learning_rate': 4.3894e-05, 'epoch': 0.23, 'throughput': 13490.63}
+
+[INFO|2025-04-28 18:15:36] logging.py:157 >> {'loss': 0.6794, 'learning_rate': 4.3419e-05, 'epoch': 0.23, 'throughput': 13493.84}
+
+[INFO|2025-04-28 18:18:12] logging.py:157 >> {'loss': 0.6946, 'learning_rate': 4.2928e-05, 'epoch': 0.24, 'throughput': 13493.72}
+
+[INFO|2025-04-28 18:20:47] logging.py:157 >> {'loss': 0.6939, 'learning_rate': 4.2423e-05, 'epoch': 0.25, 'throughput': 13495.64}
+
+[INFO|2025-04-28 18:23:21] logging.py:157 >> {'loss': 0.6759, 'learning_rate': 4.1904e-05, 'epoch': 0.26, 'throughput': 13497.79}
+
+[INFO|2025-04-28 18:26:01] logging.py:157 >> {'loss': 0.6856, 'learning_rate': 4.1372e-05, 'epoch': 0.27, 'throughput': 13485.48}
+
+[INFO|2025-04-28 18:28:39] logging.py:157 >> {'loss': 0.6753, 'learning_rate': 4.0825e-05, 'epoch': 0.28, 'throughput': 13476.76}
+
+[INFO|2025-04-28 18:31:18] logging.py:157 >> {'loss': 0.6769, 'learning_rate': 4.0266e-05, 'epoch': 0.29, 'throughput': 13467.64}
+
+[INFO|2025-04-28 18:33:57] logging.py:157 >> {'loss': 0.6664, 'learning_rate': 3.9695e-05, 'epoch': 0.30, 'throughput': 13460.91}
+
+[INFO|2025-04-28 18:36:34] logging.py:157 >> {'loss': 0.6383, 'learning_rate': 3.9111e-05, 'epoch': 0.31, 'throughput': 13456.16}
+
+[INFO|2025-04-28 18:39:13] logging.py:157 >> {'loss': 0.6726, 'learning_rate': 3.8516e-05, 'epoch': 0.32, 'throughput': 13449.18}
+
+[INFO|2025-04-28 18:41:51] logging.py:157 >> {'loss': 0.6612, 'learning_rate': 3.7910e-05, 'epoch': 0.32, 'throughput': 13443.43}
+
+[INFO|2025-04-28 18:44:30] logging.py:157 >> {'loss': 0.6763, 'learning_rate': 3.7293e-05, 'epoch': 0.33, 'throughput': 13438.40}
+
+[INFO|2025-04-28 18:47:08] logging.py:157 >> {'loss': 0.6533, 'learning_rate': 3.6667e-05, 'epoch': 0.34, 'throughput': 13433.21}
+
+[INFO|2025-04-28 18:49:46] logging.py:157 >> {'loss': 0.6598, 'learning_rate': 3.6031e-05, 'epoch': 0.35, 'throughput': 13429.21}
+
+[INFO|2025-04-28 18:52:24] logging.py:157 >> {'loss': 0.6801, 'learning_rate': 3.5385e-05, 'epoch': 0.36, 'throughput': 13424.38}
+
+[INFO|2025-04-28 18:55:03] logging.py:157 >> {'loss': 0.6668, 'learning_rate': 3.4732e-05, 'epoch': 0.37, 'throughput': 13420.03}
+
+[INFO|2025-04-28 18:57:41] logging.py:157 >> {'loss': 0.6756, 'learning_rate': 3.4070e-05, 'epoch': 0.38, 'throughput': 13414.75}
+
+[INFO|2025-04-28 19:00:19] logging.py:157 >> {'loss': 0.6589, 'learning_rate': 3.3401e-05, 'epoch': 0.39, 'throughput': 13411.37}
+
+[INFO|2025-04-28 19:02:57] logging.py:157 >> {'loss': 0.6435, 'learning_rate': 3.2725e-05, 'epoch': 0.40, 'throughput': 13408.63}
+
+[INFO|2025-04-28 19:05:35] logging.py:157 >> {'loss': 0.6602, 'learning_rate': 3.2043e-05, 'epoch': 0.41, 'throughput': 13405.22}
+
+[INFO|2025-04-28 19:08:14] logging.py:157 >> {'loss': 0.6480, 'learning_rate': 3.1355e-05, 'epoch': 0.41, 'throughput': 13401.83}
+
+[INFO|2025-04-28 19:10:52] logging.py:157 >> {'loss': 0.6493, 'learning_rate': 3.0662e-05, 'epoch': 0.42, 'throughput': 13398.84}
+
+[INFO|2025-04-28 19:13:30] logging.py:157 >> {'loss': 0.6701, 'learning_rate': 2.9965e-05, 'epoch': 0.43, 'throughput': 13395.76}
+
+[INFO|2025-04-28 19:16:08] logging.py:157 >> {'loss': 0.6674, 'learning_rate': 2.9263e-05, 'epoch': 0.44, 'throughput': 13392.56}
+
+[INFO|2025-04-28 19:18:47] logging.py:157 >> {'loss': 0.6635, 'learning_rate': 2.8558e-05, 'epoch': 0.45, 'throughput': 13389.71}
+
+[INFO|2025-04-28 19:21:25] logging.py:157 >> {'loss': 0.6598, 'learning_rate': 2.7850e-05, 'epoch': 0.46, 'throughput': 13386.60}
+
+[INFO|2025-04-28 19:24:04] logging.py:157 >> {'loss': 0.6732, 'learning_rate': 2.7139e-05, 'epoch': 0.47, 'throughput': 13383.73}
+
+[INFO|2025-04-28 19:26:42] logging.py:157 >> {'loss': 0.6546, 'learning_rate': 2.6427e-05, 'epoch': 0.48, 'throughput': 13380.78}
+
+[INFO|2025-04-28 19:29:20] logging.py:157 >> {'loss': 0.6705, 'learning_rate': 2.5714e-05, 'epoch': 0.49, 'throughput': 13379.10}
+
+[INFO|2025-04-28 19:31:58] logging.py:157 >> {'loss': 0.6463, 'learning_rate': 2.5000e-05, 'epoch': 0.50, 'throughput': 13376.64}
+
+[INFO|2025-04-28 19:34:36] logging.py:157 >> {'loss': 0.6505, 'learning_rate': 2.4286e-05, 'epoch': 0.51, 'throughput': 13375.59}
+
+[INFO|2025-04-28 19:37:14] logging.py:157 >> {'loss': 0.6524, 'learning_rate': 2.3573e-05, 'epoch': 0.51, 'throughput': 13373.95}
+
+[INFO|2025-04-28 19:39:52] logging.py:157 >> {'loss': 0.6559, 'learning_rate': 2.2861e-05, 'epoch': 0.52, 'throughput': 13371.57}
+
+[INFO|2025-04-28 19:42:31] logging.py:157 >> {'loss': 0.6511, 'learning_rate': 2.2150e-05, 'epoch': 0.53, 'throughput': 13368.89}
+
+[INFO|2025-04-28 19:45:09] logging.py:157 >> {'loss': 0.6457, 'learning_rate': 2.1442e-05, 'epoch': 0.54, 'throughput': 13366.30}
+
+[INFO|2025-04-28 19:47:48] logging.py:157 >> {'loss': 0.6492, 'learning_rate': 2.0737e-05, 'epoch': 0.55, 'throughput': 13364.48}
+
+[INFO|2025-04-28 19:50:25] logging.py:157 >> {'loss': 0.6539, 'learning_rate': 2.0035e-05, 'epoch': 0.56, 'throughput': 13363.30}
+
+[INFO|2025-04-28 19:53:03] logging.py:157 >> {'loss': 0.6417, 'learning_rate': 1.9338e-05, 'epoch': 0.57, 'throughput': 13362.41}
+
+[INFO|2025-04-28 19:55:41] logging.py:157 >> {'loss': 0.6663, 'learning_rate': 1.8645e-05, 'epoch': 0.58, 'throughput': 13360.92}
+
+[INFO|2025-04-28 19:58:19] logging.py:157 >> {'loss': 0.6572, 'learning_rate': 1.7957e-05, 'epoch': 0.59, 'throughput': 13360.07}
+
+[INFO|2025-04-28 20:00:57] logging.py:157 >> {'loss': 0.6825, 'learning_rate': 1.7275e-05, 'epoch': 0.60, 'throughput': 13358.01}
+
+[INFO|2025-04-28 20:03:35] logging.py:157 >> {'loss': 0.6509, 'learning_rate': 1.6599e-05, 'epoch': 0.60, 'throughput': 13356.51}
+
+[INFO|2025-04-28 20:06:13] logging.py:157 >> {'loss': 0.6619, 'learning_rate': 1.5930e-05, 'epoch': 0.61, 'throughput': 13355.52}
+
+[INFO|2025-04-28 20:08:52] logging.py:157 >> {'loss': 0.6759, 'learning_rate': 1.5268e-05, 'epoch': 0.62, 'throughput': 13352.79}
+
+[INFO|2025-04-28 20:11:30] logging.py:157 >> {'loss': 0.6568, 'learning_rate': 1.4615e-05, 'epoch': 0.63, 'throughput': 13352.37}
+
+[INFO|2025-04-28 20:14:08] logging.py:157 >> {'loss': 0.6472, 'learning_rate': 1.3969e-05, 'epoch': 0.64, 'throughput': 13351.50}
+
+[INFO|2025-04-28 20:16:47] logging.py:157 >> {'loss': 0.6473, 'learning_rate': 1.3333e-05, 'epoch': 0.65, 'throughput': 13349.03}
+
+[INFO|2025-04-28 20:19:25] logging.py:157 >> {'loss': 0.6485, 'learning_rate': 1.2707e-05, 'epoch': 0.66, 'throughput': 13347.43}
+
+[INFO|2025-04-28 20:22:04] logging.py:157 >> {'loss': 0.6544, 'learning_rate': 1.2090e-05, 'epoch': 0.67, 'throughput': 13345.97}
+
+[INFO|2025-04-28 20:24:42] logging.py:157 >> {'loss': 0.6763, 'learning_rate': 1.1484e-05, 'epoch': 0.68, 'throughput': 13344.93}
+
+[INFO|2025-04-28 20:27:21] logging.py:157 >> {'loss': 0.6406, 'learning_rate': 1.0889e-05, 'epoch': 0.69, 'throughput': 13342.47}
+
+[INFO|2025-04-28 20:30:00] logging.py:157 >> {'loss': 0.6502, 'learning_rate': 1.0305e-05, 'epoch': 0.69, 'throughput': 13341.02}
+
+[INFO|2025-04-28 20:32:38] logging.py:157 >> {'loss': 0.6495, 'learning_rate': 9.7338e-06, 'epoch': 0.70, 'throughput': 13339.92}
+
+[INFO|2025-04-28 20:35:16] logging.py:157 >> {'loss': 0.6469, 'learning_rate': 9.1747e-06, 'epoch': 0.71, 'throughput': 13339.19}
+
+[INFO|2025-04-28 20:37:54] logging.py:157 >> {'loss': 0.6642, 'learning_rate': 8.6285e-06, 'epoch': 0.72, 'throughput': 13337.79}
+
+[INFO|2025-04-28 20:40:32] logging.py:157 >> {'loss': 0.6598, 'learning_rate': 8.0956e-06, 'epoch': 0.73, 'throughput': 13336.77}
+
+[INFO|2025-04-28 20:43:11] logging.py:157 >> {'loss': 0.6461, 'learning_rate': 7.5766e-06, 'epoch': 0.74, 'throughput': 13335.06}
+
+[INFO|2025-04-28 20:45:50] logging.py:157 >> {'loss': 0.6706, 'learning_rate': 7.0717e-06, 'epoch': 0.75, 'throughput': 13333.27}
+
+[INFO|2025-04-28 20:48:28] logging.py:157 >> {'loss': 0.6768, 'learning_rate': 6.5815e-06, 'epoch': 0.76, 'throughput': 13332.46}
+
+[INFO|2025-04-28 20:51:06] logging.py:157 >> {'loss': 0.6446, 'learning_rate': 6.1063e-06, 'epoch': 0.77, 'throughput': 13331.91}
+
+[INFO|2025-04-28 20:53:44] logging.py:157 >> {'loss': 0.6393, 'learning_rate': 5.6465e-06, 'epoch': 0.78, 'throughput': 13331.08}
+
+[INFO|2025-04-28 20:56:22] logging.py:157 >> {'loss': 0.6585, 'learning_rate': 5.2024e-06, 'epoch': 0.78, 'throughput': 13330.39}
+
+[INFO|2025-04-28 20:59:01] logging.py:157 >> {'loss': 0.6393, 'learning_rate': 4.7746e-06, 'epoch': 0.79, 'throughput': 13329.42}
+
+[INFO|2025-04-28 21:01:38] logging.py:157 >> {'loss': 0.6613, 'learning_rate': 4.3632e-06, 'epoch': 0.80, 'throughput': 13329.14}
+
+[INFO|2025-04-28 21:04:18] logging.py:157 >> {'loss': 0.6631, 'learning_rate': 3.9687e-06, 'epoch': 0.81, 'throughput': 13327.42}
+
+[INFO|2025-04-28 21:06:56] logging.py:157 >> {'loss': 0.6356, 'learning_rate': 3.5913e-06, 'epoch': 0.82, 'throughput': 13326.22}
+
+[INFO|2025-04-28 21:09:35] logging.py:157 >> {'loss': 0.6475, 'learning_rate': 3.2313e-06, 'epoch': 0.83, 'throughput': 13325.12}
+
+[INFO|2025-04-28 21:12:13] logging.py:157 >> {'loss': 0.6387, 'learning_rate': 2.8892e-06, 'epoch': 0.84, 'throughput': 13324.10}
+
+[INFO|2025-04-28 21:14:52] logging.py:157 >> {'loss': 0.6533, 'learning_rate': 2.5650e-06, 'epoch': 0.85, 'throughput': 13323.33}
+
+[INFO|2025-04-28 21:17:30] logging.py:157 >> {'loss': 0.6606, 'learning_rate': 2.2592e-06, 'epoch': 0.86, 'throughput': 13322.67}
+
+[INFO|2025-04-28 21:20:08] logging.py:157 >> {'loss': 0.6667, 'learning_rate': 1.9719e-06, 'epoch': 0.87, 'throughput': 13322.10}
+
+[INFO|2025-04-28 21:22:46] logging.py:157 >> {'loss': 0.6599, 'learning_rate': 1.7034e-06, 'epoch': 0.87, 'throughput': 13321.16}
+
+[INFO|2025-04-28 21:25:25] logging.py:157 >> {'loss': 0.6499, 'learning_rate': 1.4539e-06, 'epoch': 0.88, 'throughput': 13319.93}
+
+[INFO|2025-04-28 21:28:03] logging.py:157 >> {'loss': 0.6490, 'learning_rate': 1.2236e-06, 'epoch': 0.89, 'throughput': 13319.17}
+
+[INFO|2025-04-28 21:30:42] logging.py:157 >> {'loss': 0.6551, 'learning_rate': 1.0127e-06, 'epoch': 0.90, 'throughput': 13318.33}
+
+[INFO|2025-04-28 21:33:20] logging.py:157 >> {'loss': 0.6680, 'learning_rate': 8.2133e-07, 'epoch': 0.91, 'throughput': 13317.49}
+
+[INFO|2025-04-28 21:35:59] logging.py:157 >> {'loss': 0.6505, 'learning_rate': 6.4970e-07, 'epoch': 0.92, 'throughput': 13316.86}
+
+[INFO|2025-04-28 21:38:36] logging.py:157 >> {'loss': 0.6504, 'learning_rate': 4.9794e-07, 'epoch': 0.93, 'throughput': 13317.27}
+
+[INFO|2025-04-28 21:41:13] logging.py:157 >> {'loss': 0.6504, 'learning_rate': 3.6615e-07, 'epoch': 0.94, 'throughput': 13317.01}
+
+[INFO|2025-04-28 21:43:52] logging.py:157 >> {'loss': 0.6491, 'learning_rate': 2.5446e-07, 'epoch': 0.95, 'throughput': 13316.43}
+
+[INFO|2025-04-28 21:46:29] logging.py:157 >> {'loss': 0.6449, 'learning_rate': 1.6296e-07, 'epoch': 0.96, 'throughput': 13316.07}
+
+[INFO|2025-04-28 21:49:07] logging.py:157 >> {'loss': 0.6729, 'learning_rate': 9.1707e-08, 'epoch': 0.97, 'throughput': 13315.82}
+
+[INFO|2025-04-28 21:51:45] logging.py:157 >> {'loss': 0.6308, 'learning_rate': 4.0772e-08, 'epoch': 0.97, 'throughput': 13315.41}
+
+[INFO|2025-04-28 21:54:24] logging.py:157 >> {'loss': 0.6695, 'learning_rate': 1.0195e-08, 'epoch': 0.98, 'throughput': 13314.76}
+
+[INFO|2025-04-28 21:57:02] logging.py:157 >> {'loss': 0.6467, 'learning_rate': 0.0000e+00, 'epoch': 0.99, 'throughput': 13314.02}
+
+[INFO|2025-04-28 21:57:02] trainer.py:3910 >> Saving model checkpoint to saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/checkpoint-110
+
+[INFO|2025-04-28 21:57:02] configuration_utils.py:420 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/checkpoint-110/config.json
+
+[INFO|2025-04-28 21:57:02] configuration_utils.py:909 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/checkpoint-110/generation_config.json
+
+[INFO|2025-04-28 21:57:27] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/checkpoint-110/model.safetensors.index.json.
+
+[INFO|2025-04-28 21:57:27] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/checkpoint-110/tokenizer_config.json
+
+[INFO|2025-04-28 21:57:27] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/checkpoint-110/special_tokens_map.json
+
+[INFO|2025-04-28 21:57:27] trainer.py:2643 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|2025-04-28 21:57:27] trainer.py:3910 >> Saving model checkpoint to saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx
+
+[INFO|2025-04-28 21:57:27] configuration_utils.py:420 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/config.json
+
+[INFO|2025-04-28 21:57:27] configuration_utils.py:909 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/generation_config.json
+
+[INFO|2025-04-28 21:57:52] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/model.safetensors.index.json.
+
+[INFO|2025-04-28 21:57:52] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/tokenizer_config.json
+
+[INFO|2025-04-28 21:57:52] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx/special_tokens_map.json
+
+[WARNING|2025-04-28 21:57:52] logging.py:162 >> No metric eval_loss to plot.
+
+[WARNING|2025-04-28 21:57:52] logging.py:162 >> No metric eval_accuracy to plot.
+
+[INFO|2025-04-28 21:57:52] modelcard.py:449 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.992108229988726,
+    "num_input_tokens_seen": 230686720,
+    "total_flos": 9.786587384895242e+18,
+    "train_loss": 0.6728460962122137,
+    "train_runtime": 17352.9545,
+    "train_samples_per_second": 3.271,
+    "train_steps_per_second": 0.006
+}

trainer_log.jsonl

@@ -0,0 +1,111 @@
+{"current_steps": 1, "total_steps": 110, "loss": 0.8968, "lr": 4.9989804820704735e-05, "epoch": 0.009019165727170236, "percentage": 0.91, "elapsed_time": "0:02:44", "remaining_time": "4:58:29", "throughput": 12763.83, "total_tokens": 2097152}
+{"current_steps": 2, "total_steps": 110, "loss": 0.8202, "lr": 4.995922759815339e-05, "epoch": 0.018038331454340473, "percentage": 1.82, "elapsed_time": "0:05:20", "remaining_time": "4:48:20", "throughput": 13091.94, "total_tokens": 4194304}
+{"current_steps": 3, "total_steps": 110, "loss": 0.7783, "lr": 4.9908293271567286e-05, "epoch": 0.02705749718151071, "percentage": 2.73, "elapsed_time": "0:07:55", "remaining_time": "4:42:50", "throughput": 13222.77, "total_tokens": 6291456}
+{"current_steps": 4, "total_steps": 110, "loss": 0.7642, "lr": 4.9837043383713753e-05, "epoch": 0.036076662908680945, "percentage": 3.64, "elapsed_time": "0:10:31", "remaining_time": "4:38:50", "throughput": 13287.21, "total_tokens": 8388608}
+{"current_steps": 5, "total_steps": 110, "loss": 0.7475, "lr": 4.9745536047023324e-05, "epoch": 0.04509582863585118, "percentage": 4.55, "elapsed_time": "0:13:06", "remaining_time": "4:35:18", "throughput": 13330.27, "total_tokens": 10485760}
+{"current_steps": 6, "total_steps": 110, "loss": 0.7319, "lr": 4.963384589619233e-05, "epoch": 0.05411499436302142, "percentage": 5.45, "elapsed_time": "0:15:41", "remaining_time": "4:31:54", "throughput": 13369.02, "total_tokens": 12582912}
+{"current_steps": 7, "total_steps": 110, "loss": 0.7125, "lr": 4.9502064027309836e-05, "epoch": 0.06313416009019165, "percentage": 6.36, "elapsed_time": "0:18:15", "remaining_time": "4:28:42", "throughput": 13397.69, "total_tokens": 14680064}
+{"current_steps": 8, "total_steps": 110, "loss": 0.7117, "lr": 4.935029792355834e-05, "epoch": 0.07215332581736189, "percentage": 7.27, "elapsed_time": "0:20:50", "remaining_time": "4:25:42", "throughput": 13417.34, "total_tokens": 16777216}
+{"current_steps": 9, "total_steps": 110, "loss": 0.724, "lr": 4.917867136754893e-05, "epoch": 0.08117249154453213, "percentage": 8.18, "elapsed_time": "0:23:25", "remaining_time": "4:22:52", "throughput": 13429.19, "total_tokens": 18874368}
+{"current_steps": 10, "total_steps": 110, "loss": 0.7144, "lr": 4.898732434036244e-05, "epoch": 0.09019165727170236, "percentage": 9.09, "elapsed_time": "0:26:00", "remaining_time": "4:20:02", "throughput": 13440.96, "total_tokens": 20971520}
+{"current_steps": 11, "total_steps": 110, "loss": 0.697, "lr": 4.877641290737884e-05, "epoch": 0.0992108229988726, "percentage": 10.0, "elapsed_time": "0:28:35", "remaining_time": "4:17:18", "throughput": 13448.05, "total_tokens": 23068672}
+{"current_steps": 12, "total_steps": 110, "loss": 0.7117, "lr": 4.854610909098812e-05, "epoch": 0.10822998872604284, "percentage": 10.91, "elapsed_time": "0:31:10", "remaining_time": "4:14:37", "throughput": 13452.88, "total_tokens": 25165824}
+{"current_steps": 13, "total_steps": 110, "loss": 0.6996, "lr": 4.829660073028631e-05, "epoch": 0.11724915445321307, "percentage": 11.82, "elapsed_time": "0:33:45", "remaining_time": "4:11:55", "throughput": 13458.1, "total_tokens": 27262976}
+{"current_steps": 14, "total_steps": 110, "loss": 0.7172, "lr": 4.802809132787125e-05, "epoch": 0.1262683201803833, "percentage": 12.73, "elapsed_time": "0:36:20", "remaining_time": "4:09:15", "throughput": 13461.87, "total_tokens": 29360128}
+{"current_steps": 15, "total_steps": 110, "loss": 0.725, "lr": 4.774079988386296e-05, "epoch": 0.13528748590755355, "percentage": 13.64, "elapsed_time": "0:38:56", "remaining_time": "4:06:35", "throughput": 13465.5, "total_tokens": 31457280}
+{"current_steps": 16, "total_steps": 110, "loss": 0.7073, "lr": 4.743496071728396e-05, "epoch": 0.14430665163472378, "percentage": 14.55, "elapsed_time": "0:41:31", "remaining_time": "4:03:55", "throughput": 13469.44, "total_tokens": 33554432}
+{"current_steps": 17, "total_steps": 110, "loss": 0.7021, "lr": 4.711082327494536e-05, "epoch": 0.15332581736189402, "percentage": 15.45, "elapsed_time": "0:44:06", "remaining_time": "4:01:15", "throughput": 13473.17, "total_tokens": 35651584}
+{"current_steps": 18, "total_steps": 110, "loss": 0.6894, "lr": 4.6768651927994434e-05, "epoch": 0.16234498308906425, "percentage": 16.36, "elapsed_time": "0:46:41", "remaining_time": "3:58:39", "throughput": 13473.99, "total_tokens": 37748736}
+{"current_steps": 19, "total_steps": 110, "loss": 0.7031, "lr": 4.640872575628973e-05, "epoch": 0.1713641488162345, "percentage": 17.27, "elapsed_time": "0:49:17", "remaining_time": "3:56:06", "throughput": 13471.1, "total_tokens": 39845888}
+{"current_steps": 20, "total_steps": 110, "loss": 0.6629, "lr": 4.6031338320779534e-05, "epoch": 0.18038331454340473, "percentage": 18.18, "elapsed_time": "0:51:52", "remaining_time": "3:53:27", "throughput": 13474.21, "total_tokens": 41943040}
+{"current_steps": 21, "total_steps": 110, "loss": 0.6774, "lr": 4.563679742406935e-05, "epoch": 0.18940248027057496, "percentage": 19.09, "elapsed_time": "0:54:27", "remaining_time": "3:50:49", "throughput": 13476.95, "total_tokens": 44040192}
+{"current_steps": 22, "total_steps": 110, "loss": 0.7041, "lr": 4.522542485937369e-05, "epoch": 0.1984216459977452, "percentage": 20.0, "elapsed_time": "0:57:02", "remaining_time": "3:48:09", "throughput": 13481.25, "total_tokens": 46137344}
+{"current_steps": 23, "total_steps": 110, "loss": 0.6894, "lr": 4.479755614805688e-05, "epoch": 0.20744081172491544, "percentage": 20.91, "elapsed_time": "0:59:37", "remaining_time": "3:45:30", "throughput": 13484.61, "total_tokens": 48234496}
+{"current_steps": 24, "total_steps": 110, "loss": 0.673, "lr": 4.4353540265977064e-05, "epoch": 0.21645997745208567, "percentage": 21.82, "elapsed_time": "1:02:11", "remaining_time": "3:42:51", "throughput": 13488.41, "total_tokens": 50331648}
+{"current_steps": 25, "total_steps": 110, "loss": 0.6691, "lr": 4.389373935885646e-05, "epoch": 0.2254791431792559, "percentage": 22.73, "elapsed_time": "1:04:46", "remaining_time": "3:40:13", "throughput": 13490.63, "total_tokens": 52428800}
+{"current_steps": 26, "total_steps": 110, "loss": 0.6794, "lr": 4.341852844691012e-05, "epoch": 0.23449830890642615, "percentage": 23.64, "elapsed_time": "1:07:20", "remaining_time": "3:37:34", "throughput": 13493.84, "total_tokens": 54525952}
+{"current_steps": 27, "total_steps": 110, "loss": 0.6946, "lr": 4.292829511897409e-05, "epoch": 0.24351747463359638, "percentage": 24.55, "elapsed_time": "1:09:56", "remaining_time": "3:34:59", "throughput": 13493.72, "total_tokens": 56623104}
+{"current_steps": 28, "total_steps": 110, "loss": 0.6939, "lr": 4.242343921638234e-05, "epoch": 0.2525366403607666, "percentage": 25.45, "elapsed_time": "1:12:31", "remaining_time": "3:32:22", "throughput": 13495.64, "total_tokens": 58720256}
+{"current_steps": 29, "total_steps": 110, "loss": 0.6759, "lr": 4.1904372506850484e-05, "epoch": 0.2615558060879369, "percentage": 26.36, "elapsed_time": "1:15:05", "remaining_time": "3:29:44", "throughput": 13497.79, "total_tokens": 60817408}
+{"current_steps": 30, "total_steps": 110, "loss": 0.6856, "lr": 4.137151834863213e-05, "epoch": 0.2705749718151071, "percentage": 27.27, "elapsed_time": "1:17:45", "remaining_time": "3:27:20", "throughput": 13485.48, "total_tokens": 62914560}
+{"current_steps": 31, "total_steps": 110, "loss": 0.6753, "lr": 4.082531134522176e-05, "epoch": 0.27959413754227735, "percentage": 28.18, "elapsed_time": "1:20:23", "remaining_time": "3:24:53", "throughput": 13476.76, "total_tokens": 65011712}
+{"current_steps": 32, "total_steps": 110, "loss": 0.6769, "lr": 4.0266196990885955e-05, "epoch": 0.28861330326944756, "percentage": 29.09, "elapsed_time": "1:23:02", "remaining_time": "3:22:25", "throughput": 13467.64, "total_tokens": 67108864}
+{"current_steps": 33, "total_steps": 110, "loss": 0.6664, "lr": 3.969463130731183e-05, "epoch": 0.2976324689966178, "percentage": 30.0, "elapsed_time": "1:25:41", "remaining_time": "3:19:56", "throughput": 13460.91, "total_tokens": 69206016}
+{"current_steps": 34, "total_steps": 110, "loss": 0.6383, "lr": 3.911108047166924e-05, "epoch": 0.30665163472378804, "percentage": 30.91, "elapsed_time": "1:28:18", "remaining_time": "3:17:24", "throughput": 13456.16, "total_tokens": 71303168}
+{"current_steps": 35, "total_steps": 110, "loss": 0.6726, "lr": 3.851602043638994e-05, "epoch": 0.3156708004509583, "percentage": 31.82, "elapsed_time": "1:30:57", "remaining_time": "3:14:54", "throughput": 13449.18, "total_tokens": 73400320}
+{"current_steps": 36, "total_steps": 110, "loss": 0.6612, "lr": 3.790993654097405e-05, "epoch": 0.3246899661781285, "percentage": 32.73, "elapsed_time": "1:33:35", "remaining_time": "3:12:23", "throughput": 13443.43, "total_tokens": 75497472}
+{"current_steps": 37, "total_steps": 110, "loss": 0.6763, "lr": 3.72933231161401e-05, "epoch": 0.3337091319052988, "percentage": 33.64, "elapsed_time": "1:36:14", "remaining_time": "3:09:52", "throughput": 13438.4, "total_tokens": 77594624}
+{"current_steps": 38, "total_steps": 110, "loss": 0.6533, "lr": 3.6666683080641846e-05, "epoch": 0.342728297632469, "percentage": 34.55, "elapsed_time": "1:38:52", "remaining_time": "3:07:20", "throughput": 13433.21, "total_tokens": 79691776}
+{"current_steps": 39, "total_steps": 110, "loss": 0.6598, "lr": 3.603052753108053e-05, "epoch": 0.35174746335963925, "percentage": 35.45, "elapsed_time": "1:41:30", "remaining_time": "3:04:47", "throughput": 13429.21, "total_tokens": 81788928}
+{"current_steps": 40, "total_steps": 110, "loss": 0.6801, "lr": 3.5385375325047166e-05, "epoch": 0.36076662908680945, "percentage": 36.36, "elapsed_time": "1:44:08", "remaining_time": "3:02:15", "throughput": 13424.38, "total_tokens": 83886080}
+{"current_steps": 41, "total_steps": 110, "loss": 0.6668, "lr": 3.4731752657934794e-05, "epoch": 0.3697857948139797, "percentage": 37.27, "elapsed_time": "1:46:47", "remaining_time": "2:59:42", "throughput": 13420.03, "total_tokens": 85983232}
+{"current_steps": 42, "total_steps": 110, "loss": 0.6756, "lr": 3.4070192633766025e-05, "epoch": 0.3788049605411499, "percentage": 38.18, "elapsed_time": "1:49:25", "remaining_time": "2:57:10", "throughput": 13414.75, "total_tokens": 88080384}
+{"current_steps": 43, "total_steps": 110, "loss": 0.6589, "lr": 3.3401234830385756e-05, "epoch": 0.3878241262683202, "percentage": 39.09, "elapsed_time": "1:52:03", "remaining_time": "2:54:36", "throughput": 13411.37, "total_tokens": 90177536}
+{"current_steps": 44, "total_steps": 110, "loss": 0.6435, "lr": 3.272542485937369e-05, "epoch": 0.3968432919954904, "percentage": 40.0, "elapsed_time": "1:54:41", "remaining_time": "2:52:02", "throughput": 13408.63, "total_tokens": 92274688}
+{"current_steps": 45, "total_steps": 110, "loss": 0.6602, "lr": 3.2043313921035743e-05, "epoch": 0.40586245772266066, "percentage": 40.91, "elapsed_time": "1:57:19", "remaining_time": "2:49:28", "throughput": 13405.22, "total_tokens": 94371840}
+{"current_steps": 46, "total_steps": 110, "loss": 0.648, "lr": 3.135545835483718e-05, "epoch": 0.41488162344983087, "percentage": 41.82, "elapsed_time": "1:59:58", "remaining_time": "2:46:54", "throughput": 13401.83, "total_tokens": 96468992}
+{"current_steps": 47, "total_steps": 110, "loss": 0.6493, "lr": 3.0662419185644115e-05, "epoch": 0.42390078917700114, "percentage": 42.73, "elapsed_time": "2:02:36", "remaining_time": "2:44:20", "throughput": 13398.84, "total_tokens": 98566144}
+{"current_steps": 48, "total_steps": 110, "loss": 0.6701, "lr": 2.996476166614364e-05, "epoch": 0.43291995490417134, "percentage": 43.64, "elapsed_time": "2:05:14", "remaining_time": "2:41:46", "throughput": 13395.76, "total_tokens": 100663296}
+{"current_steps": 49, "total_steps": 110, "loss": 0.6674, "lr": 2.92630548158156e-05, "epoch": 0.4419391206313416, "percentage": 44.55, "elapsed_time": "2:07:52", "remaining_time": "2:39:12", "throughput": 13392.56, "total_tokens": 102760448}
+{"current_steps": 50, "total_steps": 110, "loss": 0.6635, "lr": 2.8557870956832132e-05, "epoch": 0.4509582863585118, "percentage": 45.45, "elapsed_time": "2:10:31", "remaining_time": "2:36:37", "throughput": 13389.71, "total_tokens": 104857600}
+{"current_steps": 51, "total_steps": 110, "loss": 0.6598, "lr": 2.7849785247263515e-05, "epoch": 0.4599774520856821, "percentage": 46.36, "elapsed_time": "2:13:09", "remaining_time": "2:34:02", "throughput": 13386.6, "total_tokens": 106954752}
+{"current_steps": 52, "total_steps": 110, "loss": 0.6732, "lr": 2.7139375211970996e-05, "epoch": 0.4689966178128523, "percentage": 47.27, "elapsed_time": "2:15:48", "remaining_time": "2:31:28", "throughput": 13383.73, "total_tokens": 109051904}
+{"current_steps": 53, "total_steps": 110, "loss": 0.6546, "lr": 2.6427220271569203e-05, "epoch": 0.47801578354002255, "percentage": 48.18, "elapsed_time": "2:18:26", "remaining_time": "2:28:53", "throughput": 13380.78, "total_tokens": 111149056}
+{"current_steps": 54, "total_steps": 110, "loss": 0.6705, "lr": 2.5713901269842404e-05, "epoch": 0.48703494926719276, "percentage": 49.09, "elapsed_time": "2:21:04", "remaining_time": "2:26:17", "throughput": 13379.1, "total_tokens": 113246208}
+{"current_steps": 55, "total_steps": 110, "loss": 0.6463, "lr": 2.5e-05, "epoch": 0.496054114994363, "percentage": 50.0, "elapsed_time": "2:23:42", "remaining_time": "2:23:42", "throughput": 13376.64, "total_tokens": 115343360}
+{"current_steps": 56, "total_steps": 110, "loss": 0.6505, "lr": 2.42860987301576e-05, "epoch": 0.5050732807215332, "percentage": 50.91, "elapsed_time": "2:26:20", "remaining_time": "2:21:06", "throughput": 13375.59, "total_tokens": 117440512}
+{"current_steps": 57, "total_steps": 110, "loss": 0.6524, "lr": 2.35727797284308e-05, "epoch": 0.5140924464487034, "percentage": 51.82, "elapsed_time": "2:28:58", "remaining_time": "2:18:30", "throughput": 13373.95, "total_tokens": 119537664}
+{"current_steps": 58, "total_steps": 110, "loss": 0.6559, "lr": 2.2860624788029013e-05, "epoch": 0.5231116121758738, "percentage": 52.73, "elapsed_time": "2:31:36", "remaining_time": "2:15:55", "throughput": 13371.57, "total_tokens": 121634816}
+{"current_steps": 59, "total_steps": 110, "loss": 0.6511, "lr": 2.2150214752736488e-05, "epoch": 0.532130777903044, "percentage": 53.64, "elapsed_time": "2:34:15", "remaining_time": "2:13:20", "throughput": 13368.89, "total_tokens": 123731968}
+{"current_steps": 60, "total_steps": 110, "loss": 0.6457, "lr": 2.1442129043167874e-05, "epoch": 0.5411499436302142, "percentage": 54.55, "elapsed_time": "2:36:53", "remaining_time": "2:10:44", "throughput": 13366.3, "total_tokens": 125829120}
+{"current_steps": 61, "total_steps": 110, "loss": 0.6492, "lr": 2.0736945184184405e-05, "epoch": 0.5501691093573844, "percentage": 55.45, "elapsed_time": "2:39:32", "remaining_time": "2:08:09", "throughput": 13364.48, "total_tokens": 127926272}
+{"current_steps": 62, "total_steps": 110, "loss": 0.6539, "lr": 2.003523833385637e-05, "epoch": 0.5591882750845547, "percentage": 56.36, "elapsed_time": "2:42:09", "remaining_time": "2:05:32", "throughput": 13363.3, "total_tokens": 130023424}
+{"current_steps": 63, "total_steps": 110, "loss": 0.6417, "lr": 1.9337580814355888e-05, "epoch": 0.5682074408117249, "percentage": 57.27, "elapsed_time": "2:44:47", "remaining_time": "2:02:56", "throughput": 13362.41, "total_tokens": 132120576}
+{"current_steps": 64, "total_steps": 110, "loss": 0.6663, "lr": 1.8644541645162834e-05, "epoch": 0.5772266065388951, "percentage": 58.18, "elapsed_time": "2:47:25", "remaining_time": "2:00:20", "throughput": 13360.92, "total_tokens": 134217728}
+{"current_steps": 65, "total_steps": 110, "loss": 0.6572, "lr": 1.795668607896426e-05, "epoch": 0.5862457722660653, "percentage": 59.09, "elapsed_time": "2:50:03", "remaining_time": "1:57:43", "throughput": 13360.07, "total_tokens": 136314880}
+{"current_steps": 66, "total_steps": 110, "loss": 0.6825, "lr": 1.7274575140626318e-05, "epoch": 0.5952649379932357, "percentage": 60.0, "elapsed_time": "2:52:41", "remaining_time": "1:55:07", "throughput": 13358.01, "total_tokens": 138412032}
+{"current_steps": 67, "total_steps": 110, "loss": 0.6509, "lr": 1.6598765169614243e-05, "epoch": 0.6042841037204059, "percentage": 60.91, "elapsed_time": "2:55:19", "remaining_time": "1:52:31", "throughput": 13356.51, "total_tokens": 140509184}
+{"current_steps": 68, "total_steps": 110, "loss": 0.6619, "lr": 1.5929807366233977e-05, "epoch": 0.6133032694475761, "percentage": 61.82, "elapsed_time": "2:57:57", "remaining_time": "1:49:55", "throughput": 13355.52, "total_tokens": 142606336}
+{"current_steps": 69, "total_steps": 110, "loss": 0.6759, "lr": 1.5268247342065215e-05, "epoch": 0.6223224351747464, "percentage": 62.73, "elapsed_time": "3:00:36", "remaining_time": "1:47:19", "throughput": 13352.79, "total_tokens": 144703488}
+{"current_steps": 70, "total_steps": 110, "loss": 0.6568, "lr": 1.4614624674952842e-05, "epoch": 0.6313416009019166, "percentage": 63.64, "elapsed_time": "3:03:14", "remaining_time": "1:44:42", "throughput": 13352.37, "total_tokens": 146800640}
+{"current_steps": 71, "total_steps": 110, "loss": 0.6472, "lr": 1.3969472468919461e-05, "epoch": 0.6403607666290868, "percentage": 64.55, "elapsed_time": "3:05:52", "remaining_time": "1:42:05", "throughput": 13351.5, "total_tokens": 148897792}
+{"current_steps": 72, "total_steps": 110, "loss": 0.6473, "lr": 1.3333316919358157e-05, "epoch": 0.649379932356257, "percentage": 65.45, "elapsed_time": "3:08:31", "remaining_time": "1:39:29", "throughput": 13349.03, "total_tokens": 150994944}
+{"current_steps": 73, "total_steps": 110, "loss": 0.6485, "lr": 1.2706676883859903e-05, "epoch": 0.6583990980834273, "percentage": 66.36, "elapsed_time": "3:11:09", "remaining_time": "1:36:53", "throughput": 13347.43, "total_tokens": 153092096}
+{"current_steps": 74, "total_steps": 110, "loss": 0.6544, "lr": 1.2090063459025955e-05, "epoch": 0.6674182638105975, "percentage": 67.27, "elapsed_time": "3:13:48", "remaining_time": "1:34:16", "throughput": 13345.97, "total_tokens": 155189248}
+{"current_steps": 75, "total_steps": 110, "loss": 0.6763, "lr": 1.148397956361007e-05, "epoch": 0.6764374295377678, "percentage": 68.18, "elapsed_time": "3:16:26", "remaining_time": "1:31:40", "throughput": 13344.93, "total_tokens": 157286400}
+{"current_steps": 76, "total_steps": 110, "loss": 0.6406, "lr": 1.0888919528330777e-05, "epoch": 0.685456595264938, "percentage": 69.09, "elapsed_time": "3:19:05", "remaining_time": "1:29:04", "throughput": 13342.47, "total_tokens": 159383552}
+{"current_steps": 77, "total_steps": 110, "loss": 0.6502, "lr": 1.0305368692688174e-05, "epoch": 0.6944757609921083, "percentage": 70.0, "elapsed_time": "3:21:44", "remaining_time": "1:26:27", "throughput": 13341.02, "total_tokens": 161480704}
+{"current_steps": 78, "total_steps": 110, "loss": 0.6495, "lr": 9.733803009114045e-06, "epoch": 0.7034949267192785, "percentage": 70.91, "elapsed_time": "3:24:22", "remaining_time": "1:23:50", "throughput": 13339.92, "total_tokens": 163577856}
+{"current_steps": 79, "total_steps": 110, "loss": 0.6469, "lr": 9.174688654778243e-06, "epoch": 0.7125140924464487, "percentage": 71.82, "elapsed_time": "3:27:00", "remaining_time": "1:21:13", "throughput": 13339.19, "total_tokens": 165675008}
+{"current_steps": 80, "total_steps": 110, "loss": 0.6642, "lr": 8.628481651367876e-06, "epoch": 0.7215332581736189, "percentage": 72.73, "elapsed_time": "3:29:38", "remaining_time": "1:18:37", "throughput": 13337.79, "total_tokens": 167772160}
+{"current_steps": 81, "total_steps": 110, "loss": 0.6598, "lr": 8.09562749314952e-06, "epoch": 0.7305524239007892, "percentage": 73.64, "elapsed_time": "3:32:16", "remaining_time": "1:16:00", "throughput": 13336.77, "total_tokens": 169869312}
+{"current_steps": 82, "total_steps": 110, "loss": 0.6461, "lr": 7.576560783617668e-06, "epoch": 0.7395715896279594, "percentage": 74.55, "elapsed_time": "3:34:55", "remaining_time": "1:13:23", "throughput": 13335.06, "total_tokens": 171966464}
+{"current_steps": 83, "total_steps": 110, "loss": 0.6706, "lr": 7.071704881025915e-06, "epoch": 0.7485907553551296, "percentage": 75.45, "elapsed_time": "3:37:34", "remaining_time": "1:10:46", "throughput": 13333.27, "total_tokens": 174063616}
+{"current_steps": 84, "total_steps": 110, "loss": 0.6768, "lr": 6.5814715530898745e-06, "epoch": 0.7576099210822999, "percentage": 76.36, "elapsed_time": "3:40:12", "remaining_time": "1:08:09", "throughput": 13332.46, "total_tokens": 176160768}
+{"current_steps": 85, "total_steps": 110, "loss": 0.6446, "lr": 6.106260641143546e-06, "epoch": 0.7666290868094702, "percentage": 77.27, "elapsed_time": "3:42:50", "remaining_time": "1:05:32", "throughput": 13331.91, "total_tokens": 178257920}
+{"current_steps": 86, "total_steps": 110, "loss": 0.6393, "lr": 5.646459734022938e-06, "epoch": 0.7756482525366404, "percentage": 78.18, "elapsed_time": "3:45:28", "remaining_time": "1:02:55", "throughput": 13331.08, "total_tokens": 180355072}
+{"current_steps": 87, "total_steps": 110, "loss": 0.6585, "lr": 5.202443851943126e-06, "epoch": 0.7846674182638106, "percentage": 79.09, "elapsed_time": "3:48:06", "remaining_time": "1:00:18", "throughput": 13330.39, "total_tokens": 182452224}
+{"current_steps": 88, "total_steps": 110, "loss": 0.6393, "lr": 4.7745751406263165e-06, "epoch": 0.7936865839909808, "percentage": 80.0, "elapsed_time": "3:50:45", "remaining_time": "0:57:41", "throughput": 13329.42, "total_tokens": 184549376}
+{"current_steps": 89, "total_steps": 110, "loss": 0.6613, "lr": 4.36320257593065e-06, "epoch": 0.8027057497181511, "percentage": 80.91, "elapsed_time": "3:53:22", "remaining_time": "0:55:04", "throughput": 13329.14, "total_tokens": 186646528}
+{"current_steps": 90, "total_steps": 110, "loss": 0.6631, "lr": 3.968661679220468e-06, "epoch": 0.8117249154453213, "percentage": 81.82, "elapsed_time": "3:56:02", "remaining_time": "0:52:27", "throughput": 13327.42, "total_tokens": 188743680}
+{"current_steps": 91, "total_steps": 110, "loss": 0.6356, "lr": 3.591274243710277e-06, "epoch": 0.8207440811724915, "percentage": 82.73, "elapsed_time": "3:58:40", "remaining_time": "0:49:50", "throughput": 13326.22, "total_tokens": 190840832}
+{"current_steps": 92, "total_steps": 110, "loss": 0.6475, "lr": 3.2313480720055745e-06, "epoch": 0.8297632468996617, "percentage": 83.64, "elapsed_time": "4:01:19", "remaining_time": "0:47:12", "throughput": 13325.12, "total_tokens": 192937984}
+{"current_steps": 93, "total_steps": 110, "loss": 0.6387, "lr": 2.889176725054643e-06, "epoch": 0.8387824126268321, "percentage": 84.55, "elapsed_time": "4:03:57", "remaining_time": "0:44:35", "throughput": 13324.1, "total_tokens": 195035136}
+{"current_steps": 94, "total_steps": 110, "loss": 0.6533, "lr": 2.565039282716045e-06, "epoch": 0.8478015783540023, "percentage": 85.45, "elapsed_time": "4:06:36", "remaining_time": "0:41:58", "throughput": 13323.33, "total_tokens": 197132288}
+{"current_steps": 95, "total_steps": 110, "loss": 0.6606, "lr": 2.2592001161370392e-06, "epoch": 0.8568207440811725, "percentage": 86.36, "elapsed_time": "4:09:14", "remaining_time": "0:39:21", "throughput": 13322.67, "total_tokens": 199229440}
+{"current_steps": 96, "total_steps": 110, "loss": 0.6667, "lr": 1.97190867212875e-06, "epoch": 0.8658399098083427, "percentage": 87.27, "elapsed_time": "4:11:52", "remaining_time": "0:36:43", "throughput": 13322.1, "total_tokens": 201326592}
+{"current_steps": 97, "total_steps": 110, "loss": 0.6599, "lr": 1.703399269713693e-06, "epoch": 0.874859075535513, "percentage": 88.18, "elapsed_time": "4:14:30", "remaining_time": "0:34:06", "throughput": 13321.16, "total_tokens": 203423744}
+{"current_steps": 98, "total_steps": 110, "loss": 0.6499, "lr": 1.4538909090118846e-06, "epoch": 0.8838782412626832, "percentage": 89.09, "elapsed_time": "4:17:09", "remaining_time": "0:31:29", "throughput": 13319.93, "total_tokens": 205520896}
+{"current_steps": 99, "total_steps": 110, "loss": 0.649, "lr": 1.2235870926211619e-06, "epoch": 0.8928974069898534, "percentage": 90.0, "elapsed_time": "4:19:47", "remaining_time": "0:28:51", "throughput": 13319.17, "total_tokens": 207618048}
+{"current_steps": 100, "total_steps": 110, "loss": 0.6551, "lr": 1.0126756596375686e-06, "epoch": 0.9019165727170236, "percentage": 90.91, "elapsed_time": "4:22:26", "remaining_time": "0:26:14", "throughput": 13318.33, "total_tokens": 209715200}
+{"current_steps": 101, "total_steps": 110, "loss": 0.668, "lr": 8.213286324510738e-07, "epoch": 0.910935738444194, "percentage": 91.82, "elapsed_time": "4:25:04", "remaining_time": "0:23:37", "throughput": 13317.49, "total_tokens": 211812352}
+{"current_steps": 102, "total_steps": 110, "loss": 0.6505, "lr": 6.497020764416633e-07, "epoch": 0.9199549041713642, "percentage": 92.73, "elapsed_time": "4:27:43", "remaining_time": "0:20:59", "throughput": 13316.86, "total_tokens": 213909504}
+{"current_steps": 103, "total_steps": 110, "loss": 0.6504, "lr": 4.979359726901639e-07, "epoch": 0.9289740698985344, "percentage": 93.64, "elapsed_time": "4:30:20", "remaining_time": "0:18:22", "throughput": 13317.27, "total_tokens": 216006656}
+{"current_steps": 104, "total_steps": 110, "loss": 0.6504, "lr": 3.6615410380767544e-07, "epoch": 0.9379932356257046, "percentage": 94.55, "elapsed_time": "4:32:57", "remaining_time": "0:15:44", "throughput": 13317.01, "total_tokens": 218103808}
+{"current_steps": 105, "total_steps": 110, "loss": 0.6491, "lr": 2.544639529766829e-07, "epoch": 0.9470124013528749, "percentage": 95.45, "elapsed_time": "4:35:36", "remaining_time": "0:13:07", "throughput": 13316.43, "total_tokens": 220200960}
+{"current_steps": 106, "total_steps": 110, "loss": 0.6449, "lr": 1.6295661628624447e-07, "epoch": 0.9560315670800451, "percentage": 96.36, "elapsed_time": "4:38:13", "remaining_time": "0:10:29", "throughput": 13316.07, "total_tokens": 222298112}
+{"current_steps": 107, "total_steps": 110, "loss": 0.6729, "lr": 9.170672843271666e-08, "epoch": 0.9650507328072153, "percentage": 97.27, "elapsed_time": "4:40:51", "remaining_time": "0:07:52", "throughput": 13315.82, "total_tokens": 224395264}
+{"current_steps": 108, "total_steps": 110, "loss": 0.6308, "lr": 4.07724018466088e-08, "epoch": 0.9740698985343855, "percentage": 98.18, "elapsed_time": "4:43:29", "remaining_time": "0:05:14", "throughput": 13315.41, "total_tokens": 226492416}
+{"current_steps": 109, "total_steps": 110, "loss": 0.6695, "lr": 1.0195179295269252e-08, "epoch": 0.9830890642615558, "percentage": 99.09, "elapsed_time": "4:46:08", "remaining_time": "0:02:37", "throughput": 13314.76, "total_tokens": 228589568}
+{"current_steps": 110, "total_steps": 110, "loss": 0.6467, "lr": 0.0, "epoch": 0.992108229988726, "percentage": 100.0, "elapsed_time": "4:48:46", "remaining_time": "0:00:00", "throughput": 13314.02, "total_tokens": 230686720}
+{"current_steps": 110, "total_steps": 110, "epoch": 0.992108229988726, "percentage": 100.0, "elapsed_time": "4:49:11", "remaining_time": "0:00:00", "throughput": 13294.64, "total_tokens": 230686720}

trainer_state.json

@@ -0,0 +1,923 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.992108229988726,
+  "eval_steps": 500,
+  "global_step": 110,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009019165727170236,
+      "grad_norm": 0.8173778653144836,
+      "learning_rate": 4.9989804820704735e-05,
+      "loss": 0.8968,
+      "num_input_tokens_seen": 2097152,
+      "step": 1
+    },
+    {
+      "epoch": 0.018038331454340473,
+      "grad_norm": 0.5661506056785583,
+      "learning_rate": 4.995922759815339e-05,
+      "loss": 0.8202,
+      "num_input_tokens_seen": 4194304,
+      "step": 2
+    },
+    {
+      "epoch": 0.02705749718151071,
+      "grad_norm": 0.4142468571662903,
+      "learning_rate": 4.9908293271567286e-05,
+      "loss": 0.7783,
+      "num_input_tokens_seen": 6291456,
+      "step": 3
+    },
+    {
+      "epoch": 0.036076662908680945,
+      "grad_norm": 0.2738800048828125,
+      "learning_rate": 4.9837043383713753e-05,
+      "loss": 0.7642,
+      "num_input_tokens_seen": 8388608,
+      "step": 4
+    },
+    {
+      "epoch": 0.04509582863585118,
+      "grad_norm": 0.2011900395154953,
+      "learning_rate": 4.9745536047023324e-05,
+      "loss": 0.7475,
+      "num_input_tokens_seen": 10485760,
+      "step": 5
+    },
+    {
+      "epoch": 0.05411499436302142,
+      "grad_norm": 0.161564901471138,
+      "learning_rate": 4.963384589619233e-05,
+      "loss": 0.7319,
+      "num_input_tokens_seen": 12582912,
+      "step": 6
+    },
+    {
+      "epoch": 0.06313416009019165,
+      "grad_norm": 0.14298053085803986,
+      "learning_rate": 4.9502064027309836e-05,
+      "loss": 0.7125,
+      "num_input_tokens_seen": 14680064,
+      "step": 7
+    },
+    {
+      "epoch": 0.07215332581736189,
+      "grad_norm": 0.14268024265766144,
+      "learning_rate": 4.935029792355834e-05,
+      "loss": 0.7117,
+      "num_input_tokens_seen": 16777216,
+      "step": 8
+    },
+    {
+      "epoch": 0.08117249154453213,
+      "grad_norm": 0.1712426245212555,
+      "learning_rate": 4.917867136754893e-05,
+      "loss": 0.724,
+      "num_input_tokens_seen": 18874368,
+      "step": 9
+    },
+    {
+      "epoch": 0.09019165727170236,
+      "grad_norm": 0.15666206181049347,
+      "learning_rate": 4.898732434036244e-05,
+      "loss": 0.7144,
+      "num_input_tokens_seen": 20971520,
+      "step": 10
+    },
+    {
+      "epoch": 0.0992108229988726,
+      "grad_norm": 0.139165461063385,
+      "learning_rate": 4.877641290737884e-05,
+      "loss": 0.697,
+      "num_input_tokens_seen": 23068672,
+      "step": 11
+    },
+    {
+      "epoch": 0.10822998872604284,
+      "grad_norm": 0.15043741464614868,
+      "learning_rate": 4.854610909098812e-05,
+      "loss": 0.7117,
+      "num_input_tokens_seen": 25165824,
+      "step": 12
+    },
+    {
+      "epoch": 0.11724915445321307,
+      "grad_norm": 0.13478276133537292,
+      "learning_rate": 4.829660073028631e-05,
+      "loss": 0.6996,
+      "num_input_tokens_seen": 27262976,
+      "step": 13
+    },
+    {
+      "epoch": 0.1262683201803833,
+      "grad_norm": 0.12838861346244812,
+      "learning_rate": 4.802809132787125e-05,
+      "loss": 0.7172,
+      "num_input_tokens_seen": 29360128,
+      "step": 14
+    },
+    {
+      "epoch": 0.13528748590755355,
+      "grad_norm": 0.11803996562957764,
+      "learning_rate": 4.774079988386296e-05,
+      "loss": 0.725,
+      "num_input_tokens_seen": 31457280,
+      "step": 15
+    },
+    {
+      "epoch": 0.14430665163472378,
+      "grad_norm": 0.11841780692338943,
+      "learning_rate": 4.743496071728396e-05,
+      "loss": 0.7073,
+      "num_input_tokens_seen": 33554432,
+      "step": 16
+    },
+    {
+      "epoch": 0.15332581736189402,
+      "grad_norm": 0.11901742964982986,
+      "learning_rate": 4.711082327494536e-05,
+      "loss": 0.7021,
+      "num_input_tokens_seen": 35651584,
+      "step": 17
+    },
+    {
+      "epoch": 0.16234498308906425,
+      "grad_norm": 0.11239754408597946,
+      "learning_rate": 4.6768651927994434e-05,
+      "loss": 0.6894,
+      "num_input_tokens_seen": 37748736,
+      "step": 18
+    },
+    {
+      "epoch": 0.1713641488162345,
+      "grad_norm": 0.10254418104887009,
+      "learning_rate": 4.640872575628973e-05,
+      "loss": 0.7031,
+      "num_input_tokens_seen": 39845888,
+      "step": 19
+    },
+    {
+      "epoch": 0.18038331454340473,
+      "grad_norm": 0.09802790731191635,
+      "learning_rate": 4.6031338320779534e-05,
+      "loss": 0.6629,
+      "num_input_tokens_seen": 41943040,
+      "step": 20
+    },
+    {
+      "epoch": 0.18940248027057496,
+      "grad_norm": 0.0993066355586052,
+      "learning_rate": 4.563679742406935e-05,
+      "loss": 0.6774,
+      "num_input_tokens_seen": 44040192,
+      "step": 21
+    },
+    {
+      "epoch": 0.1984216459977452,
+      "grad_norm": 0.09300491958856583,
+      "learning_rate": 4.522542485937369e-05,
+      "loss": 0.7041,
+      "num_input_tokens_seen": 46137344,
+      "step": 22
+    },
+    {
+      "epoch": 0.20744081172491544,
+      "grad_norm": 0.08633296191692352,
+      "learning_rate": 4.479755614805688e-05,
+      "loss": 0.6894,
+      "num_input_tokens_seen": 48234496,
+      "step": 23
+    },
+    {
+      "epoch": 0.21645997745208567,
+      "grad_norm": 0.0841744914650917,
+      "learning_rate": 4.4353540265977064e-05,
+      "loss": 0.673,
+      "num_input_tokens_seen": 50331648,
+      "step": 24
+    },
+    {
+      "epoch": 0.2254791431792559,
+      "grad_norm": 0.09111865609884262,
+      "learning_rate": 4.389373935885646e-05,
+      "loss": 0.6691,
+      "num_input_tokens_seen": 52428800,
+      "step": 25
+    },
+    {
+      "epoch": 0.23449830890642615,
+      "grad_norm": 0.08103901892900467,
+      "learning_rate": 4.341852844691012e-05,
+      "loss": 0.6794,
+      "num_input_tokens_seen": 54525952,
+      "step": 26
+    },
+    {
+      "epoch": 0.24351747463359638,
+      "grad_norm": 0.08423160016536713,
+      "learning_rate": 4.292829511897409e-05,
+      "loss": 0.6946,
+      "num_input_tokens_seen": 56623104,
+      "step": 27
+    },
+    {
+      "epoch": 0.2525366403607666,
+      "grad_norm": 0.09218155592679977,
+      "learning_rate": 4.242343921638234e-05,
+      "loss": 0.6939,
+      "num_input_tokens_seen": 58720256,
+      "step": 28
+    },
+    {
+      "epoch": 0.2615558060879369,
+      "grad_norm": 0.07839576154947281,
+      "learning_rate": 4.1904372506850484e-05,
+      "loss": 0.6759,
+      "num_input_tokens_seen": 60817408,
+      "step": 29
+    },
+    {
+      "epoch": 0.2705749718151071,
+      "grad_norm": 0.08427103608846664,
+      "learning_rate": 4.137151834863213e-05,
+      "loss": 0.6856,
+      "num_input_tokens_seen": 62914560,
+      "step": 30
+    },
+    {
+      "epoch": 0.27959413754227735,
+      "grad_norm": 0.08769369125366211,
+      "learning_rate": 4.082531134522176e-05,
+      "loss": 0.6753,
+      "num_input_tokens_seen": 65011712,
+      "step": 31
+    },
+    {
+      "epoch": 0.28861330326944756,
+      "grad_norm": 0.09144359081983566,
+      "learning_rate": 4.0266196990885955e-05,
+      "loss": 0.6769,
+      "num_input_tokens_seen": 67108864,
+      "step": 32
+    },
+    {
+      "epoch": 0.2976324689966178,
+      "grad_norm": 0.08329298347234726,
+      "learning_rate": 3.969463130731183e-05,
+      "loss": 0.6664,
+      "num_input_tokens_seen": 69206016,
+      "step": 33
+    },
+    {
+      "epoch": 0.30665163472378804,
+      "grad_norm": 0.07759370654821396,
+      "learning_rate": 3.911108047166924e-05,
+      "loss": 0.6383,
+      "num_input_tokens_seen": 71303168,
+      "step": 34
+    },
+    {
+      "epoch": 0.3156708004509583,
+      "grad_norm": 0.07686188071966171,
+      "learning_rate": 3.851602043638994e-05,
+      "loss": 0.6726,
+      "num_input_tokens_seen": 73400320,
+      "step": 35
+    },
+    {
+      "epoch": 0.3246899661781285,
+      "grad_norm": 0.0853535607457161,
+      "learning_rate": 3.790993654097405e-05,
+      "loss": 0.6612,
+      "num_input_tokens_seen": 75497472,
+      "step": 36
+    },
+    {
+      "epoch": 0.3337091319052988,
+      "grad_norm": 0.08194194734096527,
+      "learning_rate": 3.72933231161401e-05,
+      "loss": 0.6763,
+      "num_input_tokens_seen": 77594624,
+      "step": 37
+    },
+    {
+      "epoch": 0.342728297632469,
+      "grad_norm": 0.0810617133975029,
+      "learning_rate": 3.6666683080641846e-05,
+      "loss": 0.6533,
+      "num_input_tokens_seen": 79691776,
+      "step": 38
+    },
+    {
+      "epoch": 0.35174746335963925,
+      "grad_norm": 0.08145678043365479,
+      "learning_rate": 3.603052753108053e-05,
+      "loss": 0.6598,
+      "num_input_tokens_seen": 81788928,
+      "step": 39
+    },
+    {
+      "epoch": 0.36076662908680945,
+      "grad_norm": 0.07560884952545166,
+      "learning_rate": 3.5385375325047166e-05,
+      "loss": 0.6801,
+      "num_input_tokens_seen": 83886080,
+      "step": 40
+    },
+    {
+      "epoch": 0.3697857948139797,
+      "grad_norm": 0.06700561195611954,
+      "learning_rate": 3.4731752657934794e-05,
+      "loss": 0.6668,
+      "num_input_tokens_seen": 85983232,
+      "step": 41
+    },
+    {
+      "epoch": 0.3788049605411499,
+      "grad_norm": 0.076134592294693,
+      "learning_rate": 3.4070192633766025e-05,
+      "loss": 0.6756,
+      "num_input_tokens_seen": 88080384,
+      "step": 42
+    },
+    {
+      "epoch": 0.3878241262683202,
+      "grad_norm": 0.06908991187810898,
+      "learning_rate": 3.3401234830385756e-05,
+      "loss": 0.6589,
+      "num_input_tokens_seen": 90177536,
+      "step": 43
+    },
+    {
+      "epoch": 0.3968432919954904,
+      "grad_norm": 0.07035559415817261,
+      "learning_rate": 3.272542485937369e-05,
+      "loss": 0.6435,
+      "num_input_tokens_seen": 92274688,
+      "step": 44
+    },
+    {
+      "epoch": 0.40586245772266066,
+      "grad_norm": 0.06883817166090012,
+      "learning_rate": 3.2043313921035743e-05,
+      "loss": 0.6602,
+      "num_input_tokens_seen": 94371840,
+      "step": 45
+    },
+    {
+      "epoch": 0.41488162344983087,
+      "grad_norm": 0.06745729595422745,
+      "learning_rate": 3.135545835483718e-05,
+      "loss": 0.648,
+      "num_input_tokens_seen": 96468992,
+      "step": 46
+    },
+    {
+      "epoch": 0.42390078917700114,
+      "grad_norm": 0.06895168870687485,
+      "learning_rate": 3.0662419185644115e-05,
+      "loss": 0.6493,
+      "num_input_tokens_seen": 98566144,
+      "step": 47
+    },
+    {
+      "epoch": 0.43291995490417134,
+      "grad_norm": 0.06701447814702988,
+      "learning_rate": 2.996476166614364e-05,
+      "loss": 0.6701,
+      "num_input_tokens_seen": 100663296,
+      "step": 48
+    },
+    {
+      "epoch": 0.4419391206313416,
+      "grad_norm": 0.07372091710567474,
+      "learning_rate": 2.92630548158156e-05,
+      "loss": 0.6674,
+      "num_input_tokens_seen": 102760448,
+      "step": 49
+    },
+    {
+      "epoch": 0.4509582863585118,
+      "grad_norm": 0.07131503522396088,
+      "learning_rate": 2.8557870956832132e-05,
+      "loss": 0.6635,
+      "num_input_tokens_seen": 104857600,
+      "step": 50
+    },
+    {
+      "epoch": 0.4599774520856821,
+      "grad_norm": 0.0702953040599823,
+      "learning_rate": 2.7849785247263515e-05,
+      "loss": 0.6598,
+      "num_input_tokens_seen": 106954752,
+      "step": 51
+    },
+    {
+      "epoch": 0.4689966178128523,
+      "grad_norm": 0.06502864509820938,
+      "learning_rate": 2.7139375211970996e-05,
+      "loss": 0.6732,
+      "num_input_tokens_seen": 109051904,
+      "step": 52
+    },
+    {
+      "epoch": 0.47801578354002255,
+      "grad_norm": 0.06253138929605484,
+      "learning_rate": 2.6427220271569203e-05,
+      "loss": 0.6546,
+      "num_input_tokens_seen": 111149056,
+      "step": 53
+    },
+    {
+      "epoch": 0.48703494926719276,
+      "grad_norm": 0.06724034994840622,
+      "learning_rate": 2.5713901269842404e-05,
+      "loss": 0.6705,
+      "num_input_tokens_seen": 113246208,
+      "step": 54
+    },
+    {
+      "epoch": 0.496054114994363,
+      "grad_norm": 0.06660095602273941,
+      "learning_rate": 2.5e-05,
+      "loss": 0.6463,
+      "num_input_tokens_seen": 115343360,
+      "step": 55
+    },
+    {
+      "epoch": 0.5050732807215332,
+      "grad_norm": 0.06747590005397797,
+      "learning_rate": 2.42860987301576e-05,
+      "loss": 0.6505,
+      "num_input_tokens_seen": 117440512,
+      "step": 56
+    },
+    {
+      "epoch": 0.5140924464487034,
+      "grad_norm": 0.07006718963384628,
+      "learning_rate": 2.35727797284308e-05,
+      "loss": 0.6524,
+      "num_input_tokens_seen": 119537664,
+      "step": 57
+    },
+    {
+      "epoch": 0.5231116121758738,
+      "grad_norm": 0.06662806868553162,
+      "learning_rate": 2.2860624788029013e-05,
+      "loss": 0.6559,
+      "num_input_tokens_seen": 121634816,
+      "step": 58
+    },
+    {
+      "epoch": 0.532130777903044,
+      "grad_norm": 0.06757567077875137,
+      "learning_rate": 2.2150214752736488e-05,
+      "loss": 0.6511,
+      "num_input_tokens_seen": 123731968,
+      "step": 59
+    },
+    {
+      "epoch": 0.5411499436302142,
+      "grad_norm": 0.07193508744239807,
+      "learning_rate": 2.1442129043167874e-05,
+      "loss": 0.6457,
+      "num_input_tokens_seen": 125829120,
+      "step": 60
+    },
+    {
+      "epoch": 0.5501691093573844,
+      "grad_norm": 0.06620261073112488,
+      "learning_rate": 2.0736945184184405e-05,
+      "loss": 0.6492,
+      "num_input_tokens_seen": 127926272,
+      "step": 61
+    },
+    {
+      "epoch": 0.5591882750845547,
+      "grad_norm": 0.06846100836992264,
+      "learning_rate": 2.003523833385637e-05,
+      "loss": 0.6539,
+      "num_input_tokens_seen": 130023424,
+      "step": 62
+    },
+    {
+      "epoch": 0.5682074408117249,
+      "grad_norm": 0.06885959208011627,
+      "learning_rate": 1.9337580814355888e-05,
+      "loss": 0.6417,
+      "num_input_tokens_seen": 132120576,
+      "step": 63
+    },
+    {
+      "epoch": 0.5772266065388951,
+      "grad_norm": 0.06715461611747742,
+      "learning_rate": 1.8644541645162834e-05,
+      "loss": 0.6663,
+      "num_input_tokens_seen": 134217728,
+      "step": 64
+    },
+    {
+      "epoch": 0.5862457722660653,
+      "grad_norm": 0.06593496352434158,
+      "learning_rate": 1.795668607896426e-05,
+      "loss": 0.6572,
+      "num_input_tokens_seen": 136314880,
+      "step": 65
+    },
+    {
+      "epoch": 0.5952649379932357,
+      "grad_norm": 0.06741371005773544,
+      "learning_rate": 1.7274575140626318e-05,
+      "loss": 0.6825,
+      "num_input_tokens_seen": 138412032,
+      "step": 66
+    },
+    {
+      "epoch": 0.6042841037204059,
+      "grad_norm": 0.06627509742975235,
+      "learning_rate": 1.6598765169614243e-05,
+      "loss": 0.6509,
+      "num_input_tokens_seen": 140509184,
+      "step": 67
+    },
+    {
+      "epoch": 0.6133032694475761,
+      "grad_norm": 0.06725791096687317,
+      "learning_rate": 1.5929807366233977e-05,
+      "loss": 0.6619,
+      "num_input_tokens_seen": 142606336,
+      "step": 68
+    },
+    {
+      "epoch": 0.6223224351747464,
+      "grad_norm": 0.06500604748725891,
+      "learning_rate": 1.5268247342065215e-05,
+      "loss": 0.6759,
+      "num_input_tokens_seen": 144703488,
+      "step": 69
+    },
+    {
+      "epoch": 0.6313416009019166,
+      "grad_norm": 0.07281672209501266,
+      "learning_rate": 1.4614624674952842e-05,
+      "loss": 0.6568,
+      "num_input_tokens_seen": 146800640,
+      "step": 70
+    },
+    {
+      "epoch": 0.6403607666290868,
+      "grad_norm": 0.0661671832203865,
+      "learning_rate": 1.3969472468919461e-05,
+      "loss": 0.6472,
+      "num_input_tokens_seen": 148897792,
+      "step": 71
+    },
+    {
+      "epoch": 0.649379932356257,
+      "grad_norm": 0.06678120791912079,
+      "learning_rate": 1.3333316919358157e-05,
+      "loss": 0.6473,
+      "num_input_tokens_seen": 150994944,
+      "step": 72
+    },
+    {
+      "epoch": 0.6583990980834273,
+      "grad_norm": 0.06216076388955116,
+      "learning_rate": 1.2706676883859903e-05,
+      "loss": 0.6485,
+      "num_input_tokens_seen": 153092096,
+      "step": 73
+    },
+    {
+      "epoch": 0.6674182638105975,
+      "grad_norm": 0.06877604126930237,
+      "learning_rate": 1.2090063459025955e-05,
+      "loss": 0.6544,
+      "num_input_tokens_seen": 155189248,
+      "step": 74
+    },
+    {
+      "epoch": 0.6764374295377678,
+      "grad_norm": 0.06808489561080933,
+      "learning_rate": 1.148397956361007e-05,
+      "loss": 0.6763,
+      "num_input_tokens_seen": 157286400,
+      "step": 75
+    },
+    {
+      "epoch": 0.685456595264938,
+      "grad_norm": 0.06282905489206314,
+      "learning_rate": 1.0888919528330777e-05,
+      "loss": 0.6406,
+      "num_input_tokens_seen": 159383552,
+      "step": 76
+    },
+    {
+      "epoch": 0.6944757609921083,
+      "grad_norm": 0.06371884793043137,
+      "learning_rate": 1.0305368692688174e-05,
+      "loss": 0.6502,
+      "num_input_tokens_seen": 161480704,
+      "step": 77
+    },
+    {
+      "epoch": 0.7034949267192785,
+      "grad_norm": 0.06734833121299744,
+      "learning_rate": 9.733803009114045e-06,
+      "loss": 0.6495,
+      "num_input_tokens_seen": 163577856,
+      "step": 78
+    },
+    {
+      "epoch": 0.7125140924464487,
+      "grad_norm": 0.06442791223526001,
+      "learning_rate": 9.174688654778243e-06,
+      "loss": 0.6469,
+      "num_input_tokens_seen": 165675008,
+      "step": 79
+    },
+    {
+      "epoch": 0.7215332581736189,
+      "grad_norm": 0.06670290976762772,
+      "learning_rate": 8.628481651367876e-06,
+      "loss": 0.6642,
+      "num_input_tokens_seen": 167772160,
+      "step": 80
+    },
+    {
+      "epoch": 0.7305524239007892,
+      "grad_norm": 0.06524420529603958,
+      "learning_rate": 8.09562749314952e-06,
+      "loss": 0.6598,
+      "num_input_tokens_seen": 169869312,
+      "step": 81
+    },
+    {
+      "epoch": 0.7395715896279594,
+      "grad_norm": 0.06147584691643715,
+      "learning_rate": 7.576560783617668e-06,
+      "loss": 0.6461,
+      "num_input_tokens_seen": 171966464,
+      "step": 82
+    },
+    {
+      "epoch": 0.7485907553551296,
+      "grad_norm": 0.06680367887020111,
+      "learning_rate": 7.071704881025915e-06,
+      "loss": 0.6706,
+      "num_input_tokens_seen": 174063616,
+      "step": 83
+    },
+    {
+      "epoch": 0.7576099210822999,
+      "grad_norm": 0.0657731294631958,
+      "learning_rate": 6.5814715530898745e-06,
+      "loss": 0.6768,
+      "num_input_tokens_seen": 176160768,
+      "step": 84
+    },
+    {
+      "epoch": 0.7666290868094702,
+      "grad_norm": 0.06921912729740143,
+      "learning_rate": 6.106260641143546e-06,
+      "loss": 0.6446,
+      "num_input_tokens_seen": 178257920,
+      "step": 85
+    },
+    {
+      "epoch": 0.7756482525366404,
+      "grad_norm": 0.06477085500955582,
+      "learning_rate": 5.646459734022938e-06,
+      "loss": 0.6393,
+      "num_input_tokens_seen": 180355072,
+      "step": 86
+    },
+    {
+      "epoch": 0.7846674182638106,
+      "grad_norm": 0.06648603081703186,
+      "learning_rate": 5.202443851943126e-06,
+      "loss": 0.6585,
+      "num_input_tokens_seen": 182452224,
+      "step": 87
+    },
+    {
+      "epoch": 0.7936865839909808,
+      "grad_norm": 0.06899042427539825,
+      "learning_rate": 4.7745751406263165e-06,
+      "loss": 0.6393,
+      "num_input_tokens_seen": 184549376,
+      "step": 88
+    },
+    {
+      "epoch": 0.8027057497181511,
+      "grad_norm": 0.06467007100582123,
+      "learning_rate": 4.36320257593065e-06,
+      "loss": 0.6613,
+      "num_input_tokens_seen": 186646528,
+      "step": 89
+    },
+    {
+      "epoch": 0.8117249154453213,
+      "grad_norm": 0.06616765260696411,
+      "learning_rate": 3.968661679220468e-06,
+      "loss": 0.6631,
+      "num_input_tokens_seen": 188743680,
+      "step": 90
+    },
+    {
+      "epoch": 0.8207440811724915,
+      "grad_norm": 0.06439518183469772,
+      "learning_rate": 3.591274243710277e-06,
+      "loss": 0.6356,
+      "num_input_tokens_seen": 190840832,
+      "step": 91
+    },
+    {
+      "epoch": 0.8297632468996617,
+      "grad_norm": 0.06294091790914536,
+      "learning_rate": 3.2313480720055745e-06,
+      "loss": 0.6475,
+      "num_input_tokens_seen": 192937984,
+      "step": 92
+    },
+    {
+      "epoch": 0.8387824126268321,
+      "grad_norm": 0.06395729631185532,
+      "learning_rate": 2.889176725054643e-06,
+      "loss": 0.6387,
+      "num_input_tokens_seen": 195035136,
+      "step": 93
+    },
+    {
+      "epoch": 0.8478015783540023,
+      "grad_norm": 0.06324164569377899,
+      "learning_rate": 2.565039282716045e-06,
+      "loss": 0.6533,
+      "num_input_tokens_seen": 197132288,
+      "step": 94
+    },
+    {
+      "epoch": 0.8568207440811725,
+      "grad_norm": 0.06285514682531357,
+      "learning_rate": 2.2592001161370392e-06,
+      "loss": 0.6606,
+      "num_input_tokens_seen": 199229440,
+      "step": 95
+    },
+    {
+      "epoch": 0.8658399098083427,
+      "grad_norm": 0.06468215584754944,
+      "learning_rate": 1.97190867212875e-06,
+      "loss": 0.6667,
+      "num_input_tokens_seen": 201326592,
+      "step": 96
+    },
+    {
+      "epoch": 0.874859075535513,
+      "grad_norm": 0.06431297212839127,
+      "learning_rate": 1.703399269713693e-06,
+      "loss": 0.6599,
+      "num_input_tokens_seen": 203423744,
+      "step": 97
+    },
+    {
+      "epoch": 0.8838782412626832,
+      "grad_norm": 0.06363896280527115,
+      "learning_rate": 1.4538909090118846e-06,
+      "loss": 0.6499,
+      "num_input_tokens_seen": 205520896,
+      "step": 98
+    },
+    {
+      "epoch": 0.8928974069898534,
+      "grad_norm": 0.06607792526483536,
+      "learning_rate": 1.2235870926211619e-06,
+      "loss": 0.649,
+      "num_input_tokens_seen": 207618048,
+      "step": 99
+    },
+    {
+      "epoch": 0.9019165727170236,
+      "grad_norm": 0.06387382745742798,
+      "learning_rate": 1.0126756596375686e-06,
+      "loss": 0.6551,
+      "num_input_tokens_seen": 209715200,
+      "step": 100
+    },
+    {
+      "epoch": 0.910935738444194,
+      "grad_norm": 0.07034407556056976,
+      "learning_rate": 8.213286324510738e-07,
+      "loss": 0.668,
+      "num_input_tokens_seen": 211812352,
+      "step": 101
+    },
+    {
+      "epoch": 0.9199549041713642,
+      "grad_norm": 0.07087666541337967,
+      "learning_rate": 6.497020764416633e-07,
+      "loss": 0.6505,
+      "num_input_tokens_seen": 213909504,
+      "step": 102
+    },
+    {
+      "epoch": 0.9289740698985344,
+      "grad_norm": 0.07003826647996902,
+      "learning_rate": 4.979359726901639e-07,
+      "loss": 0.6504,
+      "num_input_tokens_seen": 216006656,
+      "step": 103
+    },
+    {
+      "epoch": 0.9379932356257046,
+      "grad_norm": 0.06608197838068008,
+      "learning_rate": 3.6615410380767544e-07,
+      "loss": 0.6504,
+      "num_input_tokens_seen": 218103808,
+      "step": 104
+    },
+    {
+      "epoch": 0.9470124013528749,
+      "grad_norm": 0.0653860792517662,
+      "learning_rate": 2.544639529766829e-07,
+      "loss": 0.6491,
+      "num_input_tokens_seen": 220200960,
+      "step": 105
+    },
+    {
+      "epoch": 0.9560315670800451,
+      "grad_norm": 0.06089319288730621,
+      "learning_rate": 1.6295661628624447e-07,
+      "loss": 0.6449,
+      "num_input_tokens_seen": 222298112,
+      "step": 106
+    },
+    {
+      "epoch": 0.9650507328072153,
+      "grad_norm": 0.06895752251148224,
+      "learning_rate": 9.170672843271666e-08,
+      "loss": 0.6729,
+      "num_input_tokens_seen": 224395264,
+      "step": 107
+    },
+    {
+      "epoch": 0.9740698985343855,
+      "grad_norm": 0.06209622696042061,
+      "learning_rate": 4.07724018466088e-08,
+      "loss": 0.6308,
+      "num_input_tokens_seen": 226492416,
+      "step": 108
+    },
+    {
+      "epoch": 0.9830890642615558,
+      "grad_norm": 0.06596114486455917,
+      "learning_rate": 1.0195179295269252e-08,
+      "loss": 0.6695,
+      "num_input_tokens_seen": 228589568,
+      "step": 109
+    },
+    {
+      "epoch": 0.992108229988726,
+      "grad_norm": 0.06238327547907829,
+      "learning_rate": 0.0,
+      "loss": 0.6467,
+      "num_input_tokens_seen": 230686720,
+      "step": 110
+    },
+    {
+      "epoch": 0.992108229988726,
+      "num_input_tokens_seen": 230686720,
+      "step": 110,
+      "total_flos": 9.786587384895242e+18,
+      "train_loss": 0.6728460962122137,
+      "train_runtime": 17352.9545,
+      "train_samples_per_second": 3.271,
+      "train_steps_per_second": 0.006
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 110,
+  "num_input_tokens_seen": 230686720,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.786587384895242e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a385fe18bbb396fad013d3c746e3b8b568ab94e7dcf02f90196ced85b17debfd
+size 5624

training_args.yaml

@@ -0,0 +1,39 @@
+apollo_rank: 256
+apollo_scale: 1
+apollo_target: all
+apollo_update_interval: 200
+bf16: true
+cutoff_len: 4096
+dataset: codes_330k_nsx
+dataset_dir: data
+ddp_timeout: 180000000
+do_train: true
+enable_liger_kernel: true
+finetuning_type: freeze
+flash_attn: auto
+freeze_trainable_layers: 2
+freeze_trainable_modules: all
+gradient_accumulation_steps: 8
+include_num_input_tokens_seen: true
+learning_rate: 5.0e-05
+logging_steps: 1
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_samples: 50000000
+model_name_or_path: Qwen/Qwen2.5-Coder-7B-Instruct
+neat_packing: true
+num_train_epochs: 1.0
+output_dir: saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_nsx
+packing: true
+per_device_train_batch_size: 16
+plot_loss: true
+preprocessing_num_workers: 16
+report_to: none
+rope_scaling: llama3
+save_steps: 500
+stage: sft
+template: qwen
+trust_remote_code: true
+use_apollo: true
+use_llama_pro: true
+warmup_steps: 0