Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+license: other
+base_model: Qwen/Qwen2.5-Coder-7B-Instruct
+tags:
+- llama-factory
+- freeze
+- generated_from_trainer
+model-index:
+- name: qwen_ns
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# qwen_ns
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct) on the codes_330k_ns dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 512
+- total_eval_batch_size: 32
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.48.2
+- Pytorch 2.5.1+cu124
+- Datasets 3.2.0
+- Tokenizers 0.21.0

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9963008631319359,
+    "num_input_tokens_seen": 211812352,
+    "total_flos": 8.985866598858359e+18,
+    "train_loss": 0.8237513356869763,
+    "train_runtime": 15586.605,
+    "train_samples_per_second": 3.328,
+    "train_steps_per_second": 0.006
+}

config.json

@@ -0,0 +1,35 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 1.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 32768,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.48.2"
+}

llamaboard_config.yaml

@@ -0,0 +1,77 @@
+top.booster: liger_kernel
+top.checkpoint_path: null
+top.finetuning_type: freeze
+top.model_name: Qwen2.5-Coder-7B-Instruct
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: llama3
+top.template: qwen
+train.additional_target: ''
+train.apollo_rank: 256
+train.apollo_scale: 1
+train.apollo_target: all
+train.apollo_update_interval: 200
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 16
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 4096
+train.dataset:
+- codes_330k_ns
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.extra_args: '{}'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 2
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-5
+train.logging_steps: 1
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '1.0'
+train.max_samples: '50000000'
+train.neat_packing: true
+train.neftune_alpha: 0
+train.num_train_epochs: '1'
+train.packing: true
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to:
+- none
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 500
+train.swanlab_api_key: ''
+train.swanlab_mode: cloud
+train.swanlab_project: llamafactory
+train.swanlab_run_name: ''
+train.swanlab_workspace: ''
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_apollo: true
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: true
+train.use_pissa: false
+train.use_rslora: false
+train.use_swanlab: false
+train.val_size: 0
+train.warmup_steps: 0

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b6f069918b07c064cbba8ae4f00f529aa9bbf84b7cdfcb7fc2694a40f6aa8ef
+size 4877660776

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe239955bacb8d3fa0e387069734637cbbc4e536d61c1b5bae3b37b3e0754782
+size 4932749648

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7fe7bffab0f6953aeabfcc8a57ff1839b96e709e74ab7652b8d1a6dc0bde4cc
+size 4991480296

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ac2c988228af0089658f352bf5d55fc0ed9c54acabe7b51220cf57528fde2eb
+size 1361612328

model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 16163464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

running_log.txt

@@ -0,0 +1,400 @@
+[INFO|2025-04-28 12:29:05] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/config.json
+
+[INFO|2025-04-28 12:29:05] configuration_utils.py:768 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}
+
+
+[INFO|2025-04-28 12:29:05] tokenization_utils_base.py:2034 >> loading file vocab.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/vocab.json
+
+[INFO|2025-04-28 12:29:05] tokenization_utils_base.py:2034 >> loading file merges.txt from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/merges.txt
+
+[INFO|2025-04-28 12:29:05] tokenization_utils_base.py:2034 >> loading file tokenizer.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/tokenizer.json
+
+[INFO|2025-04-28 12:29:05] tokenization_utils_base.py:2034 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-04-28 12:29:05] tokenization_utils_base.py:2034 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-04-28 12:29:05] tokenization_utils_base.py:2034 >> loading file tokenizer_config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/tokenizer_config.json
+
+[INFO|2025-04-28 12:29:05] tokenization_utils_base.py:2034 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-04-28 12:29:05] tokenization_utils_base.py:2304 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-04-28 12:29:05] logging.py:157 >> Add <|im_end|> to stop words.
+
+[INFO|2025-04-28 12:29:05] logging.py:157 >> Loading dataset Codes_query_filtered_330k_ns.json...
+
+[INFO|2025-04-28 12:29:09] configuration_utils.py:696 >> loading configuration file config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/config.json
+
+[INFO|2025-04-28 12:29:09] configuration_utils.py:768 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}
+
+
+[WARNING|2025-04-28 12:29:09] logging.py:162 >> Input length is smaller than max length. Consider increase input length.
+
+[INFO|2025-04-28 12:29:09] logging.py:157 >> Using llama3 scaling strategy and setting scaling factor to 1.0.
+
+[INFO|2025-04-28 12:29:09] logging.py:157 >> Using block diagonal attention for sequence packing without cross-attention.
+
+[INFO|2025-04-28 12:29:09] logging.py:157 >> Liger kernel has been applied to the model.
+
+[INFO|2025-04-28 12:29:09] modeling_utils.py:3904 >> loading weights file model.safetensors from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/model.safetensors.index.json
+
+[INFO|2025-04-28 12:29:09] modeling_utils.py:1582 >> Instantiating Qwen2ForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|2025-04-28 12:29:09] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 151643,
+  "eos_token_id": 151645
+}
+
+
+[INFO|2025-04-28 12:29:14] modeling_utils.py:4888 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.
+
+
+[INFO|2025-04-28 12:29:14] modeling_utils.py:4896 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2.5-Coder-7B-Instruct.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.
+
+[INFO|2025-04-28 12:29:14] configuration_utils.py:1095 >> loading configuration file generation_config.json from cache at /home/kiho/.cache/huggingface/hub/models--Qwen--Qwen2.5-Coder-7B-Instruct/snapshots/c03e6d358207e414f1eca0bb1891e29f1db0e242/generation_config.json
+
+[INFO|2025-04-28 12:29:14] configuration_utils.py:1140 >> Generate config GenerationConfig {
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8
+}
+
+
+[INFO|2025-04-28 12:29:14] logging.py:157 >> Gradient checkpointing enabled.
+
+[INFO|2025-04-28 12:29:14] logging.py:157 >> Using torch SDPA for faster training and inference.
+
+[INFO|2025-04-28 12:29:14] logging.py:157 >> Upcasting trainable params to float32.
+
+[INFO|2025-04-28 12:29:14] logging.py:157 >> Fine-tuning method: Freeze
+
+[INFO|2025-04-28 12:29:14] logging.py:157 >> Set trainable layers: .13.,.27.
+
+[INFO|2025-04-28 12:29:14] logging.py:157 >> trainable params: 466,115,584 || all params: 7,615,616,512 || trainable%: 6.1205
+
+[INFO|2025-04-28 12:29:14] trainer.py:741 >> Using auto half precision backend
+
+[INFO|2025-04-28 12:29:15] logging.py:157 >> Found linear modules: up_proj,gate_proj,q_proj,k_proj,down_proj,v_proj,o_proj
+
+[INFO|2025-04-28 12:29:15] logging.py:157 >> Using APOLLO optimizer with args: {'rank': 256, 'proj': 'random', 'proj_type': 'std', 'update_proj_gap': 200, 'scale': 1, 'scale_type': 'channel', 'scale_front': False}.
+
+[INFO|2025-04-28 12:29:16] trainer.py:2369 >> ***** Running training *****
+
+[INFO|2025-04-28 12:29:16] trainer.py:2370 >>   Num examples = 51,880
+
+[INFO|2025-04-28 12:29:16] trainer.py:2371 >>   Num Epochs = 1
+
+[INFO|2025-04-28 12:29:16] trainer.py:2372 >>   Instantaneous batch size per device = 16
+
+[INFO|2025-04-28 12:29:16] trainer.py:2375 >>   Total train batch size (w. parallel, distributed & accumulation) = 512
+
+[INFO|2025-04-28 12:29:16] trainer.py:2376 >>   Gradient Accumulation steps = 8
+
+[INFO|2025-04-28 12:29:16] trainer.py:2377 >>   Total optimization steps = 101
+
+[INFO|2025-04-28 12:29:16] trainer.py:2378 >>   Number of trainable parameters = 466,115,584
+
+[INFO|2025-04-28 12:32:00] logging.py:157 >> {'loss': 1.0213, 'learning_rate': 4.9988e-05, 'epoch': 0.01, 'throughput': 12907.24}
+
+[INFO|2025-04-28 12:34:34] logging.py:157 >> {'loss': 1.0136, 'learning_rate': 4.9952e-05, 'epoch': 0.02, 'throughput': 13247.81}
+
+[INFO|2025-04-28 12:37:08] logging.py:157 >> {'loss': 0.9404, 'learning_rate': 4.9891e-05, 'epoch': 0.03, 'throughput': 13369.60}
+
+[INFO|2025-04-28 12:39:42] logging.py:157 >> {'loss': 0.9530, 'learning_rate': 4.9807e-05, 'epoch': 0.04, 'throughput': 13426.82}
+
+[INFO|2025-04-28 12:42:16] logging.py:157 >> {'loss': 0.9453, 'learning_rate': 4.9698e-05, 'epoch': 0.05, 'throughput': 13463.63}
+
+[INFO|2025-04-28 12:44:50] logging.py:157 >> {'loss': 0.9111, 'learning_rate': 4.9566e-05, 'epoch': 0.06, 'throughput': 13485.87}
+
+[INFO|2025-04-28 12:47:24] logging.py:157 >> {'loss': 0.8780, 'learning_rate': 4.9410e-05, 'epoch': 0.07, 'throughput': 13503.29}
+
+[INFO|2025-04-28 12:49:59] logging.py:157 >> {'loss': 0.9140, 'learning_rate': 4.9230e-05, 'epoch': 0.08, 'throughput': 13515.40}
+
+[INFO|2025-04-28 12:52:33] logging.py:157 >> {'loss': 0.8649, 'learning_rate': 4.9027e-05, 'epoch': 0.09, 'throughput': 13523.38}
+
+[INFO|2025-04-28 12:55:07] logging.py:157 >> {'loss': 0.9029, 'learning_rate': 4.8800e-05, 'epoch': 0.10, 'throughput': 13530.26}
+
+[INFO|2025-04-28 12:57:42] logging.py:157 >> {'loss': 0.8820, 'learning_rate': 4.8551e-05, 'epoch': 0.11, 'throughput': 13535.32}
+
+[INFO|2025-04-28 13:00:16] logging.py:157 >> {'loss': 0.8438, 'learning_rate': 4.8279e-05, 'epoch': 0.12, 'throughput': 13536.90}
+
+[INFO|2025-04-28 13:02:51] logging.py:157 >> {'loss': 0.8743, 'learning_rate': 4.7984e-05, 'epoch': 0.13, 'throughput': 13535.86}
+
+[INFO|2025-04-28 13:05:26] logging.py:157 >> {'loss': 0.8736, 'learning_rate': 4.7667e-05, 'epoch': 0.14, 'throughput': 13539.44}
+
+[INFO|2025-04-28 13:08:00] logging.py:157 >> {'loss': 0.8687, 'learning_rate': 4.7328e-05, 'epoch': 0.15, 'throughput': 13542.72}
+
+[INFO|2025-04-28 13:10:34] logging.py:157 >> {'loss': 0.8640, 'learning_rate': 4.6967e-05, 'epoch': 0.16, 'throughput': 13545.64}
+
+[INFO|2025-04-28 13:13:09] logging.py:157 >> {'loss': 0.8877, 'learning_rate': 4.6586e-05, 'epoch': 0.17, 'throughput': 13548.59}
+
+[INFO|2025-04-28 13:15:43] logging.py:157 >> {'loss': 0.8749, 'learning_rate': 4.6183e-05, 'epoch': 0.18, 'throughput': 13551.29}
+
+[INFO|2025-04-28 13:18:17] logging.py:157 >> {'loss': 0.8338, 'learning_rate': 4.5760e-05, 'epoch': 0.19, 'throughput': 13551.85}
+
+[INFO|2025-04-28 13:20:52] logging.py:157 >> {'loss': 0.8294, 'learning_rate': 4.5316e-05, 'epoch': 0.20, 'throughput': 13552.35}
+
+[INFO|2025-04-28 13:23:27] logging.py:157 >> {'loss': 0.8666, 'learning_rate': 4.4854e-05, 'epoch': 0.21, 'throughput': 13553.49}
+
+[INFO|2025-04-28 13:26:01] logging.py:157 >> {'loss': 0.8038, 'learning_rate': 4.4371e-05, 'epoch': 0.22, 'throughput': 13554.09}
+
+[INFO|2025-04-28 13:28:36] logging.py:157 >> {'loss': 0.8492, 'learning_rate': 4.3871e-05, 'epoch': 0.23, 'throughput': 13554.87}
+
+[INFO|2025-04-28 13:31:10] logging.py:157 >> {'loss': 0.8047, 'learning_rate': 4.3351e-05, 'epoch': 0.24, 'throughput': 13555.65}
+
+[INFO|2025-04-28 13:33:45] logging.py:157 >> {'loss': 0.8521, 'learning_rate': 4.2815e-05, 'epoch': 0.25, 'throughput': 13556.20}
+
+[INFO|2025-04-28 13:36:19] logging.py:157 >> {'loss': 0.8133, 'learning_rate': 4.2261e-05, 'epoch': 0.26, 'throughput': 13556.46}
+
+[INFO|2025-04-28 13:38:53] logging.py:157 >> {'loss': 0.8365, 'learning_rate': 4.1690e-05, 'epoch': 0.27, 'throughput': 13559.29}
+
+[INFO|2025-04-28 13:41:27] logging.py:157 >> {'loss': 0.8094, 'learning_rate': 4.1103e-05, 'epoch': 0.28, 'throughput': 13563.03}
+
+[INFO|2025-04-28 13:44:00] logging.py:157 >> {'loss': 0.8174, 'learning_rate': 4.0500e-05, 'epoch': 0.29, 'throughput': 13565.42}
+
+[INFO|2025-04-28 13:46:35] logging.py:157 >> {'loss': 0.8278, 'learning_rate': 3.9883e-05, 'epoch': 0.30, 'throughput': 13566.62}
+
+[INFO|2025-04-28 13:49:08] logging.py:157 >> {'loss': 0.8425, 'learning_rate': 3.9251e-05, 'epoch': 0.31, 'throughput': 13569.29}
+
+[INFO|2025-04-28 13:51:42] logging.py:157 >> {'loss': 0.8145, 'learning_rate': 3.8605e-05, 'epoch': 0.32, 'throughput': 13570.89}
+
+[INFO|2025-04-28 13:54:16] logging.py:157 >> {'loss': 0.8322, 'learning_rate': 3.7946e-05, 'epoch': 0.33, 'throughput': 13573.07}
+
+[INFO|2025-04-28 13:56:51] logging.py:157 >> {'loss': 0.8114, 'learning_rate': 3.7275e-05, 'epoch': 0.34, 'throughput': 13572.53}
+
+[INFO|2025-04-28 13:59:25] logging.py:157 >> {'loss': 0.8091, 'learning_rate': 3.6592e-05, 'epoch': 0.35, 'throughput': 13572.13}
+
+[INFO|2025-04-28 14:01:59] logging.py:157 >> {'loss': 0.7743, 'learning_rate': 3.5897e-05, 'epoch': 0.36, 'throughput': 13573.61}
+
+[INFO|2025-04-28 14:04:33] logging.py:157 >> {'loss': 0.8368, 'learning_rate': 3.5192e-05, 'epoch': 0.36, 'throughput': 13575.22}
+
+[INFO|2025-04-28 14:07:07] logging.py:157 >> {'loss': 0.8177, 'learning_rate': 3.4477e-05, 'epoch': 0.37, 'throughput': 13576.95}
+
+[INFO|2025-04-28 14:09:41] logging.py:157 >> {'loss': 0.8109, 'learning_rate': 3.3753e-05, 'epoch': 0.38, 'throughput': 13577.04}
+
+[INFO|2025-04-28 14:12:15] logging.py:157 >> {'loss': 0.8270, 'learning_rate': 3.3021e-05, 'epoch': 0.39, 'throughput': 13578.09}
+
+[INFO|2025-04-28 14:14:49] logging.py:157 >> {'loss': 0.8167, 'learning_rate': 3.2280e-05, 'epoch': 0.40, 'throughput': 13579.75}
+
+[INFO|2025-04-28 14:17:22] logging.py:157 >> {'loss': 0.8073, 'learning_rate': 3.1533e-05, 'epoch': 0.41, 'throughput': 13581.88}
+
+[INFO|2025-04-28 14:19:56] logging.py:157 >> {'loss': 0.7793, 'learning_rate': 3.0779e-05, 'epoch': 0.42, 'throughput': 13583.46}
+
+[INFO|2025-04-28 14:22:30] logging.py:157 >> {'loss': 0.8096, 'learning_rate': 3.0020e-05, 'epoch': 0.43, 'throughput': 13585.16}
+
+[INFO|2025-04-28 14:25:03] logging.py:157 >> {'loss': 0.8212, 'learning_rate': 2.9256e-05, 'epoch': 0.44, 'throughput': 13586.73}
+
+[INFO|2025-04-28 14:27:38] logging.py:157 >> {'loss': 0.8151, 'learning_rate': 2.8488e-05, 'epoch': 0.45, 'throughput': 13586.13}
+
+[INFO|2025-04-28 14:30:11] logging.py:157 >> {'loss': 0.8331, 'learning_rate': 2.7716e-05, 'epoch': 0.46, 'throughput': 13587.29}
+
+[INFO|2025-04-28 14:32:46] logging.py:157 >> {'loss': 0.8003, 'learning_rate': 2.6942e-05, 'epoch': 0.47, 'throughput': 13587.74}
+
+[INFO|2025-04-28 14:35:19] logging.py:157 >> {'loss': 0.8214, 'learning_rate': 2.6166e-05, 'epoch': 0.48, 'throughput': 13588.82}
+
+[INFO|2025-04-28 14:37:53] logging.py:157 >> {'loss': 0.8118, 'learning_rate': 2.5389e-05, 'epoch': 0.49, 'throughput': 13589.60}
+
+[INFO|2025-04-28 14:40:27] logging.py:157 >> {'loss': 0.8382, 'learning_rate': 2.4611e-05, 'epoch': 0.50, 'throughput': 13590.81}
+
+[INFO|2025-04-28 14:43:01] logging.py:157 >> {'loss': 0.8099, 'learning_rate': 2.3834e-05, 'epoch': 0.51, 'throughput': 13590.72}
+
+[INFO|2025-04-28 14:45:35] logging.py:157 >> {'loss': 0.7914, 'learning_rate': 2.3058e-05, 'epoch': 0.52, 'throughput': 13591.47}
+
+[INFO|2025-04-28 14:48:09] logging.py:157 >> {'loss': 0.8104, 'learning_rate': 2.2284e-05, 'epoch': 0.53, 'throughput': 13592.65}
+
+[INFO|2025-04-28 14:50:42] logging.py:157 >> {'loss': 0.8125, 'learning_rate': 2.1512e-05, 'epoch': 0.54, 'throughput': 13593.72}
+
+[INFO|2025-04-28 14:53:16] logging.py:157 >> {'loss': 0.8198, 'learning_rate': 2.0744e-05, 'epoch': 0.55, 'throughput': 13594.46}
+
+[INFO|2025-04-28 14:55:50] logging.py:157 >> {'loss': 0.8019, 'learning_rate': 1.9980e-05, 'epoch': 0.56, 'throughput': 13594.62}
+
+[INFO|2025-04-28 14:58:24] logging.py:157 >> {'loss': 0.8141, 'learning_rate': 1.9221e-05, 'epoch': 0.57, 'throughput': 13595.48}
+
+[INFO|2025-04-28 15:00:58] logging.py:157 >> {'loss': 0.7985, 'learning_rate': 1.8467e-05, 'epoch': 0.58, 'throughput': 13596.37}
+
+[INFO|2025-04-28 15:03:31] logging.py:157 >> {'loss': 0.7998, 'learning_rate': 1.7720e-05, 'epoch': 0.59, 'throughput': 13597.32}
+
+[INFO|2025-04-28 15:06:05] logging.py:157 >> {'loss': 0.7988, 'learning_rate': 1.6979e-05, 'epoch': 0.60, 'throughput': 13597.98}
+
+[INFO|2025-04-28 15:08:39] logging.py:157 >> {'loss': 0.8016, 'learning_rate': 1.6247e-05, 'epoch': 0.61, 'throughput': 13598.66}
+
+[INFO|2025-04-28 15:11:12] logging.py:157 >> {'loss': 0.8162, 'learning_rate': 1.5523e-05, 'epoch': 0.62, 'throughput': 13599.73}
+
+[INFO|2025-04-28 15:13:46] logging.py:157 >> {'loss': 0.8258, 'learning_rate': 1.4808e-05, 'epoch': 0.63, 'throughput': 13600.66}
+
+[INFO|2025-04-28 15:16:19] logging.py:157 >> {'loss': 0.8063, 'learning_rate': 1.4103e-05, 'epoch': 0.64, 'throughput': 13601.35}
+
+[INFO|2025-04-28 15:18:53] logging.py:157 >> {'loss': 0.8116, 'learning_rate': 1.3408e-05, 'epoch': 0.65, 'throughput': 13601.79}
+
+[INFO|2025-04-28 15:21:27] logging.py:157 >> {'loss': 0.7850, 'learning_rate': 1.2725e-05, 'epoch': 0.66, 'throughput': 13602.47}
+
+[INFO|2025-04-28 15:24:01] logging.py:157 >> {'loss': 0.8049, 'learning_rate': 1.2054e-05, 'epoch': 0.67, 'throughput': 13602.60}
+
+[INFO|2025-04-28 15:26:35] logging.py:157 >> {'loss': 0.8034, 'learning_rate': 1.1395e-05, 'epoch': 0.68, 'throughput': 13603.14}
+
+[INFO|2025-04-28 15:29:08] logging.py:157 >> {'loss': 0.7949, 'learning_rate': 1.0749e-05, 'epoch': 0.69, 'throughput': 13603.86}
+
+[INFO|2025-04-28 15:31:42] logging.py:157 >> {'loss': 0.8024, 'learning_rate': 1.0117e-05, 'epoch': 0.70, 'throughput': 13603.82}
+
+[INFO|2025-04-28 15:34:17] logging.py:157 >> {'loss': 0.7608, 'learning_rate': 9.4998e-06, 'epoch': 0.71, 'throughput': 13603.58}
+
+[INFO|2025-04-28 15:36:51] logging.py:157 >> {'loss': 0.8012, 'learning_rate': 8.8972e-06, 'epoch': 0.72, 'throughput': 13603.80}
+
+[INFO|2025-04-28 15:39:25] logging.py:157 >> {'loss': 0.7688, 'learning_rate': 8.3103e-06, 'epoch': 0.73, 'throughput': 13604.36}
+
+[INFO|2025-04-28 15:41:58] logging.py:157 >> {'loss': 0.8023, 'learning_rate': 7.7395e-06, 'epoch': 0.74, 'throughput': 13604.88}
+
+[INFO|2025-04-28 15:44:32] logging.py:157 >> {'loss': 0.7809, 'learning_rate': 7.1854e-06, 'epoch': 0.75, 'throughput': 13605.47}
+
+[INFO|2025-04-28 15:47:05] logging.py:157 >> {'loss': 0.8083, 'learning_rate': 6.6485e-06, 'epoch': 0.76, 'throughput': 13606.21}
+
+[INFO|2025-04-28 15:49:39] logging.py:157 >> {'loss': 0.7903, 'learning_rate': 6.1294e-06, 'epoch': 0.77, 'throughput': 13606.72}
+
+[INFO|2025-04-28 15:52:13] logging.py:157 >> {'loss': 0.7904, 'learning_rate': 5.6286e-06, 'epoch': 0.78, 'throughput': 13607.18}
+
+[INFO|2025-04-28 15:54:46] logging.py:157 >> {'loss': 0.7970, 'learning_rate': 5.1465e-06, 'epoch': 0.79, 'throughput': 13607.74}
+
+[INFO|2025-04-28 15:57:20] logging.py:157 >> {'loss': 0.7636, 'learning_rate': 4.6836e-06, 'epoch': 0.80, 'throughput': 13608.08}
+
+[INFO|2025-04-28 15:59:55] logging.py:157 >> {'loss': 0.7818, 'learning_rate': 4.2403e-06, 'epoch': 0.81, 'throughput': 13607.00}
+
+[INFO|2025-04-28 16:02:29] logging.py:157 >> {'loss': 0.7914, 'learning_rate': 3.8171e-06, 'epoch': 0.82, 'throughput': 13607.35}
+
+[INFO|2025-04-28 16:05:03] logging.py:157 >> {'loss': 0.7985, 'learning_rate': 3.4145e-06, 'epoch': 0.83, 'throughput': 13607.72}
+
+[INFO|2025-04-28 16:07:36] logging.py:157 >> {'loss': 0.7868, 'learning_rate': 3.0327e-06, 'epoch': 0.84, 'throughput': 13608.19}
+
+[INFO|2025-04-28 16:10:10] logging.py:157 >> {'loss': 0.7972, 'learning_rate': 2.6721e-06, 'epoch': 0.85, 'throughput': 13608.54}
+
+[INFO|2025-04-28 16:12:44] logging.py:157 >> {'loss': 0.7933, 'learning_rate': 2.3332e-06, 'epoch': 0.86, 'throughput': 13609.13}
+
+[INFO|2025-04-28 16:15:17] logging.py:157 >> {'loss': 0.7695, 'learning_rate': 2.0162e-06, 'epoch': 0.87, 'throughput': 13609.89}
+
+[INFO|2025-04-28 16:17:51] logging.py:157 >> {'loss': 0.7929, 'learning_rate': 1.7214e-06, 'epoch': 0.88, 'throughput': 13609.74}
+
+[INFO|2025-04-28 16:20:25] logging.py:157 >> {'loss': 0.7995, 'learning_rate': 1.4491e-06, 'epoch': 0.89, 'throughput': 13610.21}
+
+[INFO|2025-04-28 16:22:59] logging.py:157 >> {'loss': 0.7857, 'learning_rate': 1.1997e-06, 'epoch': 0.90, 'throughput': 13610.41}
+
+[INFO|2025-04-28 16:25:33] logging.py:157 >> {'loss': 0.8215, 'learning_rate': 9.7323e-07, 'epoch': 0.91, 'throughput': 13610.73}
+
+[INFO|2025-04-28 16:28:06] logging.py:157 >> {'loss': 0.7931, 'learning_rate': 7.7003e-07, 'epoch': 0.92, 'throughput': 13611.14}
+
+[INFO|2025-04-28 16:30:40] logging.py:157 >> {'loss': 0.7927, 'learning_rate': 5.9026e-07, 'epoch': 0.93, 'throughput': 13611.37}
+
+[INFO|2025-04-28 16:33:14] logging.py:157 >> {'loss': 0.7787, 'learning_rate': 4.3412e-07, 'epoch': 0.94, 'throughput': 13611.55}
+
+[INFO|2025-04-28 16:35:49] logging.py:157 >> {'loss': 0.7771, 'learning_rate': 3.0174e-07, 'epoch': 0.95, 'throughput': 13610.50}
+
+[INFO|2025-04-28 16:38:24] logging.py:157 >> {'loss': 0.7931, 'learning_rate': 1.9325e-07, 'epoch': 0.96, 'throughput': 13610.20}
+
+[INFO|2025-04-28 16:40:58] logging.py:157 >> {'loss': 0.7902, 'learning_rate': 1.0877e-07, 'epoch': 0.97, 'throughput': 13609.88}
+
+[INFO|2025-04-28 16:43:32] logging.py:157 >> {'loss': 0.7906, 'learning_rate': 4.8360e-08, 'epoch': 0.98, 'throughput': 13610.30}
+
+[INFO|2025-04-28 16:46:05] logging.py:157 >> {'loss': 0.8083, 'learning_rate': 1.2093e-08, 'epoch': 0.99, 'throughput': 13610.82}
+
+[INFO|2025-04-28 16:48:39] logging.py:157 >> {'loss': 0.7963, 'learning_rate': 0.0000e+00, 'epoch': 1.00, 'throughput': 13611.38}
+
+[INFO|2025-04-28 16:48:39] trainer.py:3910 >> Saving model checkpoint to saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/checkpoint-101
+
+[INFO|2025-04-28 16:48:39] configuration_utils.py:420 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/checkpoint-101/config.json
+
+[INFO|2025-04-28 16:48:39] configuration_utils.py:909 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/checkpoint-101/generation_config.json
+
+[INFO|2025-04-28 16:49:02] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/checkpoint-101/model.safetensors.index.json.
+
+[INFO|2025-04-28 16:49:02] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/checkpoint-101/tokenizer_config.json
+
+[INFO|2025-04-28 16:49:02] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/checkpoint-101/special_tokens_map.json
+
+[INFO|2025-04-28 16:49:03] trainer.py:2643 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|2025-04-28 16:49:03] trainer.py:3910 >> Saving model checkpoint to saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns
+
+[INFO|2025-04-28 16:49:03] configuration_utils.py:420 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/config.json
+
+[INFO|2025-04-28 16:49:03] configuration_utils.py:909 >> Configuration saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/generation_config.json
+
+[INFO|2025-04-28 16:49:26] modeling_utils.py:2996 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/model.safetensors.index.json.
+
+[INFO|2025-04-28 16:49:26] tokenization_utils_base.py:2491 >> tokenizer config file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/tokenizer_config.json
+
+[INFO|2025-04-28 16:49:26] tokenization_utils_base.py:2500 >> Special tokens file saved in saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns/special_tokens_map.json
+
+[WARNING|2025-04-28 16:49:26] logging.py:162 >> No metric eval_loss to plot.
+
+[WARNING|2025-04-28 16:49:26] logging.py:162 >> No metric eval_accuracy to plot.
+
+[INFO|2025-04-28 16:49:26] modelcard.py:449 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9963008631319359,
+    "num_input_tokens_seen": 211812352,
+    "total_flos": 8.985866598858359e+18,
+    "train_loss": 0.8237513356869763,
+    "train_runtime": 15586.605,
+    "train_samples_per_second": 3.328,
+    "train_steps_per_second": 0.006
+}

trainer_log.jsonl

@@ -0,0 +1,102 @@
+{"current_steps": 1, "total_steps": 101, "loss": 1.0213, "lr": 4.998790705729971e-05, "epoch": 0.009864364981504316, "percentage": 0.99, "elapsed_time": "0:02:42", "remaining_time": "4:30:47", "throughput": 12907.24, "total_tokens": 2097152}
+{"current_steps": 2, "total_steps": 101, "loss": 1.0136, "lr": 4.995163992833986e-05, "epoch": 0.01972872996300863, "percentage": 1.98, "elapsed_time": "0:05:16", "remaining_time": "4:21:11", "throughput": 13247.81, "total_tokens": 4194304}
+{"current_steps": 3, "total_steps": 101, "loss": 0.9404, "lr": 4.989123369922547e-05, "epoch": 0.029593094944512947, "percentage": 2.97, "elapsed_time": "0:07:50", "remaining_time": "4:16:12", "throughput": 13369.6, "total_tokens": 6291456}
+{"current_steps": 4, "total_steps": 101, "loss": 0.953, "lr": 4.980674680908192e-05, "epoch": 0.03945745992601726, "percentage": 3.96, "elapsed_time": "0:10:24", "remaining_time": "4:12:30", "throughput": 13426.82, "total_tokens": 8388608}
+{"current_steps": 5, "total_steps": 101, "loss": 0.9453, "lr": 4.969826099351892e-05, "epoch": 0.04932182490752158, "percentage": 4.95, "elapsed_time": "0:12:58", "remaining_time": "4:09:13", "throughput": 13463.63, "total_tokens": 10485760}
+{"current_steps": 6, "total_steps": 101, "loss": 0.9111, "lr": 4.9565881205556594e-05, "epoch": 0.059186189889025895, "percentage": 5.94, "elapsed_time": "0:15:33", "remaining_time": "4:06:13", "throughput": 13485.87, "total_tokens": 12582912}
+{"current_steps": 7, "total_steps": 101, "loss": 0.878, "lr": 4.940973551409018e-05, "epoch": 0.0690505548705302, "percentage": 6.93, "elapsed_time": "0:18:07", "remaining_time": "4:03:18", "throughput": 13503.29, "total_tokens": 14680064}
+{"current_steps": 8, "total_steps": 101, "loss": 0.914, "lr": 4.922997497999166e-05, "epoch": 0.07891491985203453, "percentage": 7.92, "elapsed_time": "0:20:41", "remaining_time": "4:00:30", "throughput": 13515.4, "total_tokens": 16777216}
+{"current_steps": 9, "total_steps": 101, "loss": 0.8649, "lr": 4.9026773509968115e-05, "epoch": 0.08877928483353884, "percentage": 8.91, "elapsed_time": "0:23:15", "remaining_time": "3:57:46", "throughput": 13523.38, "total_tokens": 18874368}
+{"current_steps": 10, "total_steps": 101, "loss": 0.9029, "lr": 4.8800327688318246e-05, "epoch": 0.09864364981504316, "percentage": 9.9, "elapsed_time": "0:25:49", "remaining_time": "3:55:04", "throughput": 13530.26, "total_tokens": 20971520}
+{"current_steps": 11, "total_steps": 101, "loss": 0.882, "lr": 4.855085658674973e-05, "epoch": 0.10850801479654747, "percentage": 10.89, "elapsed_time": "0:28:24", "remaining_time": "3:52:24", "throughput": 13535.32, "total_tokens": 23068672}
+{"current_steps": 12, "total_steps": 101, "loss": 0.8438, "lr": 4.827860155244149e-05, "epoch": 0.11837237977805179, "percentage": 11.88, "elapsed_time": "0:30:59", "remaining_time": "3:49:47", "throughput": 13536.9, "total_tokens": 25165824}
+{"current_steps": 13, "total_steps": 101, "loss": 0.8743, "lr": 4.798382597455591e-05, "epoch": 0.1282367447595561, "percentage": 12.87, "elapsed_time": "0:33:34", "remaining_time": "3:47:14", "throughput": 13535.86, "total_tokens": 27262976}
+{"current_steps": 14, "total_steps": 101, "loss": 0.8736, "lr": 4.7666815029426816e-05, "epoch": 0.1381011097410604, "percentage": 13.86, "elapsed_time": "0:36:08", "remaining_time": "3:44:35", "throughput": 13539.44, "total_tokens": 29360128}
+{"current_steps": 15, "total_steps": 101, "loss": 0.8687, "lr": 4.732787540466979e-05, "epoch": 0.14796547472256474, "percentage": 14.85, "elapsed_time": "0:38:42", "remaining_time": "3:41:57", "throughput": 13542.72, "total_tokens": 31457280}
+{"current_steps": 16, "total_steps": 101, "loss": 0.864, "lr": 4.696733500248172e-05, "epoch": 0.15782983970406905, "percentage": 15.84, "elapsed_time": "0:41:17", "remaining_time": "3:39:19", "throughput": 13545.64, "total_tokens": 33554432}
+{"current_steps": 17, "total_steps": 101, "loss": 0.8877, "lr": 4.658554262241659e-05, "epoch": 0.16769420468557336, "percentage": 16.83, "elapsed_time": "0:43:51", "remaining_time": "3:36:42", "throughput": 13548.59, "total_tokens": 35651584}
+{"current_steps": 18, "total_steps": 101, "loss": 0.8749, "lr": 4.6182867623944436e-05, "epoch": 0.17755856966707767, "percentage": 17.82, "elapsed_time": "0:46:25", "remaining_time": "3:34:04", "throughput": 13551.29, "total_tokens": 37748736}
+{"current_steps": 19, "total_steps": 101, "loss": 0.8338, "lr": 4.575969956911994e-05, "epoch": 0.187422934648582, "percentage": 18.81, "elapsed_time": "0:49:00", "remaining_time": "3:31:29", "throughput": 13551.85, "total_tokens": 39845888}
+{"current_steps": 20, "total_steps": 101, "loss": 0.8294, "lr": 4.531644784570626e-05, "epoch": 0.19728729963008632, "percentage": 19.8, "elapsed_time": "0:51:34", "remaining_time": "3:28:54", "throughput": 13552.35, "total_tokens": 41943040}
+{"current_steps": 21, "total_steps": 101, "loss": 0.8666, "lr": 4.485354127111884e-05, "epoch": 0.20715166461159062, "percentage": 20.79, "elapsed_time": "0:54:09", "remaining_time": "3:26:18", "throughput": 13553.49, "total_tokens": 44040192}
+{"current_steps": 22, "total_steps": 101, "loss": 0.8038, "lr": 4.437142767757225e-05, "epoch": 0.21701602959309493, "percentage": 21.78, "elapsed_time": "0:56:43", "remaining_time": "3:23:43", "throughput": 13554.09, "total_tokens": 46137344}
+{"current_steps": 23, "total_steps": 101, "loss": 0.8492, "lr": 4.387057347883143e-05, "epoch": 0.22688039457459927, "percentage": 22.77, "elapsed_time": "0:59:18", "remaining_time": "3:21:07", "throughput": 13554.87, "total_tokens": 48234496}
+{"current_steps": 24, "total_steps": 101, "loss": 0.8047, "lr": 4.335146321898651e-05, "epoch": 0.23674475955610358, "percentage": 23.76, "elapsed_time": "1:01:52", "remaining_time": "3:18:32", "throughput": 13555.65, "total_tokens": 50331648}
+{"current_steps": 25, "total_steps": 101, "loss": 0.8521, "lr": 4.281459910368768e-05, "epoch": 0.2466091245376079, "percentage": 24.75, "elapsed_time": "1:04:27", "remaining_time": "3:15:57", "throughput": 13556.2, "total_tokens": 52428800}
+{"current_steps": 26, "total_steps": 101, "loss": 0.8133, "lr": 4.226050051429367e-05, "epoch": 0.2564734895191122, "percentage": 25.74, "elapsed_time": "1:07:02", "remaining_time": "3:13:22", "throughput": 13556.46, "total_tokens": 54525952}
+{"current_steps": 27, "total_steps": 101, "loss": 0.8365, "lr": 4.168970350540384e-05, "epoch": 0.26633785450061653, "percentage": 26.73, "elapsed_time": "1:09:35", "remaining_time": "3:10:45", "throughput": 13559.29, "total_tokens": 56623104}
+{"current_steps": 28, "total_steps": 101, "loss": 0.8094, "lr": 4.110276028625995e-05, "epoch": 0.2762022194821208, "percentage": 27.72, "elapsed_time": "1:12:09", "remaining_time": "3:08:07", "throughput": 13563.03, "total_tokens": 58720256}
+{"current_steps": 29, "total_steps": 101, "loss": 0.8174, "lr": 4.050023868651938e-05, "epoch": 0.28606658446362515, "percentage": 28.71, "elapsed_time": "1:14:43", "remaining_time": "3:05:30", "throughput": 13565.42, "total_tokens": 60817408}
+{"current_steps": 30, "total_steps": 101, "loss": 0.8278, "lr": 3.988272160691665e-05, "epoch": 0.2959309494451295, "percentage": 29.7, "elapsed_time": "1:17:17", "remaining_time": "3:02:55", "throughput": 13566.62, "total_tokens": 62914560}
+{"current_steps": 31, "total_steps": 101, "loss": 0.8425, "lr": 3.925080645534457e-05, "epoch": 0.30579531442663377, "percentage": 30.69, "elapsed_time": "1:19:51", "remaining_time": "3:00:18", "throughput": 13569.29, "total_tokens": 65011712}
+{"current_steps": 32, "total_steps": 101, "loss": 0.8145, "lr": 3.8605104568900685e-05, "epoch": 0.3156596794081381, "percentage": 31.68, "elapsed_time": "1:22:25", "remaining_time": "2:57:42", "throughput": 13570.89, "total_tokens": 67108864}
+{"current_steps": 33, "total_steps": 101, "loss": 0.8322, "lr": 3.7946240622458135e-05, "epoch": 0.32552404438964244, "percentage": 32.67, "elapsed_time": "1:24:58", "remaining_time": "2:55:06", "throughput": 13573.07, "total_tokens": 69206016}
+{"current_steps": 34, "total_steps": 101, "loss": 0.8114, "lr": 3.7274852024333054e-05, "epoch": 0.3353884093711467, "percentage": 33.66, "elapsed_time": "1:27:33", "remaining_time": "2:52:32", "throughput": 13572.53, "total_tokens": 71303168}
+{"current_steps": 35, "total_steps": 101, "loss": 0.8091, "lr": 3.6591588299633186e-05, "epoch": 0.34525277435265106, "percentage": 34.65, "elapsed_time": "1:30:08", "remaining_time": "2:49:58", "throughput": 13572.13, "total_tokens": 73400320}
+{"current_steps": 36, "total_steps": 101, "loss": 0.7743, "lr": 3.589711046188428e-05, "epoch": 0.35511713933415534, "percentage": 35.64, "elapsed_time": "1:32:42", "remaining_time": "2:47:22", "throughput": 13573.61, "total_tokens": 75497472}
+{"current_steps": 37, "total_steps": 101, "loss": 0.8368, "lr": 3.519209037354222e-05, "epoch": 0.3649815043156597, "percentage": 36.63, "elapsed_time": "1:35:15", "remaining_time": "2:44:46", "throughput": 13575.22, "total_tokens": 77594624}
+{"current_steps": 38, "total_steps": 101, "loss": 0.8177, "lr": 3.447721009600949e-05, "epoch": 0.374845869297164, "percentage": 37.62, "elapsed_time": "1:37:49", "remaining_time": "2:42:11", "throughput": 13576.95, "total_tokens": 79691776}
+{"current_steps": 39, "total_steps": 101, "loss": 0.8109, "lr": 3.3753161229784766e-05, "epoch": 0.3847102342786683, "percentage": 38.61, "elapsed_time": "1:40:24", "remaining_time": "2:39:36", "throughput": 13577.04, "total_tokens": 81788928}
+{"current_steps": 40, "total_steps": 101, "loss": 0.827, "lr": 3.302064424538419e-05, "epoch": 0.39457459926017263, "percentage": 39.6, "elapsed_time": "1:42:58", "remaining_time": "2:37:01", "throughput": 13578.09, "total_tokens": 83886080}
+{"current_steps": 41, "total_steps": 101, "loss": 0.8167, "lr": 3.228036780568131e-05, "epoch": 0.40443896424167697, "percentage": 40.59, "elapsed_time": "1:45:31", "remaining_time": "2:34:25", "throughput": 13579.75, "total_tokens": 85983232}
+{"current_steps": 42, "total_steps": 101, "loss": 0.8073, "lr": 3.153304808032152e-05, "epoch": 0.41430332922318125, "percentage": 41.58, "elapsed_time": "1:48:05", "remaining_time": "2:31:50", "throughput": 13581.88, "total_tokens": 88080384}
+{"current_steps": 43, "total_steps": 101, "loss": 0.7793, "lr": 3.077940805287425e-05, "epoch": 0.4241676942046856, "percentage": 42.57, "elapsed_time": "1:50:38", "remaining_time": "2:29:14", "throughput": 13583.46, "total_tokens": 90177536}
+{"current_steps": 44, "total_steps": 101, "loss": 0.8096, "lr": 3.0020176821392964e-05, "epoch": 0.43403205918618987, "percentage": 43.56, "elapsed_time": "1:53:12", "remaining_time": "2:26:39", "throughput": 13585.16, "total_tokens": 92274688}
+{"current_steps": 45, "total_steps": 101, "loss": 0.8212, "lr": 2.925608889305997e-05, "epoch": 0.4438964241676942, "percentage": 44.55, "elapsed_time": "1:55:45", "remaining_time": "2:24:03", "throughput": 13586.73, "total_tokens": 94371840}
+{"current_steps": 46, "total_steps": 101, "loss": 0.8151, "lr": 2.848788347359808e-05, "epoch": 0.45376078914919854, "percentage": 45.54, "elapsed_time": "1:58:20", "remaining_time": "2:21:29", "throughput": 13586.13, "total_tokens": 96468992}
+{"current_steps": 47, "total_steps": 101, "loss": 0.8331, "lr": 2.7716303752136864e-05, "epoch": 0.4636251541307028, "percentage": 46.53, "elapsed_time": "2:00:54", "remaining_time": "2:18:54", "throughput": 13587.29, "total_tokens": 98566144}
+{"current_steps": 48, "total_steps": 101, "loss": 0.8003, "lr": 2.6942096182225162e-05, "epoch": 0.47348951911220716, "percentage": 47.52, "elapsed_time": "2:03:28", "remaining_time": "2:16:20", "throughput": 13587.74, "total_tokens": 100663296}
+{"current_steps": 49, "total_steps": 101, "loss": 0.8214, "lr": 2.616600975968544e-05, "epoch": 0.4833538840937115, "percentage": 48.51, "elapsed_time": "2:06:02", "remaining_time": "2:13:45", "throughput": 13588.82, "total_tokens": 102760448}
+{"current_steps": 50, "total_steps": 101, "loss": 0.8118, "lr": 2.5388795298008776e-05, "epoch": 0.4932182490752158, "percentage": 49.5, "elapsed_time": "2:08:36", "remaining_time": "2:11:10", "throughput": 13589.6, "total_tokens": 104857600}
+{"current_steps": 51, "total_steps": 101, "loss": 0.8382, "lr": 2.4611204701991227e-05, "epoch": 0.5030826140567201, "percentage": 50.5, "elapsed_time": "2:11:09", "remaining_time": "2:08:35", "throughput": 13590.81, "total_tokens": 106954752}
+{"current_steps": 52, "total_steps": 101, "loss": 0.8099, "lr": 2.3833990240314562e-05, "epoch": 0.5129469790382244, "percentage": 51.49, "elapsed_time": "2:13:43", "remaining_time": "2:06:01", "throughput": 13590.72, "total_tokens": 109051904}
+{"current_steps": 53, "total_steps": 101, "loss": 0.7914, "lr": 2.3057903817774843e-05, "epoch": 0.5228113440197287, "percentage": 52.48, "elapsed_time": "2:16:17", "remaining_time": "2:03:26", "throughput": 13591.47, "total_tokens": 111149056}
+{"current_steps": 54, "total_steps": 101, "loss": 0.8104, "lr": 2.2283696247863135e-05, "epoch": 0.5326757090012331, "percentage": 53.47, "elapsed_time": "2:18:51", "remaining_time": "2:00:51", "throughput": 13592.65, "total_tokens": 113246208}
+{"current_steps": 55, "total_steps": 101, "loss": 0.8125, "lr": 2.1512116526401928e-05, "epoch": 0.5425400739827374, "percentage": 54.46, "elapsed_time": "2:21:25", "remaining_time": "1:58:16", "throughput": 13593.72, "total_tokens": 115343360}
+{"current_steps": 56, "total_steps": 101, "loss": 0.8198, "lr": 2.0743911106940034e-05, "epoch": 0.5524044389642416, "percentage": 55.45, "elapsed_time": "2:23:58", "remaining_time": "1:55:41", "throughput": 13594.46, "total_tokens": 117440512}
+{"current_steps": 57, "total_steps": 101, "loss": 0.8019, "lr": 1.9979823178607042e-05, "epoch": 0.562268803945746, "percentage": 56.44, "elapsed_time": "2:26:33", "remaining_time": "1:53:07", "throughput": 13594.62, "total_tokens": 119537664}
+{"current_steps": 58, "total_steps": 101, "loss": 0.8141, "lr": 1.9220591947125766e-05, "epoch": 0.5721331689272503, "percentage": 57.43, "elapsed_time": "2:29:06", "remaining_time": "1:50:32", "throughput": 13595.48, "total_tokens": 121634816}
+{"current_steps": 59, "total_steps": 101, "loss": 0.7985, "lr": 1.846695191967849e-05, "epoch": 0.5819975339087546, "percentage": 58.42, "elapsed_time": "2:31:40", "remaining_time": "1:47:58", "throughput": 13596.37, "total_tokens": 123731968}
+{"current_steps": 60, "total_steps": 101, "loss": 0.7998, "lr": 1.7719632194318702e-05, "epoch": 0.591861898890259, "percentage": 59.41, "elapsed_time": "2:34:13", "remaining_time": "1:45:23", "throughput": 13597.32, "total_tokens": 125829120}
+{"current_steps": 61, "total_steps": 101, "loss": 0.7988, "lr": 1.6979355754615814e-05, "epoch": 0.6017262638717632, "percentage": 60.4, "elapsed_time": "2:36:47", "remaining_time": "1:42:49", "throughput": 13597.98, "total_tokens": 127926272}
+{"current_steps": 62, "total_steps": 101, "loss": 0.8016, "lr": 1.6246838770215233e-05, "epoch": 0.6115906288532675, "percentage": 61.39, "elapsed_time": "2:39:21", "remaining_time": "1:40:14", "throughput": 13598.66, "total_tokens": 130023424}
+{"current_steps": 63, "total_steps": 101, "loss": 0.8162, "lr": 1.552278990399052e-05, "epoch": 0.6214549938347719, "percentage": 62.38, "elapsed_time": "2:41:54", "remaining_time": "1:37:39", "throughput": 13599.73, "total_tokens": 132120576}
+{"current_steps": 64, "total_steps": 101, "loss": 0.8258, "lr": 1.4807909626457782e-05, "epoch": 0.6313193588162762, "percentage": 63.37, "elapsed_time": "2:44:28", "remaining_time": "1:35:05", "throughput": 13600.66, "total_tokens": 134217728}
+{"current_steps": 65, "total_steps": 101, "loss": 0.8063, "lr": 1.4102889538115723e-05, "epoch": 0.6411837237977805, "percentage": 64.36, "elapsed_time": "2:47:02", "remaining_time": "1:32:30", "throughput": 13601.35, "total_tokens": 136314880}
+{"current_steps": 66, "total_steps": 101, "loss": 0.8116, "lr": 1.3408411700366813e-05, "epoch": 0.6510480887792849, "percentage": 65.35, "elapsed_time": "2:49:36", "remaining_time": "1:29:56", "throughput": 13601.79, "total_tokens": 138412032}
+{"current_steps": 67, "total_steps": 101, "loss": 0.785, "lr": 1.2725147975666948e-05, "epoch": 0.6609124537607891, "percentage": 66.34, "elapsed_time": "2:52:09", "remaining_time": "1:27:21", "throughput": 13602.47, "total_tokens": 140509184}
+{"current_steps": 68, "total_steps": 101, "loss": 0.8049, "lr": 1.2053759377541866e-05, "epoch": 0.6707768187422934, "percentage": 67.33, "elapsed_time": "2:54:43", "remaining_time": "1:24:47", "throughput": 13602.6, "total_tokens": 142606336}
+{"current_steps": 69, "total_steps": 101, "loss": 0.8034, "lr": 1.1394895431099314e-05, "epoch": 0.6806411837237978, "percentage": 68.32, "elapsed_time": "2:57:17", "remaining_time": "1:22:13", "throughput": 13603.14, "total_tokens": 144703488}
+{"current_steps": 70, "total_steps": 101, "loss": 0.7949, "lr": 1.0749193544655434e-05, "epoch": 0.6905055487053021, "percentage": 69.31, "elapsed_time": "2:59:51", "remaining_time": "1:19:38", "throughput": 13603.86, "total_tokens": 146800640}
+{"current_steps": 71, "total_steps": 101, "loss": 0.8024, "lr": 1.0117278393083358e-05, "epoch": 0.7003699136868065, "percentage": 70.3, "elapsed_time": "3:02:25", "remaining_time": "1:17:04", "throughput": 13603.82, "total_tokens": 148897792}
+{"current_steps": 72, "total_steps": 101, "loss": 0.7608, "lr": 9.499761313480626e-06, "epoch": 0.7102342786683107, "percentage": 71.29, "elapsed_time": "3:04:59", "remaining_time": "1:14:30", "throughput": 13603.58, "total_tokens": 150994944}
+{"current_steps": 73, "total_steps": 101, "loss": 0.8012, "lr": 8.897239713740058e-06, "epoch": 0.720098643649815, "percentage": 72.28, "elapsed_time": "3:07:33", "remaining_time": "1:11:56", "throughput": 13603.8, "total_tokens": 153092096}
+{"current_steps": 74, "total_steps": 101, "loss": 0.7688, "lr": 8.31029649459616e-06, "epoch": 0.7299630086313194, "percentage": 73.27, "elapsed_time": "3:10:07", "remaining_time": "1:09:22", "throughput": 13604.36, "total_tokens": 155189248}
+{"current_steps": 75, "total_steps": 101, "loss": 0.8023, "lr": 7.739499485706334e-06, "epoch": 0.7398273736128237, "percentage": 74.26, "elapsed_time": "3:12:41", "remaining_time": "1:06:47", "throughput": 13604.88, "total_tokens": 157286400}
+{"current_steps": 76, "total_steps": 101, "loss": 0.7809, "lr": 7.185400896312328e-06, "epoch": 0.749691738594328, "percentage": 75.25, "elapsed_time": "3:15:14", "remaining_time": "1:04:13", "throughput": 13605.47, "total_tokens": 159383552}
+{"current_steps": 77, "total_steps": 101, "loss": 0.8083, "lr": 6.648536781013495e-06, "epoch": 0.7595561035758323, "percentage": 76.24, "elapsed_time": "3:17:48", "remaining_time": "1:01:39", "throughput": 13606.21, "total_tokens": 161480704}
+{"current_steps": 78, "total_steps": 101, "loss": 0.7903, "lr": 6.12942652116858e-06, "epoch": 0.7694204685573366, "percentage": 77.23, "elapsed_time": "3:20:21", "remaining_time": "0:59:04", "throughput": 13606.72, "total_tokens": 163577856}
+{"current_steps": 79, "total_steps": 101, "loss": 0.7904, "lr": 5.628572322427755e-06, "epoch": 0.7792848335388409, "percentage": 78.22, "elapsed_time": "3:22:55", "remaining_time": "0:56:30", "throughput": 13607.18, "total_tokens": 165675008}
+{"current_steps": 80, "total_steps": 101, "loss": 0.797, "lr": 5.1464587288811624e-06, "epoch": 0.7891491985203453, "percentage": 79.21, "elapsed_time": "3:25:29", "remaining_time": "0:53:56", "throughput": 13607.74, "total_tokens": 167772160}
+{"current_steps": 81, "total_steps": 101, "loss": 0.7636, "lr": 4.683552154293747e-06, "epoch": 0.7990135635018496, "percentage": 80.2, "elapsed_time": "3:28:02", "remaining_time": "0:51:22", "throughput": 13608.08, "total_tokens": 169869312}
+{"current_steps": 82, "total_steps": 101, "loss": 0.7818, "lr": 4.240300430880062e-06, "epoch": 0.8088779284833539, "percentage": 81.19, "elapsed_time": "3:30:38", "remaining_time": "0:48:48", "throughput": 13607.0, "total_tokens": 171966464}
+{"current_steps": 83, "total_steps": 101, "loss": 0.7914, "lr": 3.817132376055565e-06, "epoch": 0.8187422934648582, "percentage": 82.18, "elapsed_time": "3:33:11", "remaining_time": "0:46:14", "throughput": 13607.35, "total_tokens": 174063616}
+{"current_steps": 84, "total_steps": 101, "loss": 0.7985, "lr": 3.4144573775834134e-06, "epoch": 0.8286066584463625, "percentage": 83.17, "elapsed_time": "3:35:45", "remaining_time": "0:43:39", "throughput": 13607.72, "total_tokens": 176160768}
+{"current_steps": 85, "total_steps": 101, "loss": 0.7868, "lr": 3.0326649975182865e-06, "epoch": 0.8384710234278668, "percentage": 84.16, "elapsed_time": "3:38:19", "remaining_time": "0:41:05", "throughput": 13608.19, "total_tokens": 178257920}
+{"current_steps": 86, "total_steps": 101, "loss": 0.7972, "lr": 2.672124595330214e-06, "epoch": 0.8483353884093712, "percentage": 85.15, "elapsed_time": "3:40:53", "remaining_time": "0:38:31", "throughput": 13608.54, "total_tokens": 180355072}
+{"current_steps": 87, "total_steps": 101, "loss": 0.7933, "lr": 2.333184970573188e-06, "epoch": 0.8581997533908755, "percentage": 86.14, "elapsed_time": "3:43:26", "remaining_time": "0:35:57", "throughput": 13609.13, "total_tokens": 182452224}
+{"current_steps": 88, "total_steps": 101, "loss": 0.7695, "lr": 2.0161740254440896e-06, "epoch": 0.8680641183723797, "percentage": 87.13, "elapsed_time": "3:45:59", "remaining_time": "0:33:23", "throughput": 13609.89, "total_tokens": 184549376}
+{"current_steps": 89, "total_steps": 101, "loss": 0.7929, "lr": 1.7213984475585144e-06, "epoch": 0.8779284833538841, "percentage": 88.12, "elapsed_time": "3:48:34", "remaining_time": "0:30:49", "throughput": 13609.74, "total_tokens": 186646528}
+{"current_steps": 90, "total_steps": 101, "loss": 0.7995, "lr": 1.4491434132502729e-06, "epoch": 0.8877928483353884, "percentage": 89.11, "elapsed_time": "3:51:07", "remaining_time": "0:28:14", "throughput": 13610.21, "total_tokens": 188743680}
+{"current_steps": 91, "total_steps": 101, "loss": 0.7857, "lr": 1.1996723116817516e-06, "epoch": 0.8976572133168927, "percentage": 90.1, "elapsed_time": "3:53:41", "remaining_time": "0:25:40", "throughput": 13610.41, "total_tokens": 190840832}
+{"current_steps": 92, "total_steps": 101, "loss": 0.8215, "lr": 9.732264900318866e-07, "epoch": 0.9075215782983971, "percentage": 91.09, "elapsed_time": "3:56:15", "remaining_time": "0:23:06", "throughput": 13610.73, "total_tokens": 192937984}
+{"current_steps": 93, "total_steps": 101, "loss": 0.7931, "lr": 7.700250200083469e-07, "epoch": 0.9173859432799013, "percentage": 92.08, "elapsed_time": "3:58:49", "remaining_time": "0:20:32", "throughput": 13611.14, "total_tokens": 195035136}
+{"current_steps": 94, "total_steps": 101, "loss": 0.7927, "lr": 5.90264485909825e-07, "epoch": 0.9272503082614056, "percentage": 93.07, "elapsed_time": "4:01:22", "remaining_time": "0:17:58", "throughput": 13611.37, "total_tokens": 197132288}
+{"current_steps": 95, "total_steps": 101, "loss": 0.7787, "lr": 4.341187944434083e-07, "epoch": 0.93711467324291, "percentage": 94.06, "elapsed_time": "4:03:56", "remaining_time": "0:15:24", "throughput": 13611.55, "total_tokens": 199229440}
+{"current_steps": 96, "total_steps": 101, "loss": 0.7771, "lr": 3.017390064810832e-07, "epoch": 0.9469790382244143, "percentage": 95.05, "elapsed_time": "4:06:32", "remaining_time": "0:12:50", "throughput": 13610.5, "total_tokens": 201326592}
+{"current_steps": 97, "total_steps": 101, "loss": 0.7931, "lr": 1.9325319091808847e-07, "epoch": 0.9568434032059187, "percentage": 96.04, "elapsed_time": "4:09:06", "remaining_time": "0:10:16", "throughput": 13610.2, "total_tokens": 203423744}
+{"current_steps": 98, "total_steps": 101, "loss": 0.7902, "lr": 1.0876630077453487e-07, "epoch": 0.966707768187423, "percentage": 97.03, "elapsed_time": "4:11:40", "remaining_time": "0:07:42", "throughput": 13609.88, "total_tokens": 205520896}
+{"current_steps": 99, "total_steps": 101, "loss": 0.7906, "lr": 4.836007166014178e-08, "epoch": 0.9765721331689272, "percentage": 98.02, "elapsed_time": "4:14:14", "remaining_time": "0:05:08", "throughput": 13610.3, "total_tokens": 207618048}
+{"current_steps": 100, "total_steps": 101, "loss": 0.8083, "lr": 1.2092942700298038e-08, "epoch": 0.9864364981504316, "percentage": 99.01, "elapsed_time": "4:16:47", "remaining_time": "0:02:34", "throughput": 13610.82, "total_tokens": 209715200}
+{"current_steps": 101, "total_steps": 101, "loss": 0.7963, "lr": 0.0, "epoch": 0.9963008631319359, "percentage": 100.0, "elapsed_time": "4:19:21", "remaining_time": "0:00:00", "throughput": 13611.38, "total_tokens": 211812352}
+{"current_steps": 101, "total_steps": 101, "epoch": 0.9963008631319359, "percentage": 100.0, "elapsed_time": "4:19:45", "remaining_time": "0:00:00", "throughput": 13590.34, "total_tokens": 211812352}

trainer_state.json

@@ -0,0 +1,851 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9963008631319359,
+  "eval_steps": 500,
+  "global_step": 101,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009864364981504316,
+      "grad_norm": 0.7294987440109253,
+      "learning_rate": 4.998790705729971e-05,
+      "loss": 1.0213,
+      "num_input_tokens_seen": 2097152,
+      "step": 1
+    },
+    {
+      "epoch": 0.01972872996300863,
+      "grad_norm": 0.5664029717445374,
+      "learning_rate": 4.995163992833986e-05,
+      "loss": 1.0136,
+      "num_input_tokens_seen": 4194304,
+      "step": 2
+    },
+    {
+      "epoch": 0.029593094944512947,
+      "grad_norm": 0.42022261023521423,
+      "learning_rate": 4.989123369922547e-05,
+      "loss": 0.9404,
+      "num_input_tokens_seen": 6291456,
+      "step": 3
+    },
+    {
+      "epoch": 0.03945745992601726,
+      "grad_norm": 0.3317222595214844,
+      "learning_rate": 4.980674680908192e-05,
+      "loss": 0.953,
+      "num_input_tokens_seen": 8388608,
+      "step": 4
+    },
+    {
+      "epoch": 0.04932182490752158,
+      "grad_norm": 0.2700234651565552,
+      "learning_rate": 4.969826099351892e-05,
+      "loss": 0.9453,
+      "num_input_tokens_seen": 10485760,
+      "step": 5
+    },
+    {
+      "epoch": 0.059186189889025895,
+      "grad_norm": 0.23365087807178497,
+      "learning_rate": 4.9565881205556594e-05,
+      "loss": 0.9111,
+      "num_input_tokens_seen": 12582912,
+      "step": 6
+    },
+    {
+      "epoch": 0.0690505548705302,
+      "grad_norm": 0.17710435390472412,
+      "learning_rate": 4.940973551409018e-05,
+      "loss": 0.878,
+      "num_input_tokens_seen": 14680064,
+      "step": 7
+    },
+    {
+      "epoch": 0.07891491985203453,
+      "grad_norm": 0.17090857028961182,
+      "learning_rate": 4.922997497999166e-05,
+      "loss": 0.914,
+      "num_input_tokens_seen": 16777216,
+      "step": 8
+    },
+    {
+      "epoch": 0.08877928483353884,
+      "grad_norm": 0.18688030540943146,
+      "learning_rate": 4.9026773509968115e-05,
+      "loss": 0.8649,
+      "num_input_tokens_seen": 18874368,
+      "step": 9
+    },
+    {
+      "epoch": 0.09864364981504316,
+      "grad_norm": 0.19690372049808502,
+      "learning_rate": 4.8800327688318246e-05,
+      "loss": 0.9029,
+      "num_input_tokens_seen": 20971520,
+      "step": 10
+    },
+    {
+      "epoch": 0.10850801479654747,
+      "grad_norm": 0.16644370555877686,
+      "learning_rate": 4.855085658674973e-05,
+      "loss": 0.882,
+      "num_input_tokens_seen": 23068672,
+      "step": 11
+    },
+    {
+      "epoch": 0.11837237977805179,
+      "grad_norm": 0.1807110756635666,
+      "learning_rate": 4.827860155244149e-05,
+      "loss": 0.8438,
+      "num_input_tokens_seen": 25165824,
+      "step": 12
+    },
+    {
+      "epoch": 0.1282367447595561,
+      "grad_norm": 0.16156087815761566,
+      "learning_rate": 4.798382597455591e-05,
+      "loss": 0.8743,
+      "num_input_tokens_seen": 27262976,
+      "step": 13
+    },
+    {
+      "epoch": 0.1381011097410604,
+      "grad_norm": 0.14602774381637573,
+      "learning_rate": 4.7666815029426816e-05,
+      "loss": 0.8736,
+      "num_input_tokens_seen": 29360128,
+      "step": 14
+    },
+    {
+      "epoch": 0.14796547472256474,
+      "grad_norm": 0.15224798023700714,
+      "learning_rate": 4.732787540466979e-05,
+      "loss": 0.8687,
+      "num_input_tokens_seen": 31457280,
+      "step": 15
+    },
+    {
+      "epoch": 0.15782983970406905,
+      "grad_norm": 0.14535664021968842,
+      "learning_rate": 4.696733500248172e-05,
+      "loss": 0.864,
+      "num_input_tokens_seen": 33554432,
+      "step": 16
+    },
+    {
+      "epoch": 0.16769420468557336,
+      "grad_norm": 0.1372939944267273,
+      "learning_rate": 4.658554262241659e-05,
+      "loss": 0.8877,
+      "num_input_tokens_seen": 35651584,
+      "step": 17
+    },
+    {
+      "epoch": 0.17755856966707767,
+      "grad_norm": 0.1264754980802536,
+      "learning_rate": 4.6182867623944436e-05,
+      "loss": 0.8749,
+      "num_input_tokens_seen": 37748736,
+      "step": 18
+    },
+    {
+      "epoch": 0.187422934648582,
+      "grad_norm": 0.11968766152858734,
+      "learning_rate": 4.575969956911994e-05,
+      "loss": 0.8338,
+      "num_input_tokens_seen": 39845888,
+      "step": 19
+    },
+    {
+      "epoch": 0.19728729963008632,
+      "grad_norm": 0.1143808588385582,
+      "learning_rate": 4.531644784570626e-05,
+      "loss": 0.8294,
+      "num_input_tokens_seen": 41943040,
+      "step": 20
+    },
+    {
+      "epoch": 0.20715166461159062,
+      "grad_norm": 0.12028194963932037,
+      "learning_rate": 4.485354127111884e-05,
+      "loss": 0.8666,
+      "num_input_tokens_seen": 44040192,
+      "step": 21
+    },
+    {
+      "epoch": 0.21701602959309493,
+      "grad_norm": 0.11237838119268417,
+      "learning_rate": 4.437142767757225e-05,
+      "loss": 0.8038,
+      "num_input_tokens_seen": 46137344,
+      "step": 22
+    },
+    {
+      "epoch": 0.22688039457459927,
+      "grad_norm": 0.1187497228384018,
+      "learning_rate": 4.387057347883143e-05,
+      "loss": 0.8492,
+      "num_input_tokens_seen": 48234496,
+      "step": 23
+    },
+    {
+      "epoch": 0.23674475955610358,
+      "grad_norm": 0.11182423681020737,
+      "learning_rate": 4.335146321898651e-05,
+      "loss": 0.8047,
+      "num_input_tokens_seen": 50331648,
+      "step": 24
+    },
+    {
+      "epoch": 0.2466091245376079,
+      "grad_norm": 0.118191197514534,
+      "learning_rate": 4.281459910368768e-05,
+      "loss": 0.8521,
+      "num_input_tokens_seen": 52428800,
+      "step": 25
+    },
+    {
+      "epoch": 0.2564734895191122,
+      "grad_norm": 0.09803211688995361,
+      "learning_rate": 4.226050051429367e-05,
+      "loss": 0.8133,
+      "num_input_tokens_seen": 54525952,
+      "step": 26
+    },
+    {
+      "epoch": 0.26633785450061653,
+      "grad_norm": 0.10156495869159698,
+      "learning_rate": 4.168970350540384e-05,
+      "loss": 0.8365,
+      "num_input_tokens_seen": 56623104,
+      "step": 27
+    },
+    {
+      "epoch": 0.2762022194821208,
+      "grad_norm": 0.10211872309446335,
+      "learning_rate": 4.110276028625995e-05,
+      "loss": 0.8094,
+      "num_input_tokens_seen": 58720256,
+      "step": 28
+    },
+    {
+      "epoch": 0.28606658446362515,
+      "grad_norm": 0.10365650057792664,
+      "learning_rate": 4.050023868651938e-05,
+      "loss": 0.8174,
+      "num_input_tokens_seen": 60817408,
+      "step": 29
+    },
+    {
+      "epoch": 0.2959309494451295,
+      "grad_norm": 0.09515471011400223,
+      "learning_rate": 3.988272160691665e-05,
+      "loss": 0.8278,
+      "num_input_tokens_seen": 62914560,
+      "step": 30
+    },
+    {
+      "epoch": 0.30579531442663377,
+      "grad_norm": 0.10051083564758301,
+      "learning_rate": 3.925080645534457e-05,
+      "loss": 0.8425,
+      "num_input_tokens_seen": 65011712,
+      "step": 31
+    },
+    {
+      "epoch": 0.3156596794081381,
+      "grad_norm": 0.0998254269361496,
+      "learning_rate": 3.8605104568900685e-05,
+      "loss": 0.8145,
+      "num_input_tokens_seen": 67108864,
+      "step": 32
+    },
+    {
+      "epoch": 0.32552404438964244,
+      "grad_norm": 0.109571672976017,
+      "learning_rate": 3.7946240622458135e-05,
+      "loss": 0.8322,
+      "num_input_tokens_seen": 69206016,
+      "step": 33
+    },
+    {
+      "epoch": 0.3353884093711467,
+      "grad_norm": 0.09529263526201248,
+      "learning_rate": 3.7274852024333054e-05,
+      "loss": 0.8114,
+      "num_input_tokens_seen": 71303168,
+      "step": 34
+    },
+    {
+      "epoch": 0.34525277435265106,
+      "grad_norm": 0.09562862664461136,
+      "learning_rate": 3.6591588299633186e-05,
+      "loss": 0.8091,
+      "num_input_tokens_seen": 73400320,
+      "step": 35
+    },
+    {
+      "epoch": 0.35511713933415534,
+      "grad_norm": 0.09187028557062149,
+      "learning_rate": 3.589711046188428e-05,
+      "loss": 0.7743,
+      "num_input_tokens_seen": 75497472,
+      "step": 36
+    },
+    {
+      "epoch": 0.3649815043156597,
+      "grad_norm": 0.09879715740680695,
+      "learning_rate": 3.519209037354222e-05,
+      "loss": 0.8368,
+      "num_input_tokens_seen": 77594624,
+      "step": 37
+    },
+    {
+      "epoch": 0.374845869297164,
+      "grad_norm": 0.0839247927069664,
+      "learning_rate": 3.447721009600949e-05,
+      "loss": 0.8177,
+      "num_input_tokens_seen": 79691776,
+      "step": 38
+    },
+    {
+      "epoch": 0.3847102342786683,
+      "grad_norm": 0.09094128012657166,
+      "learning_rate": 3.3753161229784766e-05,
+      "loss": 0.8109,
+      "num_input_tokens_seen": 81788928,
+      "step": 39
+    },
+    {
+      "epoch": 0.39457459926017263,
+      "grad_norm": 0.09645260125398636,
+      "learning_rate": 3.302064424538419e-05,
+      "loss": 0.827,
+      "num_input_tokens_seen": 83886080,
+      "step": 40
+    },
+    {
+      "epoch": 0.40443896424167697,
+      "grad_norm": 0.08931674808263779,
+      "learning_rate": 3.228036780568131e-05,
+      "loss": 0.8167,
+      "num_input_tokens_seen": 85983232,
+      "step": 41
+    },
+    {
+      "epoch": 0.41430332922318125,
+      "grad_norm": 0.08245021849870682,
+      "learning_rate": 3.153304808032152e-05,
+      "loss": 0.8073,
+      "num_input_tokens_seen": 88080384,
+      "step": 42
+    },
+    {
+      "epoch": 0.4241676942046856,
+      "grad_norm": 0.08803369104862213,
+      "learning_rate": 3.077940805287425e-05,
+      "loss": 0.7793,
+      "num_input_tokens_seen": 90177536,
+      "step": 43
+    },
+    {
+      "epoch": 0.43403205918618987,
+      "grad_norm": 0.08340983092784882,
+      "learning_rate": 3.0020176821392964e-05,
+      "loss": 0.8096,
+      "num_input_tokens_seen": 92274688,
+      "step": 44
+    },
+    {
+      "epoch": 0.4438964241676942,
+      "grad_norm": 0.09057221561670303,
+      "learning_rate": 2.925608889305997e-05,
+      "loss": 0.8212,
+      "num_input_tokens_seen": 94371840,
+      "step": 45
+    },
+    {
+      "epoch": 0.45376078914919854,
+      "grad_norm": 0.08546467870473862,
+      "learning_rate": 2.848788347359808e-05,
+      "loss": 0.8151,
+      "num_input_tokens_seen": 96468992,
+      "step": 46
+    },
+    {
+      "epoch": 0.4636251541307028,
+      "grad_norm": 0.08286646008491516,
+      "learning_rate": 2.7716303752136864e-05,
+      "loss": 0.8331,
+      "num_input_tokens_seen": 98566144,
+      "step": 47
+    },
+    {
+      "epoch": 0.47348951911220716,
+      "grad_norm": 0.08105745166540146,
+      "learning_rate": 2.6942096182225162e-05,
+      "loss": 0.8003,
+      "num_input_tokens_seen": 100663296,
+      "step": 48
+    },
+    {
+      "epoch": 0.4833538840937115,
+      "grad_norm": 0.07993409782648087,
+      "learning_rate": 2.616600975968544e-05,
+      "loss": 0.8214,
+      "num_input_tokens_seen": 102760448,
+      "step": 49
+    },
+    {
+      "epoch": 0.4932182490752158,
+      "grad_norm": 0.08382619917392731,
+      "learning_rate": 2.5388795298008776e-05,
+      "loss": 0.8118,
+      "num_input_tokens_seen": 104857600,
+      "step": 50
+    },
+    {
+      "epoch": 0.5030826140567201,
+      "grad_norm": 0.08271751552820206,
+      "learning_rate": 2.4611204701991227e-05,
+      "loss": 0.8382,
+      "num_input_tokens_seen": 106954752,
+      "step": 51
+    },
+    {
+      "epoch": 0.5129469790382244,
+      "grad_norm": 0.07375568896532059,
+      "learning_rate": 2.3833990240314562e-05,
+      "loss": 0.8099,
+      "num_input_tokens_seen": 109051904,
+      "step": 52
+    },
+    {
+      "epoch": 0.5228113440197287,
+      "grad_norm": 0.07492537051439285,
+      "learning_rate": 2.3057903817774843e-05,
+      "loss": 0.7914,
+      "num_input_tokens_seen": 111149056,
+      "step": 53
+    },
+    {
+      "epoch": 0.5326757090012331,
+      "grad_norm": 0.07812082767486572,
+      "learning_rate": 2.2283696247863135e-05,
+      "loss": 0.8104,
+      "num_input_tokens_seen": 113246208,
+      "step": 54
+    },
+    {
+      "epoch": 0.5425400739827374,
+      "grad_norm": 0.07944825291633606,
+      "learning_rate": 2.1512116526401928e-05,
+      "loss": 0.8125,
+      "num_input_tokens_seen": 115343360,
+      "step": 55
+    },
+    {
+      "epoch": 0.5524044389642416,
+      "grad_norm": 0.07352437824010849,
+      "learning_rate": 2.0743911106940034e-05,
+      "loss": 0.8198,
+      "num_input_tokens_seen": 117440512,
+      "step": 56
+    },
+    {
+      "epoch": 0.562268803945746,
+      "grad_norm": 0.07700145244598389,
+      "learning_rate": 1.9979823178607042e-05,
+      "loss": 0.8019,
+      "num_input_tokens_seen": 119537664,
+      "step": 57
+    },
+    {
+      "epoch": 0.5721331689272503,
+      "grad_norm": 0.07811249792575836,
+      "learning_rate": 1.9220591947125766e-05,
+      "loss": 0.8141,
+      "num_input_tokens_seen": 121634816,
+      "step": 58
+    },
+    {
+      "epoch": 0.5819975339087546,
+      "grad_norm": 0.07361660897731781,
+      "learning_rate": 1.846695191967849e-05,
+      "loss": 0.7985,
+      "num_input_tokens_seen": 123731968,
+      "step": 59
+    },
+    {
+      "epoch": 0.591861898890259,
+      "grad_norm": 0.07933210581541061,
+      "learning_rate": 1.7719632194318702e-05,
+      "loss": 0.7998,
+      "num_input_tokens_seen": 125829120,
+      "step": 60
+    },
+    {
+      "epoch": 0.6017262638717632,
+      "grad_norm": 0.0801980197429657,
+      "learning_rate": 1.6979355754615814e-05,
+      "loss": 0.7988,
+      "num_input_tokens_seen": 127926272,
+      "step": 61
+    },
+    {
+      "epoch": 0.6115906288532675,
+      "grad_norm": 0.07316846400499344,
+      "learning_rate": 1.6246838770215233e-05,
+      "loss": 0.8016,
+      "num_input_tokens_seen": 130023424,
+      "step": 62
+    },
+    {
+      "epoch": 0.6214549938347719,
+      "grad_norm": 0.0750172808766365,
+      "learning_rate": 1.552278990399052e-05,
+      "loss": 0.8162,
+      "num_input_tokens_seen": 132120576,
+      "step": 63
+    },
+    {
+      "epoch": 0.6313193588162762,
+      "grad_norm": 0.07336606830358505,
+      "learning_rate": 1.4807909626457782e-05,
+      "loss": 0.8258,
+      "num_input_tokens_seen": 134217728,
+      "step": 64
+    },
+    {
+      "epoch": 0.6411837237977805,
+      "grad_norm": 0.07929656654596329,
+      "learning_rate": 1.4102889538115723e-05,
+      "loss": 0.8063,
+      "num_input_tokens_seen": 136314880,
+      "step": 65
+    },
+    {
+      "epoch": 0.6510480887792849,
+      "grad_norm": 0.08207987248897552,
+      "learning_rate": 1.3408411700366813e-05,
+      "loss": 0.8116,
+      "num_input_tokens_seen": 138412032,
+      "step": 66
+    },
+    {
+      "epoch": 0.6609124537607891,
+      "grad_norm": 0.07338231056928635,
+      "learning_rate": 1.2725147975666948e-05,
+      "loss": 0.785,
+      "num_input_tokens_seen": 140509184,
+      "step": 67
+    },
+    {
+      "epoch": 0.6707768187422934,
+      "grad_norm": 0.08025672286748886,
+      "learning_rate": 1.2053759377541866e-05,
+      "loss": 0.8049,
+      "num_input_tokens_seen": 142606336,
+      "step": 68
+    },
+    {
+      "epoch": 0.6806411837237978,
+      "grad_norm": 0.07760636508464813,
+      "learning_rate": 1.1394895431099314e-05,
+      "loss": 0.8034,
+      "num_input_tokens_seen": 144703488,
+      "step": 69
+    },
+    {
+      "epoch": 0.6905055487053021,
+      "grad_norm": 0.07995034009218216,
+      "learning_rate": 1.0749193544655434e-05,
+      "loss": 0.7949,
+      "num_input_tokens_seen": 146800640,
+      "step": 70
+    },
+    {
+      "epoch": 0.7003699136868065,
+      "grad_norm": 0.07639381289482117,
+      "learning_rate": 1.0117278393083358e-05,
+      "loss": 0.8024,
+      "num_input_tokens_seen": 148897792,
+      "step": 71
+    },
+    {
+      "epoch": 0.7102342786683107,
+      "grad_norm": 0.0737844705581665,
+      "learning_rate": 9.499761313480626e-06,
+      "loss": 0.7608,
+      "num_input_tokens_seen": 150994944,
+      "step": 72
+    },
+    {
+      "epoch": 0.720098643649815,
+      "grad_norm": 0.07084383070468903,
+      "learning_rate": 8.897239713740058e-06,
+      "loss": 0.8012,
+      "num_input_tokens_seen": 153092096,
+      "step": 73
+    },
+    {
+      "epoch": 0.7299630086313194,
+      "grad_norm": 0.07751645892858505,
+      "learning_rate": 8.31029649459616e-06,
+      "loss": 0.7688,
+      "num_input_tokens_seen": 155189248,
+      "step": 74
+    },
+    {
+      "epoch": 0.7398273736128237,
+      "grad_norm": 0.07183132320642471,
+      "learning_rate": 7.739499485706334e-06,
+      "loss": 0.8023,
+      "num_input_tokens_seen": 157286400,
+      "step": 75
+    },
+    {
+      "epoch": 0.749691738594328,
+      "grad_norm": 0.07271506637334824,
+      "learning_rate": 7.185400896312328e-06,
+      "loss": 0.7809,
+      "num_input_tokens_seen": 159383552,
+      "step": 76
+    },
+    {
+      "epoch": 0.7595561035758323,
+      "grad_norm": 0.07422608137130737,
+      "learning_rate": 6.648536781013495e-06,
+      "loss": 0.8083,
+      "num_input_tokens_seen": 161480704,
+      "step": 77
+    },
+    {
+      "epoch": 0.7694204685573366,
+      "grad_norm": 0.07867728918790817,
+      "learning_rate": 6.12942652116858e-06,
+      "loss": 0.7903,
+      "num_input_tokens_seen": 163577856,
+      "step": 78
+    },
+    {
+      "epoch": 0.7792848335388409,
+      "grad_norm": 0.07830856740474701,
+      "learning_rate": 5.628572322427755e-06,
+      "loss": 0.7904,
+      "num_input_tokens_seen": 165675008,
+      "step": 79
+    },
+    {
+      "epoch": 0.7891491985203453,
+      "grad_norm": 0.07836252450942993,
+      "learning_rate": 5.1464587288811624e-06,
+      "loss": 0.797,
+      "num_input_tokens_seen": 167772160,
+      "step": 80
+    },
+    {
+      "epoch": 0.7990135635018496,
+      "grad_norm": 0.07151419669389725,
+      "learning_rate": 4.683552154293747e-06,
+      "loss": 0.7636,
+      "num_input_tokens_seen": 169869312,
+      "step": 81
+    },
+    {
+      "epoch": 0.8088779284833539,
+      "grad_norm": 0.07672585546970367,
+      "learning_rate": 4.240300430880062e-06,
+      "loss": 0.7818,
+      "num_input_tokens_seen": 171966464,
+      "step": 82
+    },
+    {
+      "epoch": 0.8187422934648582,
+      "grad_norm": 0.08120947331190109,
+      "learning_rate": 3.817132376055565e-06,
+      "loss": 0.7914,
+      "num_input_tokens_seen": 174063616,
+      "step": 83
+    },
+    {
+      "epoch": 0.8286066584463625,
+      "grad_norm": 0.07581306993961334,
+      "learning_rate": 3.4144573775834134e-06,
+      "loss": 0.7985,
+      "num_input_tokens_seen": 176160768,
+      "step": 84
+    },
+    {
+      "epoch": 0.8384710234278668,
+      "grad_norm": 0.08039968460798264,
+      "learning_rate": 3.0326649975182865e-06,
+      "loss": 0.7868,
+      "num_input_tokens_seen": 178257920,
+      "step": 85
+    },
+    {
+      "epoch": 0.8483353884093712,
+      "grad_norm": 0.07282786071300507,
+      "learning_rate": 2.672124595330214e-06,
+      "loss": 0.7972,
+      "num_input_tokens_seen": 180355072,
+      "step": 86
+    },
+    {
+      "epoch": 0.8581997533908755,
+      "grad_norm": 0.07389923185110092,
+      "learning_rate": 2.333184970573188e-06,
+      "loss": 0.7933,
+      "num_input_tokens_seen": 182452224,
+      "step": 87
+    },
+    {
+      "epoch": 0.8680641183723797,
+      "grad_norm": 0.07451856881380081,
+      "learning_rate": 2.0161740254440896e-06,
+      "loss": 0.7695,
+      "num_input_tokens_seen": 184549376,
+      "step": 88
+    },
+    {
+      "epoch": 0.8779284833538841,
+      "grad_norm": 0.0740065649151802,
+      "learning_rate": 1.7213984475585144e-06,
+      "loss": 0.7929,
+      "num_input_tokens_seen": 186646528,
+      "step": 89
+    },
+    {
+      "epoch": 0.8877928483353884,
+      "grad_norm": 0.07559362053871155,
+      "learning_rate": 1.4491434132502729e-06,
+      "loss": 0.7995,
+      "num_input_tokens_seen": 188743680,
+      "step": 90
+    },
+    {
+      "epoch": 0.8976572133168927,
+      "grad_norm": 0.07598087936639786,
+      "learning_rate": 1.1996723116817516e-06,
+      "loss": 0.7857,
+      "num_input_tokens_seen": 190840832,
+      "step": 91
+    },
+    {
+      "epoch": 0.9075215782983971,
+      "grad_norm": 0.0752101019024849,
+      "learning_rate": 9.732264900318866e-07,
+      "loss": 0.8215,
+      "num_input_tokens_seen": 192937984,
+      "step": 92
+    },
+    {
+      "epoch": 0.9173859432799013,
+      "grad_norm": 0.07947240769863129,
+      "learning_rate": 7.700250200083469e-07,
+      "loss": 0.7931,
+      "num_input_tokens_seen": 195035136,
+      "step": 93
+    },
+    {
+      "epoch": 0.9272503082614056,
+      "grad_norm": 0.07513375580310822,
+      "learning_rate": 5.90264485909825e-07,
+      "loss": 0.7927,
+      "num_input_tokens_seen": 197132288,
+      "step": 94
+    },
+    {
+      "epoch": 0.93711467324291,
+      "grad_norm": 0.0716259703040123,
+      "learning_rate": 4.341187944434083e-07,
+      "loss": 0.7787,
+      "num_input_tokens_seen": 199229440,
+      "step": 95
+    },
+    {
+      "epoch": 0.9469790382244143,
+      "grad_norm": 0.07607847452163696,
+      "learning_rate": 3.017390064810832e-07,
+      "loss": 0.7771,
+      "num_input_tokens_seen": 201326592,
+      "step": 96
+    },
+    {
+      "epoch": 0.9568434032059187,
+      "grad_norm": 0.07854241132736206,
+      "learning_rate": 1.9325319091808847e-07,
+      "loss": 0.7931,
+      "num_input_tokens_seen": 203423744,
+      "step": 97
+    },
+    {
+      "epoch": 0.966707768187423,
+      "grad_norm": 0.07781317085027695,
+      "learning_rate": 1.0876630077453487e-07,
+      "loss": 0.7902,
+      "num_input_tokens_seen": 205520896,
+      "step": 98
+    },
+    {
+      "epoch": 0.9765721331689272,
+      "grad_norm": 0.07628636062145233,
+      "learning_rate": 4.836007166014178e-08,
+      "loss": 0.7906,
+      "num_input_tokens_seen": 207618048,
+      "step": 99
+    },
+    {
+      "epoch": 0.9864364981504316,
+      "grad_norm": 0.07593634724617004,
+      "learning_rate": 1.2092942700298038e-08,
+      "loss": 0.8083,
+      "num_input_tokens_seen": 209715200,
+      "step": 100
+    },
+    {
+      "epoch": 0.9963008631319359,
+      "grad_norm": 0.074070043861866,
+      "learning_rate": 0.0,
+      "loss": 0.7963,
+      "num_input_tokens_seen": 211812352,
+      "step": 101
+    },
+    {
+      "epoch": 0.9963008631319359,
+      "num_input_tokens_seen": 211812352,
+      "step": 101,
+      "total_flos": 8.985866598858359e+18,
+      "train_loss": 0.8237513356869763,
+      "train_runtime": 15586.605,
+      "train_samples_per_second": 3.328,
+      "train_steps_per_second": 0.006
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 101,
+  "num_input_tokens_seen": 211812352,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.985866598858359e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66830b228abfb3acf930153489ae5bf2891dfe330303465eb4ee5f20a2d34b89
+size 5624

training_args.yaml

@@ -0,0 +1,39 @@
+apollo_rank: 256
+apollo_scale: 1
+apollo_target: all
+apollo_update_interval: 200
+bf16: true
+cutoff_len: 4096
+dataset: codes_330k_ns
+dataset_dir: data
+ddp_timeout: 180000000
+do_train: true
+enable_liger_kernel: true
+finetuning_type: freeze
+flash_attn: auto
+freeze_trainable_layers: 2
+freeze_trainable_modules: all
+gradient_accumulation_steps: 8
+include_num_input_tokens_seen: true
+learning_rate: 5.0e-05
+logging_steps: 1
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_samples: 50000000
+model_name_or_path: Qwen/Qwen2.5-Coder-7B-Instruct
+neat_packing: true
+num_train_epochs: 1.0
+output_dir: saves/Qwen2.5-Coder-7B-Instruct/freeze/qwen_ns
+packing: true
+per_device_train_batch_size: 16
+plot_loss: true
+preprocessing_num_workers: 16
+report_to: none
+rope_scaling: llama3
+save_steps: 500
+stage: sft
+template: qwen
+trust_remote_code: true
+use_apollo: true
+use_llama_pro: true
+warmup_steps: 0