| name: comma_v0p1_yolooooo | |
| dump_dir: /fsx/craffel/lingua_logs/comma_v0p1/ | |
| seed: 777 | |
| grad_acc_steps: 4 | |
| gc_collect_freq: 1000 | |
| probe_freq: null | |
| steps: 500000 | |
| data: | |
| root_dir: /scratch/craffel/lingua/data/ | |
| sources: | |
| peS2o: 0.274065475510351 | |
| stackexchange: 0.134617935796937 | |
| stackv2_edu: 0.127770669195666 | |
| cccc: 0.0871992270000557 | |
| wikimedia: 0.0861800315862719 | |
| github_archive: 0.0606452345122248 | |
| uspto: 0.0413469377516883 | |
| pubmed: 0.0367902799837971 | |
| arxiv_papers: 0.0292395449667613 | |
| caselaw_access_project: 0.0193875362722656 | |
| wikiteam: 0.0137485410839637 | |
| doab: 0.0180439781895451 | |
| uk_hansard: 0.0144498535570883 | |
| pre_1929_books: 0.0115755547988338 | |
| ubuntu_irc: 0.00794254267719456 | |
| regulations: 0.00762583706405442 | |
| data_provenance_initiative: 0.00512264496834867 | |
| project_gutenberg: 0.00502100654070129 | |
| youtube: 0.00465917165839394 | |
| arxiv_abstracts: 0.00359635066160403 | |
| stackv2_html: 0.00225924255952781 | |
| usgpo: 0.00226024581728848 | |
| library_of_congress: 0.00222469340783564 | |
| biodiversity_heritage_library: 0.00221737524370278 | |
| pressbooks: 0.000865101033213598 | |
| libretexts: 0.00054149556727006 | |
| news: 0.000372716196818104 | |
| foodista: 0.000125363443065615 | |
| oercommons: 7.78696843693821e-05 | |
| python_enhancement_proposals: 1.69983991984805e-05 | |
| public_domain_review: 1.05448719635173e-05 | |
| batch_size: 2 | |
| seq_len: 4096 | |
| n_views: 2 | |
| seed: 42 | |
| add_bos: true | |
| add_eos: true | |
| load_async: true | |
| prefetch_size: 4096 | |
| tokenizer: | |
| name: tiktoken | |
| path: /fsx/craffel/lingua/tokenizers/common-pile-tokenizer.tiktoken | |
| optim: | |
| lr: 0.001 | |
| weight_decay: 0.2 | |
| epsilon: 1.0e-08 | |
| beta1: 0.9 | |
| beta2: 0.95 | |
| clip: 1.0 | |
| scheduler: cosine | |
| warmup: 2000 | |
| lr_min_ratio: 1.0e-06 | |
| cycle_length: 1.0 | |
| cosine_theta: 1.0 | |
| annealing_step: 1000 | |
| decay_fraction: 0.1 | |
| exp_factor: 0.5 | |
| model: | |
| dim: 4096 | |
| n_layers: 32 | |
| head_dim: null | |
| n_heads: 32 | |
| n_kv_heads: null | |
| ffn_dim_multiplier: 1.0 | |
| multiple_of: 256 | |
| norm_eps: 1.0e-05 | |
| rope_theta: 100000.0 | |
| init_base_std: null | |
| init_std_factor: disabled | |
| max_seqlen: 4096 | |
| seed: 42 | |
| vocab_size: 64256 | |
| weight_tying: false | |
| sliding_window: null | |
| distributed: | |
| dp_shard: 1 | |
| dp_replicate: 64 | |
| tp_size: 1 | |
| selective_activation_checkpointing: false | |
| compile: true | |
| fsdp_type: full_shard | |
| model_dtype: bf16 | |
| float8_recipe: null | |
| float8_filter: layers\.[0-9]+\. | |
| matmul_allow_tf32: false | |
| detect_anomaly: false | |
| compile_cache_size_limit: 8 | |
| spawn_method: forkserver | |
| env: | |
| MKL_SERVICE_FORCE_INTEL: GNU | |
| OMP_NUM_THREADS: '1' | |
| MKL_NUM_THREADS: '1' | |
| ENABLE_INTRA_NODE_COMM: '1' | |
| TORCH_NCCL_AVOID_RECORD_STREAMS: '1' | |
| NCCL_IB_TIMEOUT: '22' | |
| NCCL_DEBUG: INFO | |
| TORCH_NCCL_ASYNC_ERROR_HANDLING: '1' | |
| checkpoint: | |
| dump: | |
| every: 10000 | |
| keep: -1 | |
| eval: | |
| every: 2000 | |
| keep: 3 | |
| path: /fsx/craffel/lingua_logs/comma_v0p1/checkpoints | |
| init_ckpt_path: null | |
| continue_training_from_init: false | |
| profiling: | |
| run: true | |
| trace_folder: profiling | |
| mem_warmup: 0 | |
| mem_steps: 4 | |
| profile_warmup: 100 | |
| profile_steps: 4 | |
| logging: | |
| freq: 1 | |
| acc_freq: null | |
| wandb: null | |
| async_eval_gpus: 8 | |
| eval: | |
| harness: | |
| tasks: | |
| - hellaswag | |
| - task: boolq | |
| dataset_kwargs: | |
| trust_remote_code: true | |
| - piqa | |
| - task: social_iqa | |
| dataset_kwargs: | |
| trust_remote_code: true | |
| - winogrande | |
| - openbookqa | |
| - arc_easy | |
| - arc_challenge | |
| - race | |
| - commonsense_qa | |
| - task: copa | |
| dataset_kwargs: | |
| trust_remote_code: true | |
| - mmlu | |
| - mmlu_pro | |
| generator: | |
| max_tokens: 8192 | |
| dtype: bf16 | |