hf_lora_eval.yml

max_seq_len: 2048
seed: 1
precision: amp_fp16

model_name_or_path: facebook/opt-350m
# If you are using a seperated lora weight, put it here:
# lora weights must be compatible with the specified model
lora_id_or_path: ybelkada/opt-350m-lora  # Example lora weights for opt-350m

models:
-
  model_name: ${model_name_or_path}
  model:
    name: hf_causal_lm
    pretrained_model_name_or_path: ${model_name_or_path}
    init_device: mixed
    pretrained: true
    pretrained_lora_id_or_path: ${lora_id_or_path}
  tokenizer:
    name: ${model_name_or_path}
    kwargs:
      model_max_length: ${max_seq_len}

device_eval_batch_size: 4

# FSDP config for model sharding
fsdp_config:
  sharding_strategy: FULL_SHARD
  mixed_precision: FULL

icl_tasks: "eval/yamls/tasks_v0.2.yaml"
eval_gauntlet: "eval/yamls/eval_gauntlet_v0.2.yaml"