ct_java_pretrain_lm.yaml

experiment_setup:
  executable: 'code_transformer/experiments/code_transformer/language_modeling.py'

data_setup:
  language: 'java-pretrain'
  num_predict: 2
  use_validation: True
  use_no_punctuation: True
  use_pointer_network: True
  num_sub_tokens: 5

data_transforms:
  max_distance_mask: None
  relative_distances:
    - ppr
    - ancestor_sp
    - sibling_sp
    - shortest_paths
  distance_binning:
    type: 'exponential'
    growth_factor: 1.3
    n_fixed_bins: 9

transfer_learning:
  use_pretrained_model: False
  model_type: 'ct_lm'
  run_id: 27
  snapshot_iteration: 'latest'

model:
  with_cuda: True
  label_smoothing: 0.1
  lm_encoder:
    input_nonlinearity: 'tanh'
    num_languages: None
    transformer:
      num_layers: 3
      encoder_layer:
        d_model: 1024
        nhead: 8
        dim_feedforward: 2048
        dropout: 0
        activation: 'gelu'
        use_content_content: True
        use_content_pos: True
        use_pos_content: True
        use_pos_pos: True
        use_token_distances: True
  lm_decoder:
    output_nonlinearity: None
    n_layers: 1
    decoder_dropout: 0
    decoder_nhead: 8
    decoder_dim_feedforward: 2048
    decoder_activation: 'gelu'
    use_teacher_forcing: True
    pointer_attention_type: 'additive'
    use_pointer_query_linear: False
    use_pointer_query_self_attention: False
    attend_cls_token: False

optimizer:
  learning_rate: 8e-5
  reg_scale: 0

training:
  random_seed: 123
  batch_size: 2
  simulated_batch_size: 128
  simulated_batch_size_valid: 1280
  validate_every: 1000
  persistent_snapshot_every: 50000
  max_validation_samples: 10000
  metrics:
    - top1_accuracy
    - top5_accuracy
    - non_trivial_accuracy
    - precision
    - recall
    - f1_score
    - rouge_2
    - rouge_l