Releases: microsoft/ContextualSP
Releases · microsoft/ContextualSP
LogiGAN: model and data
Training data and LogiGAN pretrained model.
The warmup pre-traing data is gen_train_A.json, and the continual adversarial pre-training data is gen_train_B.json (gen_train_B.json with elastic searched pseudo samples is stored in train_data_with_es/gen_train_src_es.jsonl).
lemon_data
LEMON dataset and pre-training corpus.
Pretrained Weights of Multi BERT-based RUN
{
"ROUGE": 0.8954699040374693,
"_ROUGE1": 0.9248370079585566,
"_ROUGE2": 0.8548729804396925,
"EM": 0.4933385579937304,
"_P1": 0.7443478260869565,
"_R1": 0.6512335615693946,
"F1": 0.694684366123703,
"_P2": 0.6040515653775322,
"_R2": 0.5369713506139154,
"F2": 0.5685396504405605,
"_P3": 0.515867089789061,
"_R3": 0.4619306310071041,
"F3": 0.48741126151946734,
"_BLEU1": 0.9203424772022362,
"_BLEU2": 0.8919446800461631,
"_BLEU3": 0.8644065076063657,
"BLEU4": 0.836555297206264,
"loss": 0.012869063752786444
}Pretrained Weights of Multi RUN
{
"ROUGE": 0.8761914250271031,
"_ROUGE1": 0.9096109449987978,
"_ROUGE2": 0.8299625766685176,
"EM": 0.43260188087774293,
"_P1": 0.663828689370485,
"_R1": 0.5592870340180415,
"F1": 0.6070901905267504,
"_P2": 0.5118311660626732,
"_R2": 0.43674397453387903,
"F2": 0.4713156990724837,
"_P3": 0.42003325833903943,
"_R3": 0.3588800668616799,
"F3": 0.38705606634216694,
"_BLEU1": 0.9049646708813686,
"_BLEU2": 0.8726836197111325,
"_BLEU3": 0.8417857314177102,
"BLEU4": 0.810576724115273,
"loss": 0.01269258979597015
}Pretrained Weights of Rewrite BERT-based RUN
{
"ROUGE": 0.9394040084189113,
"_ROUGE1": 0.961865057419486,
"_ROUGE2": 0.9113051224617216,
"EM": 0.688,
"_P1": 0.9451903332806824,
"_R1": 0.8668694770389685,
"F1": 0.9043373129817137,
"_P2": 0.8648273949812838,
"_R2": 0.7989241803278688,
"F2": 0.8305705345849144,
"_P3": 0.8075098814229249,
"_R3": 0.7449860216360763,
"F3": 0.774988935954985,
"_BLEU1": 0.9405510823944796,
"_BLEU2": 0.9172718486250105,
"_BLEU3": 0.8932687251641028,
"BLEU4": 0.8691863201601382,
"loss": 0.2084200546145439
}Pretrained Weights of Rewrite RUN
{
"ROUGE": 0.8927719935645371,
"_ROUGE1": 0.9235141523874184,
"_ROUGE2": 0.8481181915297369,
"EM": 0.536,
"_P1": 0.8750417083750417,
"_R1": 0.7598145733738955,
"F1": 0.8133674497945259,
"_P2": 0.7759128386336867,
"_R2": 0.6749487704918032,
"F2": 0.7219178082191782,
"_P3": 0.7060550971892043,
"_R3": 0.6137109517442567,
"F3": 0.6566523605150214,
"_BLEU1": 0.8953007499145113,
"_BLEU2": 0.8627271154439679,
"_BLEU3": 0.8298150265279266,
"BLEU4": 0.7961327186223426,
"loss": 0.07524961760888497
}Pretrained Weights of CANARD RUN
{
"ROUGE": 0.7672702010110017,
"_ROUGE1": 0.8060129493808116,
"_ROUGE2": 0.6272019845050271,
"EM": 0.1825525040387722,
"_P1": 0.6173277359583093,
"_R1": 0.34435562015503873,
"F1": 0.44210035247771096,
"_P2": 0.39885553659271156,
"_R2": 0.24790964682391115,
"F2": 0.3057682687497595,
"_P3": 0.31307374374592123,
"_R3": 0.20793481087641227,
"F3": 0.24989581886373105,
"_BLEU1": 0.7020165933536937,
"_BLEU2": 0.6217319698491652,
"_BLEU3": 0.5573085608208992,
"BLEU4": 0.4977857601746373,
"loss": 0.01892541069261615
}Pretrained Weights of SParC Turn Model
{
"best_epoch": 48,
"peak_cpu_memory_MB": 0,
"peak_gpu_0_memory_MB": 1,
"peak_gpu_1_memory_MB": 3489,
"peak_gpu_2_memory_MB": 16884,
"peak_gpu_3_memory_MB": 16857,
"peak_gpu_4_memory_MB": 16886,
"peak_gpu_5_memory_MB": 16816,
"peak_gpu_6_memory_MB": 20163,
"peak_gpu_7_memory_MB": 19975,
"training_duration": "8:57:20.749721",
"training_start_epoch": 0,
"training_epochs": 57,
"epoch": 57,
"training_action_exact_match": 0,
"training__action_inter_exact_match": 0,
"training__action_turn_1_exact_match": 0,
"training__action_turn_2_exact_match": 0,
"training__action_turn_3_exact_match": 0,
"training__action_turn_4_exact_match": 0,
"training_sql_exact_match": 0,
"training__sql_inter_exact_match": 0,
"training__sql_turn_1_exact_match": 0,
"training__sql_turn_2_exact_match": 0,
"training__sql_turn_3_exact_match": 0,
"training__sql_turn_4_exact_match": 0,
"training_loss": 0.03923457075834353,
"training_cpu_memory_MB": 0.0,
"training_gpu_0_memory_MB": 1,
"training_gpu_1_memory_MB": 1751,
"training_gpu_2_memory_MB": 16884,
"training_gpu_3_memory_MB": 16857,
"training_gpu_4_memory_MB": 16886,
"training_gpu_5_memory_MB": 16816,
"training_gpu_6_memory_MB": 20163,
"training_gpu_7_memory_MB": 2,
"validation_action_exact_match": 0.4056525353283458,
"validation__action_inter_exact_match": 0.1966824644549763,
"validation__action_turn_1_exact_match": 0.5521327014218009,
"validation__action_turn_2_exact_match": 0.3767772511848341,
"validation__action_turn_3_exact_match": 0.29259259259259257,
"validation__action_turn_4_exact_match": 0.19101123595505617,
"validation_sql_exact_match": 0.42726517040731504,
"validation__sql_inter_exact_match": 0.2132701421800948,
"validation__sql_turn_1_exact_match": 0.5592417061611374,
"validation__sql_turn_2_exact_match": 0.3933649289099526,
"validation__sql_turn_3_exact_match": 0.3296296296296296,
"validation__sql_turn_4_exact_match": 0.25842696629213485,
"validation_loss": 0.1820486258486621,
"best_validation_action_exact_match": 0.4222776392352452,
"best_validation__action_inter_exact_match": 0.2014218009478673,
"best_validation__action_turn_1_exact_match": 0.5710900473933649,
"best_validation__action_turn_2_exact_match": 0.3933649289099526,
"best_validation__action_turn_3_exact_match": 0.3,
"best_validation__action_turn_4_exact_match": 0.2247191011235955,
"best_validation_sql_exact_match": 0.43557772236076475,
"best_validation__sql_inter_exact_match": 0.2037914691943128,
"best_validation__sql_turn_1_exact_match": 0.5687203791469194,
"best_validation__sql_turn_2_exact_match": 0.3981042654028436,
"best_validation__sql_turn_3_exact_match": 0.3296296296296296,
"best_validation__sql_turn_4_exact_match": 0.30337078651685395,
"best_validation_loss": 0.17382763872011436
}Pretrained Weights of SParC Action Copy Model
{
"best_epoch": 22,
"peak_cpu_memory_MB": 0,
"peak_gpu_0_memory_MB": 2796,
"peak_gpu_1_memory_MB": 16881,
"peak_gpu_2_memory_MB": 1479,
"peak_gpu_3_memory_MB": 16586,
"peak_gpu_4_memory_MB": 496,
"peak_gpu_5_memory_MB": 1968,
"peak_gpu_6_memory_MB": 496,
"peak_gpu_7_memory_MB": 507,
"training_duration": "5:31:39.494340",
"training_start_epoch": 0,
"training_epochs": 31,
"epoch": 31,
"training_action_exact_match": 0,
"training__action_inter_exact_match": 0,
"training__action_turn_1_exact_match": 0,
"training__action_turn_2_exact_match": 0,
"training__action_turn_3_exact_match": 0,
"training__action_turn_4_exact_match": 0,
"training_sql_exact_match": 0,
"training__sql_inter_exact_match": 0,
"training__sql_turn_1_exact_match": 0,
"training__sql_turn_2_exact_match": 0,
"training__sql_turn_3_exact_match": 0,
"training__sql_turn_4_exact_match": 0,
"training__copy": 0,
"training_info": 0,
"training_loss": 0.06896944765227608,
"training_cpu_memory_MB": 0.0,
"training_gpu_0_memory_MB": 2796,
"training_gpu_1_memory_MB": 16879,
"training_gpu_2_memory_MB": 1479,
"training_gpu_3_memory_MB": 16586,
"training_gpu_4_memory_MB": 496,
"training_gpu_5_memory_MB": 1968,
"training_gpu_6_memory_MB": 496,
"training_gpu_7_memory_MB": 507,
"validation_action_exact_match": 0.37323358270989193,
"validation__action_inter_exact_match": 0.17535545023696683,
"validation__action_turn_1_exact_match": 0.5165876777251185,
"validation__action_turn_2_exact_match": 0.33886255924170616,
"validation__action_turn_3_exact_match": 0.2851851851851852,
"validation__action_turn_4_exact_match": 0.12359550561797752,
"validation_sql_exact_match": 0.3873649210307564,
"validation__sql_inter_exact_match": 0.1895734597156398,
"validation__sql_turn_1_exact_match": 0.518957345971564,
"validation__sql_turn_2_exact_match": 0.35545023696682465,
"validation__sql_turn_3_exact_match": 0.3,
"validation__sql_turn_4_exact_match": 0.1797752808988764,
"validation__copy": 0,
"validation_info": 0,
"validation_loss": 0.13728585836002538,
"best_validation_action_exact_match": 0.372402327514547,
"best_validation__action_inter_exact_match": 0.18009478672985782,
"best_validation__action_turn_1_exact_match": 0.5260663507109005,
"best_validation__action_turn_2_exact_match": 0.33886255924170616,
"best_validation__action_turn_3_exact_match": 0.25925925925925924,
"best_validation__action_turn_4_exact_match": 0.14606741573033707,
"best_validation_sql_exact_match": 0.38902743142144636,
"best_validation__sql_inter_exact_match": 0.1966824644549763,
"best_validation__sql_turn_1_exact_match": 0.5355450236966824,
"best_validation__sql_turn_2_exact_match": 0.35545023696682465,
"best_validation__sql_turn_3_exact_match": 0.2740740740740741,
"best_validation__sql_turn_4_exact_match": 0.20224719101123595,
"best_validation__copy": 0,
"best_validation_info": 0,
"best_validation_loss": 0.13164386158200506
}Pretrained Weights of SParC BERT-based Turn Model
{
"best_epoch": 45,
"peak_cpu_memory_MB": 0,
"peak_gpu_0_memory_MB": 1,
"peak_gpu_1_memory_MB": 4185,
"peak_gpu_2_memory_MB": 16884,
"peak_gpu_3_memory_MB": 16857,
"peak_gpu_4_memory_MB": 16886,
"peak_gpu_5_memory_MB": 16816,
"peak_gpu_6_memory_MB": 22499,
"peak_gpu_7_memory_MB": 20477,
"training_duration": "15:38:25.817253",
"training_start_epoch": 0,
"training_epochs": 54,
"epoch": 54,
"training_action_exact_match": 0,
"training__action_inter_exact_match": 0,
"training__action_turn_1_exact_match": 0,
"training__action_turn_2_exact_match": 0,
"training__action_turn_3_exact_match": 0,
"training__action_turn_4_exact_match": 0,
"training_sql_exact_match": 0,
"training__sql_inter_exact_match": 0,
"training__sql_turn_1_exact_match": 0,
"training__sql_turn_2_exact_match": 0,
"training__sql_turn_3_exact_match": 0,
"training__sql_turn_4_exact_match": 0,
"training_loss": 0.03840326214035231,
"training_cpu_memory_MB": 0.0,
"training_gpu_0_memory_MB": 1,
"training_gpu_1_memory_MB": 3238,
"training_gpu_2_memory_MB": 16884,
"training_gpu_3_memory_MB": 16857,
"training_gpu_4_memory_MB": 16886,
"training_gpu_5_memory_MB": 2,
"training_gpu_6_memory_MB": 22499,
"training_gpu_7_memory_MB": 20477,
"validation_action_exact_match": 0.4405652535328346,
"validation__action_inter_exact_match": 0.23459715639810427,
"validation__action_turn_1_exact_match": 0.6018957345971564,
"validation__action_turn_2_exact_match": 0.4218009478672986,
"validation__action_turn_3_exact_match": 0.2962962962962963,
"validation__action_turn_4_exact_match": 0.20224719101123595,
"validation_sql_exact_match": 0.46467165419783873,
"validation__sql_inter_exact_match": 0.26303317535545023,
"validation__sql_turn_1_exact_match": 0.6184834123222749,
"validation__sql_turn_2_exact_match": 0.45260663507109006,
"validation__sql_turn_3_exact_match": 0.32592592592592595,
"validation__sql_turn_4_exact_match": 0.21348314606741572,
"validation_loss": 0.20552104420625325,
"best_validation_action_exact_match": 0.4513715710723192,
"best_validation__action_inter_exact_match": 0.23933649289099526,
"best_validation__action_turn_1_exact_match": 0.6137440758293838,
"best_validation__action_turn_2_exact_match": 0.43364928909952605,
"best_validation__action_turn_3_exact_match": 0.28888888888888886,
"best_validation__action_turn_4_exact_match": 0.25842696629213485,
"best_validation_sql_exact_match": 0.4696591853699086,
"best_validation__sql_inter_exact_match": 0.26303317535545023,
"best_validation__sql_turn_1_exact_match": 0.6255924170616114,
"best_validation__sql_turn_2_exact_match": 0.45734597156398105,
"best_validation__sql_turn_3_exact_match": 0.3148148148148148,
"best_validation__sql_turn_4_exact_match": 0.25842696629213485,
"best_validation_loss": 0.18703062564368952
}