-
Notifications
You must be signed in to change notification settings - Fork 3
/
evaluate.sh
28 lines (21 loc) · 802 Bytes
/
evaluate.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
#!/usr/bin/env
# PACKAGE TO BE INCLUDED WHICH HOUSES ALL THE CODE
INCLUDE_PACKAGE=semqa
export GPU=-1
export BEAMSIZE=1
export DEBUG=true
# SAVED MODEL
MODEL_DIR=./resources/iclr_cameraready/ckpt
MODEL_TAR=${MODEL_DIR}/model.tar.gz
PREDICTION_DIR=${MODEL_DIR}/predictions
mkdir ${PREDICTION_DIR}
DATASET_DIR=./resources/iclr_cameraready
DATASET_NAME=iclr_drop_data
VALDATA_FILE=${DATASET_DIR}/${DATASET_NAME}/drop_dataset_mydev.json
VAL_METRICS_FILE=${PREDICTION_DIR}/drop_mydev_metrics.json
# Validation over complete dataset
allennlp evaluate --output-file ${VAL_METRICS_FILE} \
--cuda-device ${GPU} \
--overrides "{"model": {"cuda_device": ${GPU} }}" \
--include-package ${INCLUDE_PACKAGE} \
${MODEL_TAR} ${VALDATA_FILE}