-
Notifications
You must be signed in to change notification settings - Fork 118
/
.gitlab-ci.yml
163 lines (155 loc) · 9.78 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
image: registry.gitlab.com/plasticity/ubuntu-dind:16.04
services:
- docker:dind
stages:
- Test
- Deploy to PyPI
- Deploy Linux Wheels
before_script:
# Setup package manager
- apt-get update 2>&1 >/dev/null
- apt-get install curl wget git -y
- apt-get install libblas-dev liblapack-dev libatlas-base-dev gfortran -y
- apt-get install python -y
- apt-get install python-pip -y
- apt-get install python-setuptools -y
- apt-get install python3-setuptools python3-dev python3-pip -y
- apt-get install python-dev -y
- apt-get install openssh-server -y
# Setup SSH configuration
- mkdir -p ~/.ssh
- echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
# Install gettext for envsubst
- apt-get install gettext -y
Lint:
stage: Test
script:
- apt-get install python3 -y
- apt-get install python3-pip -y
- pip3 install flake8
- pip3 install pep8-naming
- flake8 --version
- flake8
Test Python 2:
stage: Test
script:
- apt-get install python-pip -y
- pip2 install torch\>=0.4.1 # This is not in requirements.txt so must be installed like this
- pip2 install -U pip setuptools
- python2 setup.py install
- pip install gensim
- wget --quiet http://magnitude.plasticity.ai/word2vec/light/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.light.magnitude
- wget --quiet http://magnitude.plasticity.ai/word2vec/medium/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.medium.magnitude
- wget --quiet http://magnitude.plasticity.ai/word2vec/heavy/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.heavy.magnitude
- python2 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude
- python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude
- python2 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude
- python2 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude
- python2 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude
- python2 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude
- python2 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s
- python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s
- python2 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s
- python2 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s
- python2 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s
- python2 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s
- python2 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s -a
- python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s -a
- python2 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s -a
- python2 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s -a
- python2 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s -a
- python2 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s -a
- python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o w2v.magnitude
- python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude
- python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s
- python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s -a
- python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude
- python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -s
- python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -a
- ls
- python2 -m tests.tests -i GoogleNews-vectors-negative300.light.magnitude -s GoogleNews-vectors-negative300.medium.magnitude -a GoogleNews-vectors-negative300.heavy.magnitude -- -v
Test Python 3:
stage: Test
script:
- apt-get install build-essential -y
- apt-get install libffi-dev libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev -y
- apt-get install libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev -y
- wget https://www.python.org/ftp/python/3.7.1/Python-3.7.1.tar.xz
- tar xf Python-3.7.1.tar.xz
- cd Python-3.7.1 && ./configure --enable-shared && make -j 8 && make altinstall && cd ../
- echo $(pwd)/Python-3.7.1
- ls $(pwd)/Python-3.7.1
- export LD_LIBRARY_PATH=$(pwd)/Python-3.7.1:$LD_LIBRARY_PATH
- echo $LD_LIBRARY_PATH
- curl https://bootstrap.pypa.io/get-pip.py | python3.7
- python3.7 -m pip install torch\>=0.4.1 # This is not in requirements.txt so must be installed like this
- python3.7 -m pip install -U wheel setuptools # TEMP: removing upgrading of pip here, because the newest one is broken for Python 3
- python3.7 setup.py install
- python3.7 -m pip install gensim
- wget --quiet http://magnitude.plasticity.ai/word2vec/light/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.light.magnitude
- wget --quiet http://magnitude.plasticity.ai/word2vec/medium/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.medium.magnitude
- wget --quiet http://magnitude.plasticity.ai/word2vec/heavy/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.heavy.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s
- python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s
- python3.7 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s
- python3.7 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s
- python3.7 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s
- python3.7 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s
- python3.7 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s -a
- python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s -a
- python3.7 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s -a
- python3.7 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s -a
- python3.7 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s -a
- python3.7 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s -a
- python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o w2v.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s
- python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s -a
- python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude
- python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -s
- python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -a
- ls
- python3.7 -m tests.tests -i GoogleNews-vectors-negative300.light.magnitude -s GoogleNews-vectors-negative300.medium.magnitude -a GoogleNews-vectors-negative300.heavy.magnitude -- -v
Deploy to PyPI:
stage: Deploy to PyPI
script:
# Add GitLab SSH private deploy key
- eval $(ssh-agent -s)
- tmpfile=$(mktemp ~/pk.XXXXXX)
- echo "$SSH_PRIVATE_KEY" > $tmpfile
- ssh-add $tmpfile
- rm $tmpfile
# Tag the release on GitLab
- rm -rf ../tagger
- mkdir -p ../tagger
- cd ../tagger
- git clone git@gitlab.com:$CI_PROJECT_PATH.git
- cd $CI_PROJECT_NAME
- git config --global user.email "dev_ci_bot@plasticity.ai"
- git config --global user.name "Plasticity Developer Bot"
- git tag -a $(python setup.py -V) -m "Release $(python setup.py -V)"
- git push origin --tags
- cd $CI_PROJECT_DIR
- rm -rf ../tagger
- sleep 60 # Wait for GitLab to mirror to GitHub
# Create a release on GitHub
- curl -u plasticity-admin:$GITHUB_TOKEN -d "{\"tag_name\":\"$(python setup.py -V)\", \"name\":\"Release $(python setup.py -V)\"}" -H "Content-Type:"" application/json" -X POST https://api.github.com/repos/plasticityai/$CI_PROJECT_NAME/releases
# Upload to PyPI
- envsubst < deployment/.pypirc > ~/.pypirc
- chmod 600 ~/.pypirc
- pip2 install setuptools wheel twine -U
- pip3 install setuptools wheel twine -U
- python3 setup.py sdist upload -r pypitest
- python3 setup.py sdist upload -r pypi
# Trigger Travis CI
- curl -s -X POST -H "Content-Type:application/json" -H "Accept:application/json" -H "Travis-API-Version:3" -H "Authorization:token $TRAVIS_CI_TOKEN" -d '{"request":{"branch":"master"}}' https://api.travis-ci.org/repo/plasticityai%2Fmagnitude/requests
# Trigger Appveyor CI
- curl -s -X POST -H "Content-Type:application/json" -H "Accept:application/json" -H "Authorization:Bearer $APPVEYOR_CI_TOKEN" -d '{"accountName":"plasticity-admin","projectSlug":"magnitude","branch":"master"}' https://ci.appveyor.com/api/builds
only:
- master