-
Notifications
You must be signed in to change notification settings - Fork 28
/
azure-pipelines.yml
251 lines (228 loc) 路 8.77 KB
/
azure-pipelines.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
trigger:
branches:
include:
- branch-for-*
pr: none
variables:
# Name of the service connection set up in Azure Pipelines, used to upload
# release artifacts to GitHub. Set to false to disable.
GH_CONNECTION: wheelwright
REPO_DIR: checkout
ARTIFACT_NAME: artifacts
LINUX_TARGET: /io
BUILD_DEPENDS: "-Ur checkout/requirements.txt setuptools"
TEST_DEPENDS: "-Ur checkout/requirements.txt setuptools"
PLAT: x86_64
UNICODE_WIDTH: 32
HOMEBREW_NO_AUTO_UPDATE: 1
MANYLINUX_VERSION: 2014
jobs:
- job: "sdist"
pool:
vmImage: 'ubuntu-20.04'
variables:
python.version: '3.9'
CC: gcc-9
CXX: g++-9
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
# This is messy, but the most convenient way to turn a JSON file into env
# variables without requiring extra dependencies. The "##vso" stuff makes the
# variables available to later steps.
#
# We're also setting py35 as an output variable to be used in the next
# step and its build matrix. This lets us avoid having to add complex
# conditions to each build step (as there seems to be no convenient way to
# just skip the whole rest of the pipeline).
- bash: |
eval $(python -c "
import json; import sys
with open('build-spec.json') as f: spec = json.load(f)
release_id = spec.get('upload-to', {}).get('release-id', '').replace('/', '-')
build_py35 = spec.get('options', {}).get('py35', '')
skip_tests = spec.get('options', {}).get('skip_tests', '')
universal = spec.get('options', {}).get('universal', '')
sys.stdout.write('BUILD_SPEC_CLONE_URL={}\n'.format(spec.get('clone-url')))
sys.stdout.write('BUILD_SPEC_COMMIT={}\n'.format(spec.get('commit')))
sys.stdout.write('BUILD_SPEC_PACKAGE_NAME={}\n'.format(spec.get('package-name')))
sys.stdout.write('BUILD_SPEC_RELEASE_ID={}\n'.format(release_id))
sys.stdout.write('BUILD_SPEC_PY35={}\n'.format(build_py35))
sys.stdout.write('BUILD_SPEC_SKIP_TESTS={}\n'.format(skip_tests))
sys.stdout.write('BUILD_SPEC_UNIVERSAL={}\n'.format(universal))")
echo "##vso[task.setvariable variable=clone_url]$BUILD_SPEC_CLONE_URL"
echo "##vso[task.setvariable variable=commit]$BUILD_SPEC_COMMIT"
echo "##vso[task.setvariable variable=package_name]$BUILD_SPEC_PACKAGE_NAME"
echo "##vso[task.setvariable variable=checkout]$REPO_DIR"
echo "##vso[task.setvariable variable=release_tag]$BUILD_SPEC_RELEASE_ID"
echo "##vso[task.setvariable variable=py35;isOutput=true]$BUILD_SPEC_PY35"
echo "##vso[task.setvariable variable=skip_tests;isOutput=true]$BUILD_SPEC_SKIP_TESTS"
echo "##vso[task.setvariable variable=universal;isOutput=true]$BUILD_SPEC_UNIVERSAL"
name: set_variables
displayName: 'Set variables'
- script: |
export CC=$(CC)
export CXX=$(CXX)
git clone $(clone_url) $(checkout)
cd $(checkout)
git checkout $(commit)
displayName: 'Checkout commit'
- script: |
cd $(checkout)
git submodule update --init
displayName: 'Update submodules'
- script: |
cd $(checkout)
python -m pip install -U build pip setuptools
python -m build --sdist
displayName: 'Install and build sdist'
- script: |
cd $(checkout)
export CC=$(CC)
export CXX=$(CXX)
python -m pip install -U wheel
python -m build --wheel
displayName: 'Build universal wheel'
condition: eq(variables['set_variables.universal'], 'True')
- bash: |
cd $(Build.SourcesDirectory)/$(checkout)
SDIST=$(python -c "import os;print(os.listdir('./dist')[0])" 2>&1)
rm -rf $(package_name)
python -m pip freeze > installed.txt
python -m pip uninstall -y -r installed.txt
python -m pip install dist/$SDIST
pip install -r requirements.txt
python -m pytest --pyargs $(package_name)
condition: ne(variables['set_variables.skip_tests'], 'True')
displayName: 'Install from sdist & run tests'
# - bash: |
# cd $(Build.SourcesDirectory)/$(checkout)
# python -m pip freeze > dist/pip-freeze-sdist.txt
# condition: ne(variables['set_variables.skip_tests'], 'True')
# displayName: 'Save pip freeze from test env'
- task: CopyFiles@2
inputs:
contents: '$(checkout)/dist/**'
targetFolder: $(Build.ArtifactStagingDirectory)
flattenFolders: true
displayName: 'Copy sdist'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)
artifactName: '$(ARTIFACT_NAME)'
displayName: 'Publish sdist to artifacts'
- task: GitHubRelease@1
inputs:
gitHubConnection: '$(GH_CONNECTION)'
repositoryName: '$(Build.Repository.Name)'
action: 'edit'
tagSource: manual
tag: '$(release_tag)'
addChangeLog: false
assetUploadMode: replace
assets: '$(Build.ArtifactStagingDirectory)/*'
displayName: 'Upload to GitHub release'
- job: 'wheels'
dependsOn: 'sdist'
condition: not(eq(dependencies.sdist.outputs['set_variables.universal'], 'True'))
continueOnError: true
timeoutInMinutes: 240
strategy:
matrix:
Python311Windows:
imageName: 'windows-2019'
python.version: '3.11'
os: win
Python311Linux:
imageName: 'ubuntu-20.04'
python.version: '3.11'
os: linux
Python311Mac:
imageName: 'macos-11'
python.version: '3.11'
os: osx
maxParallel: 4
pool:
vmImage: $(imageName)
variables:
py35: $[ dependencies.sdist.outputs['set_variables.py35'] ]
steps:
- template: azure-wheels-steps.yml
parameters:
python_version: '$(python.version)'
- job: 'ec2wheels_aarch64'
dependsOn: 'sdist'
condition: not(eq(dependencies.sdist.outputs['set_variables.universal'], 'True'))
continueOnError: true
timeoutInMinutes: 120
strategy:
matrix:
Python310Linux:
imageName: 'ubuntu-20.04'
python.version: '3.10'
os: linux
maxParallel: 4
pool:
vmImage: $(imageName)
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
# This is messy, but the most convenient way to turn a JSON file into env
# variables without requiring extra dependencies. The "##vso" stuff makes the
# variables available to later steps.
- bash: |
eval $(python -c "
import json; import sys
with open('build-spec.json') as f: spec = json.load(f)
release_id = spec.get('upload-to', {}).get('release-id', '').replace('/', '-')
skip_tests = spec.get('options', {}).get('skip_tests', '')
universal = spec.get('options', {}).get('universal', '')
sys.stdout.write('BUILD_SPEC_CLONE_URL={}\n'.format(spec.get('clone-url')))
sys.stdout.write('BUILD_SPEC_COMMIT={}\n'.format(spec.get('commit')))
sys.stdout.write('BUILD_SPEC_PACKAGE_NAME={}\n'.format(spec.get('package-name')))
sys.stdout.write('BUILD_SPEC_RELEASE_ID={}\n'.format(release_id))
sys.stdout.write('BUILD_SPEC_SKIP_TESTS={}\n'.format(skip_tests))
sys.stdout.write('BUILD_SPEC_UNIVERSAL={}\n'.format(universal))")
echo "##vso[task.setvariable variable=clone_url]$BUILD_SPEC_CLONE_URL"
echo "##vso[task.setvariable variable=commit]$BUILD_SPEC_COMMIT"
echo "##vso[task.setvariable variable=package_name]$BUILD_SPEC_PACKAGE_NAME"
echo "##vso[task.setvariable variable=checkout]$REPO_DIR"
echo "##vso[task.setvariable variable=release_tag]$BUILD_SPEC_RELEASE_ID"
echo "##vso[task.setvariable variable=skip_tests;isOutput=true]$BUILD_SPEC_SKIP_TESTS"
echo "##vso[task.setvariable variable=universal;isOutput=true]$BUILD_SPEC_UNIVERSAL"
name: set_variables
displayName: 'Set variables'
- script: |
git clone https://github.com/explosion/ec2buildwheel
cd ec2buildwheel
python -m pip install --upgrade pip setuptools wheel
python -m pip install -r requirements.txt
python go.py --non-interactive $(clone_url) $(commit) --package-name $(package_name)
displayName: 'Build wheels'
env:
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
- task: CopyFiles@2
inputs:
contents: 'ec2buildwheel/wheelhouse/**'
targetFolder: $(Build.ArtifactStagingDirectory)
flattenFolders: true
displayName: 'Copy wheels'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: $(Build.ArtifactStagingDirectory)
artifactName: '$(ARTIFACT_NAME)'
displayName: 'Publish artifact'
- task: GitHubRelease@1
inputs:
gitHubConnection: '$(GH_CONNECTION)'
repositoryName: '$(Build.Repository.Name)'
action: 'edit'
tagSource: manual
tag: '$(release_tag)'
addChangeLog: false
assetUploadMode: replace
assets: '$(Build.ArtifactStagingDirectory)/*'
condition: not(eq(variables['GH_CONNECTION'], 'false'))
displayName: 'Upload to GitHub release'