/
SconsCaching.py
412 lines (326 loc) · 14.5 KB
/
SconsCaching.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
# Copyright 2022, Kay Hayen, mailto:kay.hayen@gmail.com
#
# Part of "Nuitka", an optimizing Python compiler that is compatible and
# integrates with CPython, but also works on its own.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
""" Caching of C compiler output.
"""
import ast
import os
import platform
import re
import sys
from collections import defaultdict
from nuitka.Tracing import scons_details_logger, scons_logger
from nuitka.utils.AppDirs import getCacheDir
from nuitka.utils.Download import getCachedDownload
from nuitka.utils.FileOperations import (
areSamePaths,
getExternalUsePath,
getFileContentByLine,
getFileContents,
getLinkTarget,
makePath,
)
from nuitka.utils.Importing import importFromInlineCopy
from nuitka.utils.Utils import isMacOS, isWin32Windows
from .SconsProgress import updateSconsProgressBar
from .SconsUtils import (
getExecutablePath,
getSconsReportValue,
setEnvironmentVariable,
)
def _getPythonDirCandidates(python_prefix):
result = [python_prefix]
for python_dir in (
sys.prefix,
os.environ.get("CONDA_PREFIX"),
os.environ.get("CONDA"),
):
if python_dir and python_dir not in result:
result.append(python_dir)
return result
def _getCcacheGuessedPaths(python_prefix):
if isWin32Windows():
# Search the compiling Python, the Scons Python (likely the same, but not necessarily)
# and then Anaconda, if an environment variable present from activated, or installed in
# CI like GitHub actions.
for python_dir in _getPythonDirCandidates(python_prefix):
yield os.path.join(python_dir, "bin", "ccache.exe")
yield os.path.join(python_dir, "scripts", "ccache.exe")
elif isMacOS():
# For macOS, we might find Homebrew ccache installed but not in PATH.
yield "/usr/local/opt/ccache"
yield "/opt/homebrew/bin/ccache"
def _injectCcache(env, cc_path, python_prefix, target_arch, assume_yes_for_downloads):
ccache_binary = os.environ.get("NUITKA_CCACHE_BINARY")
# If not provided, search it in PATH and guessed directories.
if ccache_binary is None:
ccache_binary = getExecutablePath("ccache", env=env)
if ccache_binary is None:
for candidate in _getCcacheGuessedPaths(python_prefix):
scons_details_logger.info(
"Checking if ccache is at '%s' guessed path." % candidate
)
if os.path.exists(candidate):
ccache_binary = candidate
scons_details_logger.info(
"Using ccache '%s' from guessed path." % ccache_binary
)
break
if ccache_binary is None:
if isWin32Windows():
url = "https://github.com/ccache/ccache/releases/download/v4.6/ccache-4.6-windows-32.zip"
ccache_binary = getCachedDownload(
url=url,
is_arch_specific=False,
specificity=url.rsplit("/", 2)[1],
flatten=True,
binary="ccache.exe",
message="Nuitka will make use of ccache to speed up repeated compilation.",
reject=None,
assume_yes_for_downloads=assume_yes_for_downloads,
)
elif isMacOS():
# TODO: Do not yet have M1 access to create one and 10.14 is minimum
# we managed to compile ccache for.
if target_arch != "arm64" and tuple(
int(d) for d in platform.release().split(".")
) >= (18, 2):
url = "https://nuitka.net/ccache/v4.2.1/ccache-4.2.1.zip"
ccache_binary = getCachedDownload(
url=url,
is_arch_specific=False,
specificity=url.rsplit("/", 2)[1],
flatten=True,
binary="ccache",
message="Nuitka will make use of ccache to speed up repeated compilation.",
reject=None,
assume_yes_for_downloads=assume_yes_for_downloads,
)
else:
scons_details_logger.info(
"Using ccache '%s' from NUITKA_CCACHE_BINARY environment variable."
% ccache_binary
)
if ccache_binary is not None and os.path.exists(ccache_binary):
# Make sure the
# In case we are on Windows, make sure the Anaconda form runs outside of Anaconda
# environment, by adding DLL folder to PATH.
assert areSamePaths(
getExecutablePath(os.path.basename(env.the_compiler), env=env), cc_path
)
# We use absolute paths for CC, pass it like this, as ccache does not like absolute.
env["CXX"] = env["CC"] = '"%s" "%s"' % (ccache_binary, cc_path)
# Spare ccache the detection of the compiler, seems it will also misbehave when it's
# prefixed with "ccache" on old gcc versions in terms of detecting need for C++ linkage.
env["LINK"] = cc_path
scons_details_logger.info(
"Found ccache '%s' to cache C compilation result." % ccache_binary
)
scons_details_logger.info(
"Providing real CC path '%s' via PATH extension." % cc_path
)
def enableCcache(
env,
source_dir,
python_prefix,
target_arch,
assume_yes_for_downloads,
):
# The ccache needs absolute path, otherwise it will not work.
ccache_logfile = os.path.abspath(
os.path.join(source_dir, "ccache-%d.txt" % os.getpid())
)
setEnvironmentVariable(env, "CCACHE_LOGFILE", ccache_logfile)
env["CCACHE_LOGFILE"] = ccache_logfile
# Unless asked to do otherwise, store ccache files in our own directory.
if "CCACHE_DIR" not in os.environ:
ccache_dir = os.path.join(getCacheDir(), "ccache")
makePath(ccache_dir)
ccache_dir = getExternalUsePath(ccache_dir)
setEnvironmentVariable(env, "CCACHE_DIR", ccache_dir)
env["CCACHE_DIR"] = ccache_dir
# We know the include files we created are safe to use.
setEnvironmentVariable(
env, "CCACHE_SLOPPINESS", "include_file_ctime,include_file_mtime"
)
# First check if it's not already supposed to be a ccache, then do nothing.
cc_path = getExecutablePath(env.the_compiler, env=env)
cc_is_link, cc_link_path = getLinkTarget(cc_path)
if cc_is_link and os.path.basename(cc_link_path) == "ccache":
scons_details_logger.info(
"Chosen compiler %s is pointing to ccache %s already."
% (cc_path, cc_link_path)
)
return True
return _injectCcache(
env=env,
cc_path=cc_path,
python_prefix=python_prefix,
target_arch=target_arch,
assume_yes_for_downloads=assume_yes_for_downloads,
)
def enableClcache(env, source_dir):
importFromInlineCopy("atomicwrites", must_exist=True)
importFromInlineCopy("clcache", must_exist=True)
# Avoid importing this in threads, triggers CPython 3.9 importing bugs at least,
# do it now, so it's not a race issue.
import concurrent.futures.thread # pylint: disable=I0021,unused-import,unused-variable
cl_binary = getExecutablePath(env.the_compiler, env)
# The compiler is passed via environment.
setEnvironmentVariable(env, "CLCACHE_CL", cl_binary)
env["CXX"] = env["CC"] = "<clcache>"
setEnvironmentVariable(env, "CLCACHE_HIDE_OUTPUTS", "1")
# Use the mode of clcache that is not dependent on MSVC filenames output
if "CLCACHE_NODIRECT" not in os.environ:
setEnvironmentVariable(env, "CLCACHE_NODIRECT", "1")
# The clcache stats filename needs absolute path, otherwise it will not work.
clcache_stats_filename = os.path.abspath(
os.path.join(source_dir, "clcache-stats.%d.txt" % os.getpid())
)
setEnvironmentVariable(env, "CLCACHE_STATS", clcache_stats_filename)
env["CLCACHE_STATS"] = clcache_stats_filename
# Unless asked to do otherwise, store ccache files in our own directory.
if "CLCACHE_DIR" not in os.environ:
clcache_dir = os.path.join(getCacheDir(), "clcache")
makePath(clcache_dir)
clcache_dir = getExternalUsePath(clcache_dir)
setEnvironmentVariable(env, "CLCACHE_DIR", clcache_dir)
env["CLCACHE_DIR"] = clcache_dir
scons_details_logger.info(
"Using inline copy of clcache with %r cl binary." % cl_binary
)
import atexit
atexit.register(_writeClcacheStatistics)
def _writeClcacheStatistics():
try:
# pylint: disable=I0021,import-error,no-name-in-module,redefined-outer-name
from clcache.caching import stats
if stats is not None:
stats.save()
except IOError:
raise
except Exception: # Catch all the things, pylint: disable=broad-except
# This is run in "atexit" even without the module being loaded, or
# the stats being begun or usable.
pass
def _getCcacheStatistics(ccache_logfile):
data = {}
if os.path.exists(ccache_logfile):
re_command = re.compile(r"\[.*? (\d+) *\] Command line: (.*)$")
re_result = re.compile(r"\[.*? (\d+) *\] Result: (.*)$")
re_anything = re.compile(r"\[.*? (\d+) *\] (.*)$")
# Remember command from the pid, so later decision logged against pid
# can be matched against it.
commands = {}
for line in getFileContentByLine(ccache_logfile):
match = re_command.match(line)
if match:
pid, command = match.groups()
commands[pid] = command
match = re_result.match(line)
if match:
pid, result = match.groups()
result = result.strip()
try:
command = data[commands[pid]]
except KeyError:
# It seems writing to the file can be lossy, so we can have results for
# unknown commands, but we don't use the command yet anyway, so just
# be unique.
command = "unknown command leading to " + line
# Older ccache on e.g. RHEL6 wasn't explicit about linking.
if result == "unsupported compiler option":
if " -o " in command or "unknown command" in command:
result = "called for link"
# But still try to catch this with log output if it happens.
if result == "unsupported compiler option":
scons_logger.warning(
"Encountered unsupported compiler option for ccache in '%s'."
% command
)
all_text = []
for line2 in getFileContentByLine(ccache_logfile):
match = re_anything.match(line2)
if match:
pid2, result = match.groups()
if pid == pid2:
all_text.append(result)
scons_logger.warning("Full scons output: %s" % all_text)
if result != "called for link":
data[command] = result
return data
def checkCachingSuccess(source_dir):
ccache_logfile = getSconsReportValue(source_dir=source_dir, key="CCACHE_LOGFILE")
if ccache_logfile is not None:
stats = _getCcacheStatistics(ccache_logfile)
if not stats:
scons_logger.warning("You are not using ccache.")
else:
counts = defaultdict(int)
for _command, result in stats.items():
# These are not important to our users, time based decisions differentiate these.
if result in ("cache hit (direct)", "cache hit (preprocessed)"):
result = "cache hit"
# Newer ccache has these, but they duplicate:
if result in (
"direct_cache_hit",
"direct_cache_miss",
"preprocessed_cache_hit",
"preprocessed_cache_miss",
"primary_storage_miss",
):
continue
if result == "primary_storage_hit":
result = "cache hit"
if result == "cache_miss":
result = "cache miss"
# Usage of incbin causes this for the constants blob integration.
if result in ("unsupported code directive", "disabled"):
continue
counts[result] += 1
scons_logger.info("Compiled %d C files using ccache." % len(stats))
for result, count in counts.items():
scons_logger.info(
"Cached C files (using ccache) with result '%s': %d"
% (result, count)
)
if os.name == "nt":
clcache_stats_filename = getSconsReportValue(
source_dir=source_dir, key="CLCACHE_STATS"
)
if clcache_stats_filename is not None and os.path.exists(
clcache_stats_filename
):
stats = ast.literal_eval(getFileContents(clcache_stats_filename))
clcache_hit = stats["CacheHits"]
clcache_miss = stats["CacheMisses"]
scons_logger.info(
"Compiled %d C files using clcache with %d cache hits and %d cache misses."
% (clcache_hit + clcache_miss, clcache_hit, clcache_miss)
)
def runClCache(args, env):
# pylint: disable=I0021,import-error,no-name-in-module,redefined-outer-name
from clcache.caching import runClCache
# No Python2 compatibility
if str is bytes:
scons_logger.sysexit("Error, cannot use Python2 for scons when using MSVC.")
# The first argument is "<clcache>" and should not be used.
result = runClCache(
os.environ["CLCACHE_CL"], [arg.strip('"') for arg in args[1:]], env
)
updateSconsProgressBar()
return result