makePDF.py

# ST2/ST3 compat
from __future__ import print_function
import sublime
if sublime.version() < '3000':
    # we are on ST2 and Python 2.X
	_ST3 = False
	import getTeXRoot
	import parseTeXlog
else:
	_ST3 = True
	from . import getTeXRoot
	from . import parseTeXlog

import sublime_plugin
import sys, os, os.path, platform, threading, functools
import subprocess
import types
import re
# Retrieve the LyTeXTools path to facilitate the polltexmk.sh invocation.
import inspect
filename = inspect.getframeinfo(inspect.currentframe()).filename
lytex_path = os.path.dirname(os.path.abspath(filename))
# lytex_path = os.getcwd()
import time
import codecs

DEBUG = False

# Compile current .tex file using platform-specific tool
# On Windows, use texify; on Mac, use latexmk
# Assumes executables are on the path
# Warning: we do not do "deep" safety checks

# This is basically a specialized exec command: we do not capture output,
# but instead look at log files to parse errors

# Encoding: especially useful for Windows
# TODO: counterpart for OSX? Guess encoding of files?
def getOEMCP():
	# Windows OEM/Ansi codepage mismatch issue.
	# We need the OEM cp, because texify and friends are console programs
	import ctypes
	codepage = ctypes.windll.kernel32.GetOEMCP()
	return str(codepage)


# Log parsing, TNG :-)
# Input: tex log file (decoded), split into lines
# Output: content to be displayed in output panel, split into lines

def parseTeXlog(log):
	print ("Parsing log file")
	errors = []
	warnings = []

	# loop over all log lines; construct error message as needed
	# This will be useful for multi-file documents

	# some regexes
	file_rx = re.compile(r"\(([^)]+)$")     # file name: "(" followed by anyting but "(" through the end of the line
	line_rx = re.compile(r"^l\.(\d+)\s(.*)")        # l.nn <text>
	warning_rx = re.compile(r"^(.*?) Warning: (.+)")  # Warnings, first line
	line_rx_latex_warn = re.compile(r"input line (\d+)\.$")  # Warnings, line number
	matched_parens_rx = re.compile(r"\([^()]*\)")  # matched parentheses, to be deleted (note: not if nested)
	assignment_rx = re.compile(r"\\[^=]*=")  # assignment, heuristics for line merging

	files = []

	# Support function to handle warnings
	def handle_warning(l):
		warn_match_line = line_rx_latex_warn.search(l)
		if warn_match_line:
			warn_line = warn_match_line.group(1)
			warnings.append(files[-1] + ":" + warn_line + ": " + l)
		else:
			warnings.append(files[-1] + ": " + l)

	# State definitions
	STATE_NORMAL = 0
	STATE_SKIP = 1
	STATE_REPORT_ERROR = 2
	STATE_REPORT_WARNING = 3

	state = STATE_NORMAL

	# Use our own iterator instead of for loop
	log_iterator = log.__iter__()
	line_num = 0
	line = ""

	recycle_extra = False  # just in case

	while True:
		# first of all, see if we have a line to recycle (see heuristic for "l.<nn>" lines)
		if recycle_extra:
			line = extra
			#print ("Recycling line")
			recycle_extra = False
		else:
			# save previous line for "! File ended while scanning use of..." message
			prev_line = line
			try:
				if _ST3:
					line = log_iterator.__next__()  # will fail when no more lines
				else:
					line = log_iterator.next()
			except StopIteration:
				break
		line_num += 1
		# Now we deal with TeX's decision to truncate all log lines at 79 characters
		# If we find a line of exactly 79 characters, we add the subsequent line to it, and continue
		# until we find a line of less than 79 characters
		# The problem is that there may be a line of EXACTLY 79 chars. We keep our fingers crossed but also
		# use some heuristics to avoid disastrous consequences
		# We are inspired by latexmk (which has no heuristics, though)

		# HEURISTIC: the first line is always long, and we don't care about it
		# also, the **<file name> line may be long, but we skip it, too (to avoid edge cases)
		if line_num > 1 and len(line) >= 79 and line[0:2] != "**":
			# print ("Line %d is %d characters long; last char is %s" % (line_num, len(line), line[-1]))
			# HEURISTICS HERE
			extend_line = True
			recycle_extra = False
			while extend_line:
				try:
					if _ST3:
						extra = log_iterator.__next__()
					else:
						extra = log_iterator.next()
					line_num += 1  # for debugging purposes
					# HEURISTIC: if extra line begins with "Package:" "File:" "Document Class:",
					# or other "well-known markers",
					# we just had a long file name, so do not add
					if len(extra) > 0 and \
					   (extra[0:5] == "File:" or extra[0:8] == "Package:" or extra[0:15] == "Document Class:") or \
					   (extra[0:9] == "LaTeX2e <") or assignment_rx.match(extra):
						extend_line = False
						# no need to recycle extra, as it's nothing we are interested in
					# HEURISTIC: when TeX reports an error, it prints some surrounding text
					# and may use the whole line. Then it prints "...", and "l.<nn> <text>" on a new line
					# If so, do not extend
					elif line[-3:] == "..." and line_rx.match(extra):  # a bit inefficient as we match twice
						#print ("Found l. <nn> regex")
						extend_line = False
						recycle_extra = True  # make sure we process the "l.<nn>" line!
					else:
						line += extra
						if len(extra) < 79:
							extend_line = False
				except StopIteration:
					extend_line = False  # end of file, so we must be done. This shouldn't happen, btw
		# Check various states
		if state == STATE_SKIP:
			state = STATE_NORMAL
			continue
		if state == STATE_REPORT_ERROR:
			# skip everything except "l.<nn> <text>"
			print (line)
			err_match = line_rx.match(line)
			if not err_match:
				continue
			# now we match!
			state = STATE_NORMAL
			err_line = err_match.group(1)
			err_text = err_match.group(2)
			# err_msg is set from last time
			errors.append(files[-1] + ":" + err_line + ": " + err_msg + " [" + err_text + "]")
			continue
		if state == STATE_REPORT_WARNING:
			# add current line and check if we are done or not
			current_warning += line
			if line[-1] == '.':
				handle_warning(current_warning)
				current_warning = None
				state = STATE_NORMAL # otherwise the state stays at REPORT_WARNING
			continue
		if line == "":
			continue
		# Remove matched parentheses: they do not add new files to the stack
		# Do this iteratatively; for instance, on Windows 64, miktex has some files in
		# "Program Files (x86)", which wreaks havoc
		# NOTE: this means that your file names CANNOT have parentheses!!!
		while True:
			line_purged = matched_parens_rx.sub("", line)
			# if line != line_purged:
				# print ("Purged parens on line %d:" % (line_num, ))
				# print (line)
				# print (line_purged)
			if line != line_purged:
				line = line_purged
			else:
				break
		# Special error reporting for e.g. \footnote{text NO MATCHING PARENS & co
		if "! File ended while scanning use of" in line:
			scanned_command = line[35:-2]  # skip space and period at end
			# we may be unable to report a file by popping it, so HACK HACK HACK
			if _ST3:
				file_name = log_iterator.__next__()  # <inserted text>
				file_name = log_iterator.__next__()  # \par
				file_name = log_iterator.__next__()[3:]  # here is the file name with <*> in front
			else:
				file_name = log_iterator.next()  # <inserted text>
				file_name = log_iterator.next()  # \par
				file_name = log_iterator.next()[3:]  # here is the file name with <*> in front
			errors.append("TeX STOPPED: " + line[2:-2] + prev_line[:-5])
			errors.append("TeX reports the error was in file:" + file_name)
			continue
		# Here, make sure there was no uncaught error, in which case we do more special processing
		if "!  ==> Fatal error occurred, no output" in line:
			if errors == []:
				errors.append("TeX STOPPED: fatal errors occurred but LyTeXTools did not see them")
				errors.append("Check the TeX log file, and please let me know via GitHub. Thanks!")
			continue
		if "! Emergency stop." in line:
			state = STATE_SKIP
			continue
		# catch over/underfull
		# skip everything for now
		# Over/underfull messages end with [] so look for that
		if line[0:8] in ["Overfull", "Underfull"]:
			if line[-2:] == "[]":  # one-line over/underfull message
				continue
			ou_processing = True
			while ou_processing:
				try:
					if _ST3:
						line = log_iterator.__next__()  # will fail when no more lines
					else:
						line = log_iterator.next()  # will fail when no more lines
				except StopIteration:
					break
				line_num += 1
				if len(line) > 0 and line[0:3] == " []":
					ou_processing = False
			if ou_processing:
				errors.append("Malformed LOG file: over/underfull")
				break
			else:
				continue
		line.strip()  # get rid of initial spaces
		# note: in the next line, and also when we check for "!", we use the fact that "and" short-circuits
		while len(line)>0 and line[0] == ')':  # denotes end of processing of current file: pop it from stack
			# files.pop()
			if DEBUG:
				print (" " * len(files) + files[-1] + " (%d)" % (line_num,))
			if files:
				files.pop()
			else:
				errors.append("LyTeXTools cannot correctly detect file names in this LOG file.")
				errors.append("Please let me know via GitHub. Thanks!")
				if DEBUG:
					print ("Popping inexistent files")
				break
			line = line[1:]  # lather, rinse, repeat
		line.strip()  # again, to make sure there is no ") (filename" pattern
		file_match = file_rx.search(line)  # search matches everywhere, not just at the beginning of a line
		if file_match:
			file_name = file_match.group(1)
			# remove quotes
			if file_name[0] == "\"" and file_name[-1] == "\"":
				file_name = file_name[1:-1]
			files.append(file_name)
			if DEBUG:
				print (" " * len(files) + files[-1] + " (%d)" % (line_num,))
		if len(line) > 0 and line[0] == '!':  # Now it's surely an error
			print (line)
			err_msg = line[2:]  # skip "! "
			# next time around, err_msg will be set and we'll extract all info
			state = STATE_REPORT_ERROR
			continue
		warning_match = warning_rx.match(line)
		if warning_match:
			# if last character is a dot, it's a single line
			if line[-1] == '.':
				handle_warning(line)
				continue
			# otherwise, accumulate it
			current_warning = line
			state = STATE_REPORT_WARNING
			continue

	return (errors, warnings)


# First, define thread class for async processing

class CmdThread(threading.Thread):

	# Use __init__ to pass things we need
	# in particular, we pass the caller in teh main thread, so we can display stuff!
	def __init__(self, caller):
		self.caller = caller
		threading.Thread.__init__(self)

	def run(self):
		print ("Welcome to thread " + self.getName())

		self.caller.file_path = self.caller.file_name.replace(os.path.basename(self.caller.file_name), "")
		# Obtain filename extension
		self.caller.split_file_name = os.path.splitext(self.caller.file_name)
		print ("self.caller.split_file_name = ", self.caller.split_file_name)
		print ("Filename extension is: " + str(self.caller.split_file_name[1]))

		cmd_el = make_cmd_str = ""
		for cmd_el in self.caller.make_cmd:
		 	make_cmd_str = make_cmd_str + cmd_el + ' '
		# TeX Root filename has .tex extension: lilypond-book not to be invoked
		if self.caller.split_file_name[1].upper() in ('.TEX'):
			cmd = '"' + os.path.join(lytex_path, "latexcmd.sh") + '" "' + self.caller.file_path + '" "' + "latexmk -cd -e '\$pdflatex=q/pdflatex -interaction=nonstopmode -synctex=1 %S %O/' -g -pdf" + '" "' + self.caller.split_file_name[0] + '"'
			self.caller.output("[Compiling " + self.caller.file_name + "]")
			if DEBUG:
				print (cmd.encode('UTF-8'))
		# TeX Root filename has .lytex extension: lilypond-book to be invoked
		else:
			cmd = '"' + os.path.join(lytex_path, "lytexcmd.sh") + '" "' + self.caller.file_path + '" "' + "latexmk -cd -e '\$pdflatex=q/pdflatex -interaction=nonstopmode -synctex=1 %S %O/' -g -pdf" + '" "' + self.caller.split_file_name[0] + '"'
			self.caller.output("[Compiling " + self.caller.file_name + "]")
			if DEBUG:
				print (cmd.encode('UTF-8'))

		# Initialize log file object.
		data = open(self.caller.tex_base + ".log", 'w').truncate(0)

		# Handle path; copied from exec.py
		if self.caller.path:
			old_path = os.environ["PATH"]
			# The user decides in the build system  whether he wants to append $PATH
			# or tuck it at the front: "$PATH;C:\\new\\path", "C:\\new\\path;$PATH"
		# Handle differently in Python 2 and 3, to be safe:
		if not _ST3:
			os.environ["PATH"] = os.path.expandvars(self.caller.path).encode(sys.getfilesystemencoding())
		else:
			os.environ["PATH"] = os.path.expandvars(self.caller.path)
		if platform.system() == "Windows":
			# make sure console does not come up
			startupinfo = subprocess.STARTUPINFO()
			startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
			proc = subprocess.Popen(cmd, startupinfo=startupinfo)
		else:
			proc = subprocess.Popen(cmd, shell=True)
			# The folowing hackwork should block the thread until the cmd is completed.
			time.sleep(2)
			print("LyTeXTools start: ", time.gmtime())
			proc = subprocess.Popen(os.path.join(lytex_path, "polltexmk.sh"))
			proc.wait()
			print("LyTeXTools finish: ", time.gmtime())

		# restore path if needed
		if self.caller.path:
			os.environ["PATH"] = old_path

		# Handle killing
		# First, save process handle into caller; then communicate (which blocks)
		self.caller.proc = proc
		# out, err = proc.communicate()
		proc.wait()  # TODO: if needed, must use tempfiles instead of stdout/err

		# if DEBUG:
		#   self.caller.output(out)

		# Here the process terminated, but it may have been killed. If so, do not process log file.
		# Since we set self.caller.proc above, if it is None, the process must have been killed.
		# TODO: clean up?
		if not self.caller.proc:
			print (proc.returncode)
			self.caller.output("\n\n[User terminated compilation process]\n")
			self.caller.finish(False)   # We kill, so won't switch to PDF anyway
			return
		# Here we are done cleanly:
		self.caller.proc = None
		print ("Finished normally")
		print (proc.returncode)

		# Now, first check for a lilypond-book failure. The cmd stores its exit code in a .lytexerr.log file. Key to values:
		# lil: lilypond error
		# lat: LaTeX error
		# ok:  No error
		content = ["", ""]
		try:
			lilyerr = open(".lytexerr.log", 'r').read()
			print ("command line error code (lil/lat/ok) =", lilyerr)
		except:
			content.append("The error-checking subsystem of LyTeXTools is malfunctioning. This does not need to be fatal but we are aborting to avoid possibly misleading error messages.")
			return

		# If lilypond-book exited with a non-zero error code, terminate the build.
		if (lilyerr == "lil\n"):
			content.append("lilypond-book produced an error. Also, as a result, LaTeX did not run at all. Check out lilypond-book.log for more specific information.")
			self.caller.output(content)
			self.caller.output("\n\n[Failed...]\n")
			return

		# Now we can be sure that in the worst case we only have LaTeX errors.

		# this is a conundrum. We used (ST1) to open in binary mode ('rb') to avoid
		# issues, but maybe we just need to decode?
		# OK, this seems solid: first we decode using the self.caller.encoding,
		# then we reencode using the default locale's encoding.
		# Note: we get this using ST2's own getdefaultencoding(), not the locale module
		# We ignore bad chars in both cases.

		# CHANGED 12/10/19: use platform encoding (self.caller.encoding), then
		# keep it that way!

		# CHANGED 12-10-27. OK, here's the deal. We must open in binary mode on Windows
		# because silly MiKTeX inserts ASCII control characters in over/underfull warnings.
		# In particular it inserts EOFs, which stop reading altogether; reading in binary
		# prevents that. However, that's not the whole story: if a FS character is encountered,
		# AND if we invoke splitlines on a STRING, it sadly breaks the line in two. This messes up
		# line numbers in error reports. If, on the other hand, we invoke splitlines on a
		# byte array (? whatever read() returns), this does not happen---we only break at \n, etc.
		# However, we must still decode the resulting lines using the relevant encoding.
		# 121101 -- moved splitting and decoding logic to parseTeXlog, where it belongs.

		# Note to self: need to think whether we don't want to codecs.open this, too...
		errors = []
		warnings = []

		try:
			if _ST3:
				data = open(self.caller.tex_base + ".log", 'r') \
				.read().splitlines()
			else:
				data = open(self.caller.tex_base + ".log", 'r') \
				.read().decode(self.caller.encoding, 'ignore') \
				.encode(sublime_plugin.sys.getdefaultencoding(), 'ignore').splitlines()
		except:
			content.append("Critical error with the LaTeX log.")
			print (sys.exc_info())
			self.caller.output(content)
			self.caller.output("\n\n[Failed...]\n")
			return

		(errors, warnings) = parseTeXlog(data)
		if errors:
			content.append("There were errors in your LaTeX source")
			content.append("")
			content.extend(errors)
		else:
			content.append("Texification succeeded: no LaTeX errors!")
			content.append("")
		if warnings:
			if errors:
				content.append("")
				content.append("There were also warnings.")
			else:
				content.append("However, there were warnings in your LaTeX source")
			content.append("")
			content.extend(warnings)

		self.caller.output(content)
		self.caller.output("\n\n[Done!]\n")
		self.caller.finish(len(errors) == 0)


# Actual Command

# TODO latexmk stops if bib files can't be found (on subsequent compiles)
# I.e. it doesn't even start bibtex!
# If so, find out! Otherwise log file is never refreshed
# Work-around: check file creation times

# We get the texification command (cmd), file regex and path (TODO) from
# the sublime-build file. This allows us to use the ST2 magic: we can keep both
# windows and osx settings there, and we get handed the right one depending on
# the platform! Cool!

class make_pdfCommand(sublime_plugin.WindowCommand):
	def run(self, cmd="", file_regex="", path=""):

		# Try to handle killing
		if hasattr(self, 'proc') and self.proc:  # if we are running, try to kill running process
			self.output("\n\n### Got request to terminate compilation ###")
			self.proc.kill()
			self.proc = None
			return
		else:  # either it's the first time we run, or else we have no running processes
			self.proc = None

		view = self.window.active_view()
		# self.file_name = getTeXRoot.get_tex_root(view.file_name())
		self.file_name = getTeXRoot.get_tex_root(view)

		self.tex_base, self.tex_ext = os.path.splitext(self.file_name)
		# On OSX, change to file directory, or latexmk will spew stuff into root!
		tex_dir = os.path.dirname(self.file_name)

		# Extra paths
		self.path = path

		# Output panel: from exec.py
		if not hasattr(self, 'output_view'):
			self.output_view = self.window.get_output_panel("exec")

		# Dumb, but required for the moment for the output panel to be picked
		# up as the result buffer
		self.window.get_output_panel("exec")

		self.output_view.settings().set("result_file_regex", "^([^:\n\r]*):([0-9]+):?([0-9]+)?:? (.*)$")
		# self.output_view.settings().set("result_line_regex", line_regex)
		self.output_view.settings().set("result_base_dir", tex_dir)

		self.window.run_command("show_panel", {"panel": "output.exec"})

		# Get parameters from sublime-build file:
		self.make_cmd = cmd
		# I actually think self.file_name is it already
		self.engine = 'pdflatex' # Standard pdflatex
		for line in codecs.open(self.file_name, "r", "UTF-8", "ignore").readlines():
			if not line.startswith('%'):
				break
			else:
				# We have a comment match; check for a TS-program match
				mroot = re.match(r"%\s*!TEX\s+(?:TS-)?program *= *(xelatex|lualatex|pdflatex)\s*$",line)
				if mroot:
					# Sanity checks
					if "texify" == self.make_cmd[0]:
						sublime.error_message("Sorry, cannot select engine using a %!TEX program directive on MikTeX.")
						return
					if not ("$pdflatex = '%E" in self.make_cmd[3]):
						sublime.error_message("You are using a custom LaTeX.sublime-build file (in User maybe?). Cannot select engine using a %!TEX program directive.")
						return
					self.engine = mroot.group(1)
					break
		if self.engine != 'pdflatex': # Since pdflatex is standard, we do not output a msg. for it.
			self.output("Using engine " + self.engine + "\n")
		self.make_cmd[3] = self.make_cmd[3].replace("%E", self.engine)
		self.output_view.settings().set("result_file_regex", file_regex)

		if view.is_dirty():
			print ("saving...")
			view.run_command('save')  # call this on view, not self.window

		if self.tex_ext.upper() not in (".TEX", ".LYTEX"):
			sublime.error_message("%s is not a TeX source file. Cannot compile." % (os.path.basename(view.file_name()),))
			return

		s = platform.system()
		if s == "Darwin":
			self.encoding = "UTF-8"
		elif s == "Windows":
			self.encoding = getOEMCP()
		elif s == "Linux":
			self.encoding = "UTF-8"
		else:
			sublime.error_message("Platform as yet unsupported. Sorry!")
			return
		# print (self.make_cmd + [self.file_name])

		os.chdir(tex_dir)
		CmdThread(self).start()
		print (threading.active_count())

	# Threading headaches :-)
	# The following function is what gets called from CmdThread; in turn,
	# this spawns append_data, but on the main thread.

	def output(self, data):
		sublime.set_timeout(functools.partial(self.do_output, data), 0)

	def do_output(self, data):
		# if proc != self.proc:
		#     # a second call to exec has been made before the first one
		#     # finished, ignore it instead of intermingling the output.
		#     if proc:
		#         proc.kill()
		#     return

		# try:
		#     str = data.decode(self.encoding)
		# except:
		#     str = "[Decode error - output not " + self.encoding + "]"
		#     proc = None

		# decoding in thread, so we can pass coded and decoded data
		# handle both lists and strings
		# Need different handling for python 2 and 3
		if not _ST3:
			strdata = data if isinstance(data, types.StringTypes) else "\n".join(data)
		else:
			strdata = data if isinstance(data, str) else "\n".join(data)

		# Normalize newlines, Sublime Text always uses a single \n separator
		# in memory.
		strdata = strdata.replace('\r\n', '\n').replace('\r', '\n')

		selection_was_at_end = (len(self.output_view.sel()) == 1
			and self.output_view.sel()[0]
				== sublime.Region(self.output_view.size()))
		self.output_view.set_read_only(False)

		# Move this to a TextCommand for compatibility with ST3
		self.output_view.run_command("do_output_edit", {"data": strdata, "selection_was_at_end": selection_was_at_end})
		# edit = self.output_view.begin_edit()
		# self.output_view.insert(edit, self.output_view.size(), strdata)
		# if selection_was_at_end:
		#     self.output_view.show(self.output_view.size())
		# self.output_view.end_edit(edit)
		self.output_view.set_read_only(True)

	# Also from exec.py
	# Set the selection to the start of the output panel, so next_result works
	# Then run viewer

	def finish(self, can_switch_to_pdf):
		sublime.set_timeout(functools.partial(self.do_finish, can_switch_to_pdf), 0)

	def do_finish(self, can_switch_to_pdf):
		# Move to TextCommand for compatibility with ST3
		# edit = self.output_view.begin_edit()
		# self.output_view.sel().clear()
		# reg = sublime.Region(0)
		# self.output_view.sel().add(reg)
		# self.output_view.show(reg) # scroll to top
		# self.output_view.end_edit(edit)
		self.output_view.run_command("do_finish_edit")
		if can_switch_to_pdf:
			self.window.active_view().run_command("jump_to_pdf", {"from_keybinding": False})


class DoOutputEditCommand(sublime_plugin.TextCommand):
    def run(self, edit, data, selection_was_at_end):
        self.view.insert(edit, self.view.size(), data)
        if selection_was_at_end:
            self.view.show(self.view.size())

class DoFinishEditCommand(sublime_plugin.TextCommand):
    def run(self, edit):
        self.view.sel().clear()
        reg = sublime.Region(0)
        self.view.sel().add(reg)
        self.view.show(reg)