After upgrading to sublime 4, I am getting an issue with the log from a custom build system plugin (e.g. custom “target” in the project settings) getting reset with a single line:
shell_cmd or cmd is required
This is running cmd : None
This doesn’t seem to be coming from my build system, and it nukes the entire build log.
To clarify, my build system works, and starts logging, but some action seems to cause sublime to want to run a None command occasionally which is clearing the build log.
Can you share what the build system looks like so we can perhaps better diagnose what might be going wrong?
This is what it looks like in my project settings:
{
"shell": true,
"cmd": [
"Win64",
"Debug",
"configure",
"build",
"test",
"--no-incremental",
"--j",
"31",
],
"working_dir": "${project_path}/Project
"name": "Project Win64 Debug",
"target": "waf"
}
This is the full build system:
import collections
import functools
import html
import os
import subprocess
import sys
import threading
import time
import re
import sublime
import sublime_plugin
def UndoLowerCasePath(p):
tokens = p.split('\\')
result = tokens[0].upper()
idx = 1
while idx < len(tokens):
current = tokens[idx].lower()
result += os.sep
found = False
for fn in os.listdir(result):
if fn.lower() == current:
result += fn
found = True
break
if not found:
return p
idx += 1
if idx == len(tokens):
return result
class WafProcessListener(object):
def on_data(self, proc, data):
pass
def on_finished(self, proc):
pass
class WafAsyncProcess(object):
"""
Encapsulates subprocess.Popen, forwarding stdout to a supplied
ProcessListener (on a separate thread)
"""
def __init__(self, cmd, shell_cmd, env, listener, path="", shell=False):
""" "path" and "shell" are options in build systems """
if not shell_cmd and not cmd:
raise ValueError("shell_cmd or cmd is required")
if shell_cmd and not isinstance(shell_cmd, str):
raise ValueError("shell_cmd must be a string")
self.listener = listener
self.killed = False
self.start_time = time.time()
# Hide the console window on Windows
startupinfo = None
if os.name == "nt":
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
# Set temporary PATH to locate executable in cmd
if path:
old_path = os.environ["PATH"]
# The user decides in the build system whether he wants to append $PATH
# or tuck it at the front: "$PATH;C:\\new\\path", "C:\\new\\path;$PATH"
os.environ["PATH"] = os.path.expandvars(path)
proc_env = os.environ.copy()
proc_env.update(env)
# force colors on for waf
# proc_env['COLOR'] = '1'
for k, v in proc_env.items():
proc_env[k] = os.path.expandvars(v)
if shell_cmd and sys.platform == "win32":
# Use shell=True on Windows, so shell_cmd is passed through with the correct escaping
self.proc = subprocess.Popen(
shell_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
startupinfo=startupinfo,
env=proc_env,
shell=True)
elif shell_cmd and sys.platform == "darwin":
# Use a login shell on OSX, otherwise the users expected env vars won't be setup
self.proc = subprocess.Popen(
["/bin/bash", "-l", "-c", shell_cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
startupinfo=startupinfo,
env=proc_env,
shell=False)
elif shell_cmd and sys.platform == "linux":
# Explicitly use /bin/bash on Linux, to keep Linux and OSX as
# similar as possible. A login shell is explicitly not used for
# linux, as it's not required
self.proc = subprocess.Popen(
["/bin/bash", "-c", shell_cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
startupinfo=startupinfo,
env=proc_env,
shell=False)
else:
# Old style build system, just do what it asks
self.proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
startupinfo=startupinfo,
env=proc_env,
shell=shell)
if path:
os.environ["PATH"] = old_path
if self.proc.stdout:
threading.Thread(target=self.read_stdout).start()
if self.proc.stderr:
threading.Thread(target=self.read_stderr).start()
def kill(self):
if not self.killed:
self.killed = True
if sys.platform == "win32":
# terminate would not kill process opened by the shell cmd.exe,
# it will only kill cmd.exe leaving the child running
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
subprocess.Popen(
"taskkill /PID " + str(self.proc.pid),
startupinfo=startupinfo)
else:
self.proc.terminate()
self.listener = None
def poll(self):
return self.proc.poll() is None
def exit_code(self):
return self.proc.poll()
def read_stdout(self):
while True:
data = os.read(self.proc.stdout.fileno(), 2**15)
if len(data) > 0:
if self.listener:
self.listener.on_data(self, data)
else:
self.proc.stdout.close()
if self.listener:
self.listener.on_finished(self)
break
def read_stderr(self):
while True:
data = os.read(self.proc.stderr.fileno(), 2**15)
if len(data) > 0:
if self.listener:
self.listener.on_data(self, data)
else:
self.proc.stderr.close()
break
class WafCommand(sublime_plugin.WindowCommand, WafProcessListener):
BLOCK_SIZE = 2**14
text_queue = collections.deque()
text_queue_proc = None
text_queue_lock = threading.Lock()
proc = None
errs_by_file = {}
phantom_sets_by_buffer = {}
show_errors_inline = True
def run(
self,
cmd=None,
shell_cmd=None,
file_regex=R"^(.+)\((\d+)[,\d\-]*\): ",
line_regex="",
working_dir="",
encoding="utf-8",
env={},
quiet=False,
kill=False,
update_phantoms_only=False,
hide_phantoms_only=False,
word_wrap=True,
syntax="waflog.sublime-syntax",
# Catches "path" and "shell"
**kwargs):
if not cmd and not shell_cmd:
return;
self.file_regex_proc = re.compile(file_regex)
if update_phantoms_only:
if self.show_errors_inline:
self.update_phantoms()
return
if hide_phantoms_only:
self.hide_phantoms()
return
# clear the text_queue
self.text_queue_lock.acquire()
try:
self.text_queue.clear()
self.text_queue_proc = None
finally:
self.text_queue_lock.release()
# always kill the previous build
if self.proc:
self.proc.kill()
self.proc = None
self.append_string(None, "[Cancelled Previous Build]\n")
if kill:
return
if not hasattr(self, 'output_view'):
# Try not to call get_output_panel until the regexes are assigned
self.output_view = self.window.create_output_panel("exec")
# Default the to the current files directory if no working directory was given
if working_dir == "" and self.window.active_view() and self.window.active_view().file_name():
working_dir = os.path.dirname(self.window.active_view().file_name())
self.output_view.settings().set("result_file_regex", file_regex)
self.output_view.settings().set("result_line_regex", line_regex)
self.output_view.settings().set("result_base_dir", working_dir)
self.output_view.settings().set("word_wrap", word_wrap)
self.output_view.settings().set("line_numbers", False)
self.output_view.settings().set("gutter", False)
self.output_view.settings().set("scroll_past_end", False)
self.output_view.assign_syntax(syntax)
# Call create_output_panel a second time after assigning the above
# settings, so that it'll be picked up as a result buffer
self.window.create_output_panel("exec")
self.encoding = encoding
self.quiet = quiet
self.proc = None
if not self.quiet:
if shell_cmd:
print("Running " + shell_cmd)
elif cmd:
cmd_string = cmd
if not isinstance(cmd, str):
cmd_string = " ".join(cmd)
print("Running " + cmd_string)
sublime.status_message("Building")
show_panel_on_build = sublime.load_settings("Preferences.sublime-settings").get("show_panel_on_build", True)
if show_panel_on_build:
self.window.run_command("show_panel", {"panel": "output.exec"})
self.hide_phantoms()
self.show_errors_inline = sublime.load_settings("Preferences.sublime-settings").get("show_errors_inline", True)
merged_env = env.copy()
if self.window.active_view():
user_env = self.window.active_view().settings().get('build_env')
if user_env:
merged_env.update(user_env)
# Change to the working dir, rather than spawning the process with it,
# so that emitted working dir relative path names make sense
if working_dir != "":
os.chdir(working_dir)
self.debug_text = ""
if shell_cmd:
shell_cmd = os.environ['BUILDSYSTEM'] + ' ' + shell_cmd + ' localpostbuild'
self.debug_text += "[shell_cmd: " + shell_cmd + "]\n"
else:
cmd = os.environ['BUILDSYSTEM'].split() + cmd + ['localpostbuild']
self.debug_text += "[cmd: " + str(cmd) + "]\n"
self.debug_text += "[dir: " + str(os.getcwd()) + "]\n"
if "PATH" in merged_env:
self.debug_text += "[path: " + str(merged_env["PATH"]) + "]"
else:
self.debug_text += "[path: " + str(os.environ["PATH"]) + "]"
try:
# Forward kwargs to WafAsyncProcess
self.proc = WafAsyncProcess(cmd, shell_cmd, merged_env, self, **kwargs)
self.text_queue_lock.acquire()
try:
self.text_queue_proc = self.proc
finally:
self.text_queue_lock.release()
except Exception as e:
self.append_string(None, str(e) + "\n")
self.append_string(None, self.debug_text + "\n")
if not self.quiet:
self.append_string(None, "[Finished]")
def is_enabled(self, kill=False, **kwargs):
if kill:
return (self.proc is not None) and self.proc.poll()
else:
return True
def append_string(self, proc, instr):
self.text_queue_lock.acquire()
was_empty = False
try:
if proc != self.text_queue_proc:
# a second call to exec has been made before the first one
# finished, ignore it instead of intermingling the output.
if proc:
proc.kill()
return
if len(self.text_queue) == 0:
was_empty = True
self.text_queue.append("")
available = self.BLOCK_SIZE - len(self.text_queue[-1])
new_strings = []
for line in re.split('(\n)', instr):
if line != '\n':
results = self.file_regex_proc.match(line)
if results:
try:
fixed_path = UndoLowerCasePath(results.group(1))
new_strings.append(results.group(0).replace(results.group(1), fixed_path) + line[len(results.group(0)):])
except:
# timing errors can cause us to try and parse partial lines
new_strings.append(line + "|")
else:
new_strings.append(line)
else:
new_strings.append('\n')
new_str = ''.join(new_strings)
if len(new_str) < available:
cur = self.text_queue.pop()
self.text_queue.append(cur + new_str)
else:
self.text_queue.append(new_str)
finally:
self.text_queue_lock.release()
if was_empty:
sublime.set_timeout(self.service_text_queue, 0)
def service_text_queue(self):
self.text_queue_lock.acquire()
is_empty = False
try:
if len(self.text_queue) == 0:
# this can happen if a new build was started, which will clear
# the text_queue
return
characters = self.text_queue.popleft()
is_empty = (len(self.text_queue) == 0)
finally:
self.text_queue_lock.release()
self.output_view.run_command(
'append',
{'characters': characters, 'force': True, 'scroll_to_end': True})
if self.show_errors_inline and characters.find('\n') >= 0:
errs = self.output_view.find_all_results_with_text()
errs_by_file = {}
for file, line, column, text in errs:
if file not in errs_by_file:
errs_by_file[file] = []
errs_by_file[file].append((line, column, text))
self.errs_by_file = errs_by_file
self.update_phantoms()
if not is_empty:
sublime.set_timeout(self.service_text_queue, 1)
def finish(self, proc):
if not self.quiet:
elapsed = time.time() - proc.start_time
exit_code = proc.exit_code()
if exit_code == 0 or exit_code is None:
self.append_string(proc, "[Finished in %.1fs]" % elapsed)
else:
self.append_string(proc, "[Finished in %.1fs with exit code %d]\n" % (elapsed, exit_code))
self.append_string(proc, self.debug_text)
if proc != self.proc:
return
self.proc = None
errs = self.output_view.find_all_results()
if len(errs) == 0:
sublime.status_message("Build finished")
else:
sublime.status_message("Build finished with %d errors" % len(errs))
def on_data(self, proc, data):
try:
characters = data.decode(self.encoding)
except:
characters = "[Decode error - output not " + self.encoding + "]\n"
proc = None
# Normalize newlines, Sublime Text always uses a single \n separator
# in memory.
characters = characters.replace('\r\n', '\n').replace('\r', '\n')
self.append_string(proc, characters)
def on_finished(self, proc):
sublime.set_timeout(functools.partial(self.finish, proc), 0)
def update_phantoms(self):
stylesheet = '''
<style>
div.error-arrow {
border-top: 0.4rem solid transparent;
border-left: 0.5rem solid color(var(--redish) blend(var(--background) 30%));
width: 0;
height: 0;
}
div.error {
padding: 0.4rem 0 0.4rem 0.7rem;
margin: 0 0 0.2rem;
border-radius: 0 0.2rem 0.2rem 0.2rem;
}
div.error span.message {
padding-right: 0.7rem;
}
div.error a {
text-decoration: inherit;
padding: 0.35rem 0.7rem 0.45rem 0.8rem;
position: relative;
bottom: 0.05rem;
border-radius: 0 0.2rem 0.2rem 0;
font-weight: bold;
}
html.dark div.error a {
background-color: #00000018;
}
html.light div.error a {
background-color: #ffffff18;
}
</style>
'''
for file, errs in self.errs_by_file.items():
view = self.window.find_open_file(file)
if view:
buffer_id = view.buffer_id()
if buffer_id not in self.phantom_sets_by_buffer:
phantom_set = sublime.PhantomSet(view, "exec")
self.phantom_sets_by_buffer[buffer_id] = phantom_set
else:
phantom_set = self.phantom_sets_by_buffer[buffer_id]
phantoms = []
errs_by_line = {}
for line, column, text in errs:
try:
if not errs_by_line.contains((column, text)):
errs_by_line[line].append((column, text))
except:
errs_by_line[line] = [(column, text)]
for line, line_errs in errs_by_line.items():
pt = view.text_point(line - 1, line_errs[0][0] - 1)
combined_message = '<br>'.join([html.escape(text, quote=False) for _, text in line_errs])
phantoms.append(sublime.Phantom(
sublime.Region(pt, view.line(pt).b),
('<body id=inline-error>' + stylesheet +
('<div class="error-arrow"></div>' if line_errs[0] != 0 else '') +
'<div class="error">' +
'<span class="message">' + combined_message + '</span>' +
'<a href=hide>' + chr(0x00D7) + '</a></div>' +
'</body>'),
sublime.LAYOUT_BELOW,
on_navigate=self.on_phantom_navigate))
phantom_set.update(phantoms)
def hide_phantoms(self):
for file, errs in self.errs_by_file.items():
view = self.window.find_open_file(file)
if view:
view.erase_phantoms("exec")
self.errs_by_file = {}
self.phantom_sets_by_buffer = {}
self.show_errors_inline = False
def on_phantom_navigate(self, url):
self.hide_phantoms()
class ExecEventListener(sublime_plugin.EventListener):
def on_load(self, view):
w = view.window()
if w is not None:
w.run_command('exec')
I believe I started with some built-in build system and hacked away at it, to undo some stuff making sublime not work (like all paths are getting converted to lower case causing the error links to not work)
%YAML 1.2
---
name: waflog
file_extensions: []
first_line_match: ^Build system .*
scope: text.waflog
contexts:
main:
- match: ^(\[\d+/\d+\] \(C.?\)?) ([\w ]+)([:]) (.*)( -> )(.*)
captures:
1: string.waflog
2: entity.name.waflog
3: comment.waflog
4: variable.function.waflog
5: comment.waflog
6: constant.language.waflog
- match: ^(\[\d+/\d+\]) ([\w ]+)([:]) (.*)( -> )(.*)
captures:
1: string.waflog
2: entity.name.waflog
3: comment.waflog
4: variable.function.waflog
5: comment.waflog
6: constant.language.waflog
- match: ^(.+)\((\d+[,\-\d]*)(\):)( error .*)
captures:
1: markup.underline.link.waflog
2: support.constant.waflog
3: text.waflog
4: message.error.waflog
- match: ^(.+)\((\d+[,\-\d]*)(\):)( warning .*)
captures:
1: markup.underline.link.waflog
2: support.constant.waflog
3: text.waflog
4: string.waflog
- match: ^(.+)\((\d+[,\-\d]*)(\):)( note[:] .*)
captures:
1: markup.underline.link.waflog
2: support.constant.waflog
3: text.waflog
4: comment.waflog
- match: .* fatal error .*
scope: invalid.waflog
- match: ^waf:info .*
scope: comment.waflog
- match: ^ .*
scope: comment.waflog
- match: ^waf:warning .*
scope: string.waflog
- match: Linker[:] .*
scope: string.waflog
- match: ^waf:error .*
scope: message.error.waflog
- match: .*[:] warning .*
scope: string.waflog
- match: ^(.*)(error .*)
captures:
1: entity.name.waflog
2: message.error.waflog
scope: keyword.waflog
- match: .*ERROR[:] Exception in.*
scope: keyword.waflog
Also the associated syntax file if that is necessary
It seems to nuke the log after double-clicking an error link.
edit: more specifically, double-clicking on an error link where the file is not open already.
I have added debugging and such to the “shell_cmd or cmd is required” line and it doesn’t seem to be coming from my build system (e.g. removing it or changing the text does nothing)
Also if it helps this is a C++ project, it seems like there may be something trying to invoke a build system that doesn’t exist, perhaps to try and get contextual complete info
This is interesting; does removing the default value for file_regex stop it from doing that?
I am not sure what you mean by this:
This is interesting; does removing the default value for
file_regexstop it from doing that?
I have no default file_regex in the project settings.
If I remove it in the plugin no error links are clickable so it won’t repro.
Also if I remove it from the plugin and put it back in the project settings, it still repros.
I meant that your sublime-build file doesn’t specify the option, but your plugin has a value that’s different from the default of "".
Your problem is most likely related to this:
class ExecEventListener(sublime_plugin.EventListener):
def on_load(self, view):
w = view.window()
if w is not None:
w.run_command('exec')
Overall, your plugin is a copy of the Default/exec.py from a prior build of Sublime Text, which provides the default exec command used in build systems. In use, the file_regex marks results in the built output as special regions that can be opened with a click.
The mechanism that does that is in the core, and just opens the file by the name captured in the regex.
In the core Default/exec.py of ST3 builds is this:
class ExecEventListener(sublime_plugin.EventListener):
def on_load(self, view):
w = view.window()
if w is not None:
w.run_command('exec', {'update_phantoms_only': True})
That is, there’s an event listener that says that if a file is loaded, please ask the exec command to update phantoms. The effect of that is that when you open a file that had an error in it, the exec command will use the information captured from the recent build to add error phantoms into the file that was just loaded, allowing you to see the errors.
In your version, it’s just invoking exec with no arguments, which is the same as a build system that is just { }; that is, there’s no shell_cmd or cmd, so it gets mad and aborts.
The reason the build log gets cleared is this line in WafCommand:
if not hasattr(self, 'output_view'):
# Try not to call get_output_panel until the regexes are assigned
self.output_view = self.window.create_output_panel("exec")
That is, it’s creating a panel by the same name as the one that the internal exec command uses, and by executing exec in the listener you’re effectively trying to start a new build, which clears the panel and then promptly fails.
The WafCommand in your plugin is a duplicate of exec (with probably some changes; I didn’t do any diffs), so as a first step I would alter the listener in your version to call waf instead of exec and pass in the same arguments that the default used to pass:
class WafEventListener(sublime_plugin.EventListener):
def on_load(self, view):
w = view.window()
if w is not None:
w.run_command('waf', {'update_phantoms_only': True})
In ST4, exec uses annotations instead of phantoms, so the exec command has replaced update_phantoms_only with update_annotations_only.
I also renamed ExecEventListener to WafEventListener, which is likely not required, but it does make it clearer what’s supposed to happen.
The exec command has undergone changes in ST4 (for example, it will kill an existing build with a new one starts instead of orphaning it in the background), so it may be interesting at some point to update what you have here to follow the new version.
One last thing which may or may not matter is that your build specifies target, but it doesn’t specify cancel, which means that it can’t cancel a running build. You can add:
"cancel": { "kill": true },
To your sublime-build if you’d like that functionality.