I have a Django view that runs a slow script server-side, and streams the script output to Javascript. This is the bit of code that runs the script and turns the output into a stream of events:
def stream_output(proc):
'''
Take a subprocess.Popen object and generate its output, line by line,
annotated with "stdout" or "stderr". At process termination it generates
one last element: ("result", return_code) with the return code of the
process.
'''
fds = [proc.stdout, proc.stderr]
bufs = [b"", b""]
types = ["stdout", "stderr"]
# Set both pipes as non-blocking
for fd in fds:
fcntl.fcntl(fd, fcntl.F_SETFL, os.O_NONBLOCK)
# Multiplex stdout and stderr with different prefixes
while len(fds) > 0:
s = select.select(fds, (), ())
for fd in s[0]:
idx = fds.index(fd)
buf = fd.read()
if len(buf) == 0:
fds.pop(idx)
if len(bufs[idx]) != 0:
yield types[idx], bufs.pop(idx)
types.pop(idx)
else:
bufs[idx] += buf
lines = bufs[idx].split(b"\n")
bufs[idx] = lines.pop()
for l in lines:
yield types[idx], l
res = proc.wait()
yield "result", res
I used to just serialize its output and stream it to JavaScript, then monitor
onreadystatechange
on the XMLHttpRequest
object browser-side, but then it
started failing on Chrome, which won't trigger onreadystatechange
until
something like a kilobyte of data has been received.
I didn't want to stream a kilobyte of padding just to work-around this, so it was time to try out Server-sent events. See also this.
This is the Django view that sends the events:
class HookRun(View):
def get(self, request):
proc = run_script(request)
def make_events():
for evtype, data in utils.stream_output(proc):
if evtype == "result":
yield "event: {}\ndata: {}\n\n".format(evtype, data)
else:
yield "event: {}\ndata: {}\n\n".format(evtype, data.decode("utf-8", "replace"))
return http.StreamingHttpResponse(make_events(), content_type='text/event-stream')
@method_decorator(never_cache)
def dispatch(self, *args, **kwargs):
return super().dispatch(*args, **kwargs)
And this is the template that renders it:
{% extends "base.html" %}
{% load i18n %}
{% block head_resources %}
{{block.super}}
<style type="text/css">
.out {
font-family: monospace;
padding: 0;
margin: 0;
}
.stdout {}
.stderr { color: red; }
.result {}
.ok { color: green; }
.ko { color: red; }
</style>
{# Polyfill for IE, typical... https://github.com/remy/polyfills/blob/master/EventSource.js #}
<script src="{{ STATIC_URL }}js/EventSource.js"></script>
<script type="text/javascript">
$(function() {
// Manage spinners and other ajax-related feedback
$(document).nav();
$(document).nav("ajax_start");
var out = $("#output");
var event_source = new EventSource("{% url 'session_hookrun' name=name %}");
event_source.addEventListener("open", function(e) {
//console.log("EventSource open:", arguments);
});
event_source.addEventListener("stdout", function(e) {
out.append($("<p>").attr("class", "out stdout").text(e.data));
});
event_source.addEventListener("stderr", function(e) {
out.append($("<p>").attr("class", "out stderr").text(e.data));
});
event_source.addEventListener("result", function(e) {
if (+e.data == 0)
out.append($("<p>").attr("class", "result ok").text("{% trans 'Success' %}"));
else
out.append($("<p>").attr("class", "result ko").text("{% trans 'Script failed with code' %} " + e.data));
event_source.close();
$(document).nav("ajax_end");
});
event_source.addEventListener("error", function(e) {
// There is an annoyance here: e does not contain any kind of error
// message.
out.append($("<p>").attr("class", "result ko").text("{% trans 'Error receiving script output from the server' %}"));
console.error("EventSource error:", arguments);
event_source.close();
$(document).nav("ajax_end");
});
});
</script>
{% endblock %}
{% block content %}
<h1>{% trans "Processing..." %}</h1>
<div id="output">
</div>
{% endblock %}
It's simple enough, it seems reasonably well supported besides needing a polyfill for IE and, astonishingly, it even works!