diff --git a/Project.toml b/Project.toml index 6c1b6642..b9cc5e8d 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "1.23.2" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +IJuliaCore = "ccee42ee-1239-4771-b50a-e8cbc7e05233" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -22,6 +23,7 @@ ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1" [compat] Conda = "1" +IJuliaCore = "1" JSON = "0.18,0.19,0.20,0.21,1" MbedTLS = "0.5,0.6,0.7,1" SoftGlobalScope = "1" diff --git a/src/IJulia.jl b/src/IJulia.jl index 4d5a4469..e85cb932 100644 --- a/src/IJulia.jl +++ b/src/IJulia.jl @@ -33,7 +33,9 @@ The `IJulia` module is used in three ways module IJulia export notebook, jupyterlab, installkernel -using ZMQ, JSON, SoftGlobalScope +using ZMQ, JSON, SoftGlobalScope, IJuliaCore +using IJuliaCore: @vprintln, orig_stdin, orig_stdout, orig_stderr, display_dict, + register_mime, register_jsonmime import Base.invokelatest import Dates using Dates: now @@ -49,23 +51,7 @@ const depfile = joinpath(dirname(@__FILE__), "..", "deps", "deps.jl") isfile(depfile) || error("IJulia not properly installed. Please run Pkg.build(\"IJulia\")") include(depfile) # generated by Pkg.build("IJulia") -####################################################################### -# Debugging IJulia - -# in the Jupyter front-end, enable verbose output via IJulia.set_verbose() -verbose = IJULIA_DEBUG -""" - set_verbose(v=true) - -This function enables (or disables, for `set_verbose(false)`) verbose -output from the IJulia kernel, when called within a running notebook. -This consists of log messages printed to the terminal window where -`jupyter` was launched, displaying information about every message sent -or received by the kernel. Used for debugging IJulia. -""" -function set_verbose(v::Bool=true) - global verbose = v -end +IJuliaCore.verbose = IJULIA_DEBUG """ `inited` is a global variable that is set to `true` if the IJulia @@ -75,16 +61,8 @@ whether you are in an IJulia notebook, therefore, you can check """ inited = false -# set this to false for debugging, to disable stderr redirection -""" -The IJulia kernel captures all [stdout and stderr](https://en.wikipedia.org/wiki/Standard_streams) -output and redirects it to the notebook. When debugging IJulia problems, -however, it can be more convenient to *not* capture stdout and stderr output -(since the notebook may not be functioning). This can be done by editing -`IJulia.jl` to set `capture_stderr` and/or `capture_stdout` to `false`. -""" -const capture_stdout = true -const capture_stderr = !IJULIA_DEBUG +# TODO Reenable something like this again +# const capture_stderr = !IJULIA_DEBUG set_current_module(m::Module) = current_module[] = m const current_module = Ref{Module}(Main) diff --git a/src/display.jl b/src/display.jl index a7da369e..fbf9aaba 100644 --- a/src/display.jl +++ b/src/display.jl @@ -1,128 +1,7 @@ - -# define our own method to avoid type piracy with Base.showable -_showable(a::AbstractVector{<:MIME}, x) = any(m -> showable(m, x), a) -_showable(m, x) = showable(m, x) - -""" -A vector of MIME types (or vectors of MIME types) that IJulia will try to -render. IJulia will try to render every MIME type specified in the first level -of the vector. If a vector of MIME types is specified, IJulia will include only -the first MIME type that is renderable (this allows for the expression of -priority and exclusion of redundant data). - -For example, since "text/plain" is specified as a first-child of the array, -IJulia will always try to include a "text/plain" representation of anything that -is displayed. Since markdown and html are specified within a sub-vector, IJulia -will always try to render "text/markdown", and will only try to render -"text/html" if markdown isn't possible. -""" -const ijulia_mime_types = Vector{Union{MIME, AbstractVector{MIME}}}([ - MIME("text/plain"), - MIME("image/svg+xml"), - [MIME("image/png"),MIME("image/jpeg")], - [ - MIME("text/markdown"), - MIME("text/html"), - ], - MIME("text/latex"), -]) - -""" -MIME types that when rendered (via stringmime) return JSON data. See -`ijulia_mime_types` for a description of how MIME types are selected. - -This is necessary to embed the JSON as is in the displaydata bundle (rather than -as stringify'd JSON). -""" -const ijulia_jsonmime_types = Vector{Union{MIME, Vector{MIME}}}([ - [[MIME("application/vnd.vegalite.v$n+json") for n in 4:-1:2]..., - [MIME("application/vnd.vega.v$n+json") for n in 5:-1:3]...], - MIME("application/vnd.dataresource+json"), MIME("application/vnd.plotly.v1+json") -]) - -register_mime(x::Union{MIME, Vector{MIME}})= push!(ijulia_mime_types, x) -register_mime(x::AbstractVector{<:MIME}) = push!(ijulia_mime_types, Vector{Mime}(x)) -register_jsonmime(x::Union{MIME, Vector{MIME}}) = push!(ijulia_jsonmime_types, x) -register_jsonmime(x::AbstractVector{<:MIME}) = push!(ijulia_jsonmime_types, Vector{Mime}(x)) - # return a String=>Any dictionary to attach as metadata # in Jupyter display_data and pyout messages metadata(x) = Dict() -""" -Generate the preferred MIME representation of x. - -Returns a tuple with the selected MIME type and the representation of the data -using that MIME type. -""" -function display_mimestring(mime_array::Vector{MIME}, x) - for m in mime_array - if _showable(m, x) - return display_mimestring(m, x) - end - end - error("No displayable MIME types in mime array.") -end - -display_mimestring(m::MIME, x) = (m, limitstringmime(m, x)) - -# text/plain output must have valid Unicode data to display in Jupyter -function display_mimestring(m::MIME"text/plain", x) - s = limitstringmime(m, x) - return m, (isvalid(s) ? s : "(binary data)") -end - -""" -Generate the preferred json-MIME representation of x. - -Returns a tuple with the selected MIME type and the representation of the data -using that MIME type (as a `JSONText`). -""" -function display_mimejson(mime_array::Vector{MIME}, x) - for m in mime_array - if _showable(m, x) - return display_mimejson(m, x) - end - end - error("No displayable MIME types in mime array.") -end - -display_mimejson(m::MIME, x) = (m, JSON.JSONText(limitstringmime(m, x))) - -""" -Generate a dictionary of `mime_type => data` pairs for all registered MIME -types. This is the format that Jupyter expects in display_data and -execute_result messages. -""" -function display_dict(x) - data = Dict{String, Union{String, JSONText}}() - for m in ijulia_mime_types - try - if _showable(m, x) - mime, mime_repr = display_mimestring(m, x) - data[string(mime)] = mime_repr - end - catch - if m == MIME("text/plain") - rethrow() # text/plain is required - end - end - end - - for m in ijulia_jsonmime_types - try - if _showable(m, x) - mime, mime_repr = display_mimejson(m, x) - data[string(mime)] = mime_repr - end - catch - end - end - - return data - -end - # queue of objects to display at end of cell execution const displayqueue = Any[] diff --git a/src/execute_request.jl b/src/execute_request.jl index d5548806..67e58f24 100644 --- a/src/execute_request.jl +++ b/src/execute_request.jl @@ -7,9 +7,6 @@ import Pkg # global variable so that display can be done in the correct Msg context execute_msg = Msg(["julia"], Dict("username"=>"jlkernel", "session"=>uuid4()), Dict()) -# global variable tracking the number of bytes written in the current execution -# request -const stdio_bytes = Ref(0) import REPL: helpmode diff --git a/src/init.jl b/src/init.jl index 3b6a038a..f1cd8e4c 100644 --- a/src/init.jl +++ b/src/init.jl @@ -6,9 +6,6 @@ const IJulia_RNG = seed!(Random.MersenneTwister(0)) import UUIDs uuid4() = string(UUIDs.uuid4(IJulia_RNG)) -const orig_stdin = Ref{IO}() -const orig_stdout = Ref{IO}() -const orig_stderr = Ref{IO}() const SOFTSCOPE = Ref{Bool}() function __init__() seed!(IJulia_RNG) @@ -25,8 +22,6 @@ const requests = Ref{Socket}() const control = Ref{Socket}() const heartbeat = Ref{Socket}() const profile = Dict{String,Any}() -const read_stdout = Ref{Base.PipeEndpoint}() -const read_stderr = Ref{Base.PipeEndpoint}() const socket_locks = Dict{Socket,ReentrantLock}() # similar to Pkg.REPLMode.MiniREPL, a minimal REPL-like emulator @@ -100,13 +95,13 @@ function init(args) start_heartbeat(heartbeat[]) if capture_stdout read_stdout[], = redirect_stdout() - redirect_stdout(IJuliaStdio(stdout,"stdout")) + redirect_stdout(IJuliaStdio(stdout,send_callback,"stdout")) end if capture_stderr read_stderr[], = redirect_stderr() - redirect_stderr(IJuliaStdio(stderr,"stderr")) + redirect_stderr(IJuliaStdio(stderr,send_callback,"stderr")) end - redirect_stdin(IJuliaStdio(stdin,"stdin")) + redirect_stdin(IJuliaStdio(stdin,send_callback,"stdin")) minirepl[] = MiniREPL(TextDisplay(stdout)) logger = Base.CoreLogging.SimpleLogger(Base.stderr) diff --git a/src/inline.jl b/src/inline.jl index a08baa80..f15bb15c 100644 --- a/src/inline.jl +++ b/src/inline.jl @@ -19,41 +19,6 @@ const ipy_mime = [ "application/javascript" ] -# need special handling for showing a string as a textmime -# type, since in that case the string is assumed to be -# raw data unless it is text/plain -israwtext(::MIME, x::AbstractString) = true -israwtext(::MIME"text/plain", x::AbstractString) = false -israwtext(::MIME, x) = false - -InlineIOContext(io, KVs::Pair...) = IOContext( - io, - :limit=>true, :color=>true, :jupyter=>true, - KVs... -) - -# convert x to a string of type mime, making sure to use an -# IOContext that tells the underlying show function to limit output -function limitstringmime(mime::MIME, x) - buf = IOBuffer() - if istextmime(mime) - if israwtext(mime, x) - return String(x) - else - show(InlineIOContext(buf), mime, x) - end - else - b64 = Base64EncodePipe(buf) - if isa(x, Vector{UInt8}) - write(b64, x) # x assumed to be raw binary data - else - show(InlineIOContext(b64), mime, x) - end - close(b64) - end - return String(take!(buf)) -end - for mime in ipy_mime @eval begin function display(d::InlineDisplay, ::MIME{Symbol($mime)}, x) diff --git a/src/kernel.jl b/src/kernel.jl index e6a548b5..01571006 100644 --- a/src/kernel.jl +++ b/src/kernel.jl @@ -35,7 +35,7 @@ pushdisplay(IJulia.InlineDisplay()) ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0) println(IJulia.orig_stdout[], "Starting kernel event loops.") -IJulia.watch_stdio() +IJulia.watch_stdio(send_callback) # workaround JuliaLang/julia#4259 delete!(task_local_storage(),:SOURCE_PATH) diff --git a/src/stdio.jl b/src/stdio.jl index 1eb82588..449492e3 100644 --- a/src/stdio.jl +++ b/src/stdio.jl @@ -1,278 +1,10 @@ -# IJulia redirects stdout and stderr into "stream" messages sent to the -# Jupyter front-end. - -# create a wrapper type around redirected stdio streams, -# both for overloading things like `flush` and so that we -# can set properties like `color`. -struct IJuliaStdio{IO_t <: IO} <: Base.AbstractPipe - io::IOContext{IO_t} -end -IJuliaStdio(io::IO, stream::AbstractString="unknown") = - IJuliaStdio{typeof(io)}(IOContext(io, :color=>Base.have_color, - :jupyter_stream=>stream, - :displaysize=>displaysize())) -Base.pipe_reader(io::IJuliaStdio) = io.io.io -Base.pipe_writer(io::IJuliaStdio) = io.io.io -Base.lock(io::IJuliaStdio) = lock(io.io.io) -Base.unlock(io::IJuliaStdio) = unlock(io.io.io) -Base.in(key_value::Pair, io::IJuliaStdio) = in(key_value, io.io) -Base.haskey(io::IJuliaStdio, key) = haskey(io.io, key) -Base.getindex(io::IJuliaStdio, key) = getindex(io.io, key) -Base.get(io::IJuliaStdio, key, default) = get(io.io, key, default) -Base.displaysize(io::IJuliaStdio) = displaysize(io.io) -Base.unwrapcontext(io::IJuliaStdio) = Base.unwrapcontext(io.io) -Base.setup_stdio(io::IJuliaStdio, readable::Bool) = Base.setup_stdio(io.io.io, readable) - -if VERSION < v"1.7.0-DEV.254" - for s in ("stdout", "stderr", "stdin") - f = Symbol("redirect_", s) - sq = QuoteNode(Symbol(s)) - @eval function Base.$f(io::IJuliaStdio) - io[:jupyter_stream] != $s && throw(ArgumentError(string("expecting ", $s, " stream"))) - Core.eval(Base, Expr(:(=), $sq, io)) - return io - end - end -end - -# logging in verbose mode goes to original stdio streams. Use macros -# so that we do not even evaluate the arguments in no-verbose modes - -using Printf -function get_log_preface() - t = now() - taskname = get(task_local_storage(), :IJulia_task, "") - @sprintf("%02d:%02d:%02d(%s): ", Dates.hour(t),Dates.minute(t),Dates.second(t),taskname) -end - -macro vprintln(x...) - quote - if verbose::Bool - println(orig_stdout[], get_log_preface(), $(map(esc, x)...)) - end - end -end - -macro verror_show(e, bt) - quote - if verbose::Bool - showerror(orig_stderr[], $(esc(e)), $(esc(bt))) - end - end -end - -#name=>iobuffer for each stream ("stdout","stderr") so they can be sent in flush -const bufs = Dict{String,IOBuffer}() -const stream_interval = 0.1 -# maximum number of bytes in libuv/os buffer before emptying -const max_bytes = 10*1024 -# max output per code cell is 512 kb by default -const max_output_per_request = Ref(1 << 19) - -""" -Continually read from (size limited) Libuv/OS buffer into an `IObuffer` to avoid problems when -the Libuv/OS buffer gets full (https://github.com/JuliaLang/julia/issues/8789). Send data immediately -when buffer contains more than `max_bytes` bytes. Otherwise, if data is available it will be sent every -`stream_interval` seconds (see the Timers set up in watch_stdio). Truncate the output to `max_output_per_request` -bytes per execution request since excessive output can bring browsers to a grinding halt. -""" -function watch_stream(rd::IO, name::AbstractString) - task_local_storage(:IJulia_task, "read $name task") - try - buf = IOBuffer() - bufs[name] = buf - while !eof(rd) # blocks until something is available - nb = bytesavailable(rd) - if nb > 0 - stdio_bytes[] += nb - # if this stream has surpassed the maximum output limit then ignore future bytes - if stdio_bytes[] >= max_output_per_request[] - read(rd, nb) # read from libuv/os buffer and discard - if stdio_bytes[] - nb < max_output_per_request[] - send_ipython(publish[], msg_pub(execute_msg, "stream", - Dict("name" => "stderr", "text" => "Excessive output truncated after $(stdio_bytes[]) bytes."))) - end - else - write(buf, read(rd, nb)) - end - end - if buf.size > 0 - if buf.size >= max_bytes - #send immediately - send_stream(name) - end - end - end - catch e - # the IPython manager may send us a SIGINT if the user - # chooses to interrupt the kernel; don't crash on this - if isa(e, InterruptException) - watch_stream(rd, name) - else - rethrow() - end - end -end - -function send_stdio(name) - if verbose::Bool && !haskey(task_local_storage(), :IJulia_task) - task_local_storage(:IJulia_task, "send $name task") - end - send_stream(name) -end - -send_stdout(t::Timer) = send_stdio("stdout") -send_stderr(t::Timer) = send_stdio("stderr") - -""" -Jupyter associates cells with message headers. Once a cell's execution state has -been set as to idle, it will silently drop stream messages (i.e. output to -stdout and stderr) - see https://github.com/jupyter/notebook/issues/518. -When using Interact, and a widget's state changes, a new -message header is sent to the IJulia kernel, and while Reactive -is updating Signal graph state, it's execution state is busy, meaning Jupyter -will not drop stream messages if Interact can set the header message under which -the stream messages will be sent. Hence the need for this function. -""" -function set_cur_msg(msg) - global execute_msg = msg -end - -function send_stream(name::AbstractString) - buf = bufs[name] - if buf.size > 0 - d = take!(buf) - n = num_utf8_trailing(d) - dextra = d[end-(n-1):end] - resize!(d, length(d) - n) - s = String(copy(d)) - if isvalid(String, s) - write(buf, dextra) # assume that the rest of the string will be written later - length(d) == 0 && return - else - # fallback: base64-encode non-UTF8 binary data - sbuf = IOBuffer() - print(sbuf, "base64 binary data: ") - b64 = Base64EncodePipe(sbuf) - write(b64, d) - write(b64, dextra) - close(b64) - print(sbuf, '\n') - s = String(take!(sbuf)) - end - send_ipython(publish[], - msg_pub(execute_msg, "stream", - Dict("name" => name, "text" => s))) - end -end - -""" -If `d` ends with an incomplete UTF8-encoded character, return the number of trailing incomplete bytes. -Otherwise, return `0`. -""" -function num_utf8_trailing(d::Vector{UInt8}) - i = length(d) - # find last non-continuation byte in d: - while i >= 1 && ((d[i] & 0xc0) == 0x80) - i -= 1 - end - i < 1 && return 0 - c = d[i] - # compute number of expected UTF-8 bytes starting at i: - n = c <= 0x7f ? 1 : c < 0xe0 ? 2 : c < 0xf0 ? 3 : 4 - nend = length(d) + 1 - i # num bytes from i to end - return nend == n ? 0 : nend -end - -""" - readprompt(prompt::AbstractString; password::Bool=false) - -Display the `prompt` string, request user input, -and return the string entered by the user. If `password` -is `true`, the user's input is not displayed during typing. -""" -function readprompt(prompt::AbstractString; password::Bool=false) - if !execute_msg.content["allow_stdin"] - error("IJulia: this front-end does not implement stdin") - end - send_ipython(raw_input[], - msg_reply(execute_msg, "input_request", - Dict("prompt"=>prompt, "password"=>password))) - while true - msg = recv_ipython(raw_input[]) - if msg.header["msg_type"] == "input_reply" - return msg.content["value"] - else - error("IJulia error: unknown stdin reply") - end - end -end - -# override prompts using julia#28038 in 0.7 -function check_prompt_streams(input::IJuliaStdio, output::IJuliaStdio) - if get(input,:jupyter_stream,"unknown") != "stdin" || - get(output,:jupyter_stream,"unknown") != "stdout" - throw(ArgumentError("prompt on IJulia stdio streams only works for stdin/stdout")) - end - end -function Base.prompt(input::IJuliaStdio, output::IJuliaStdio, message::AbstractString; default::AbstractString="") - check_prompt_streams(input, output) - val = chomp(readprompt(message * ": ")) - return isempty(val) ? default : val -end -function Base.getpass(input::IJuliaStdio, output::IJuliaStdio, message::AbstractString) - check_prompt_streams(input, output) - # fixme: should we do more to zero memory associated with the password? - # doing this properly might require working with the raw ZMQ message buffer here - return Base.SecretBuffer!(Vector{UInt8}(codeunits(readprompt(message * ": ", password=true)))) -end - -# IJulia issue #42: there doesn't seem to be a good way to make a task -# that blocks until there is a read request from STDIN ... this makes -# it very hard to properly redirect all reads from STDIN to pyin messages. -# In the meantime, however, we can just hack it so that readline works: -import Base.readline -function readline(io::IJuliaStdio) - if get(io,:jupyter_stream,"unknown") == "stdin" - return readprompt("stdin> ") - else - readline(io.io) - end -end - -function watch_stdio() - task_local_storage(:IJulia_task, "init task") - if capture_stdout - read_task = @async watch_stream(read_stdout[], "stdout") - #send stdout stream msgs every stream_interval secs (if there is output to send) - Timer(send_stdout, stream_interval, interval=stream_interval) - end - if capture_stderr - readerr_task = @async watch_stream(read_stderr[], "stderr") - #send STDERR stream msgs every stream_interval secs (if there is output to send) - Timer(send_stderr, stream_interval, interval=stream_interval) - end -end - -function flush_all() - flush_cstdio() # flush writes to stdout/stderr by external C code - flush(stdout) - flush(stderr) -end - -function oslibuv_flush() - #refs: https://github.com/JuliaLang/IJulia.jl/issues/347#issuecomment-144505862 - # https://github.com/JuliaLang/IJulia.jl/issues/347#issuecomment-144605024 - @static if Sys.iswindows() - ccall(:SwitchToThread, stdcall, Cvoid, ()) - end - yield() - yield() -end - -import Base.flush -function flush(io::IJuliaStdio) - flush(io.io) - oslibuv_flush() - send_stream(get(io,:jupyter_stream,"unknown")) +function send_callback(name, data) + send_ipython( + publish[], + msg_pub( + execute_msg, + "stream", + Dict("name" => name, "text" => data) + ) + ) end