change precompile strategy to FakePTYs (#305)

* change precompile strategy to FakePTYs

* disable precompile on windows

Project.toml

@@ -9,14 +9,12 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
-TerminalRegressionTests = "98bfdc55-cc95-5876-a49a-74609291cbe0"
 Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624"
 
 [compat]
 Crayons = "1, 2, 3, 4"
 JLFzf = "^0.1.1"
 Tokenize = "0.5"
-TerminalRegressionTests = "0.2"
 julia = "1.6"
 
 [extras]

src/FakePTYs.jl

@@ -0,0 +1,64 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+module FakePTYs
+
+if Sys.iswindows()
+    pushfirst!(LOAD_PATH, Sys.STDLIB)
+    using REPL.Sockets
+    popfirst!(LOAD_PATH)
+end
+
+
+function open_fake_pty()
+    @static if Sys.iswindows()
+        # Fake being cygwin
+        pid = string(getpid(), base=16, pad=16)
+        pipename = """\\\\?\\pipe\\cygwin-$pid-pty10-abcdefg"""
+        server = listen(pipename)
+        pts = connect(pipename)
+        @assert ccall(:jl_ispty, Cint, (Ptr{Cvoid},), pts.handle) == 1
+        ptm = accept(server)
+        close(server)
+        # extract just the file descriptor
+        fds = Libc.dup(Base._fd(pts))
+        close(pts)
+        pts = fds
+        # convert pts handle to a TTY
+        #fds = pts.handle
+        #pts.status = Base.StatusClosed
+        #pts.handle = C_NULL
+        #pts = Base.TTY(fds, Base.StatusOpen)
+    else
+        O_RDWR = Base.Filesystem.JL_O_RDWR
+        O_NOCTTY = Base.Filesystem.JL_O_NOCTTY
+
+        fdm = ccall(:posix_openpt, Cint, (Cint,), O_RDWR | O_NOCTTY)
+        fdm == -1 && error("Failed to open ptm")
+        rc = ccall(:grantpt, Cint, (Cint,), fdm)
+        rc != 0 && error("grantpt failed")
+        rc = ccall(:unlockpt, Cint, (Cint,), fdm)
+        rc != 0 && error("unlockpt")
+
+        fds = ccall(:open, Cint, (Ptr{UInt8}, Cint),
+            ccall(:ptsname, Ptr{UInt8}, (Cint,), fdm), O_RDWR | O_NOCTTY)
+        pts = RawFD(fds)
+
+        # pts = fdio(fds, true)
+        # pts = Base.Filesystem.File(RawFD(fds))
+        # pts = Base.TTY(RawFD(fds); readable = false)
+        ptm = Base.TTY(RawFD(fdm))
+    end
+    return pts, ptm
+end
+
+function with_fake_pty(f)
+    pts, ptm = open_fake_pty()
+    try
+        f(pts, ptm)
+    finally
+        close(ptm)
+    end
+    nothing
+end
+
+end

src/OhMyREPL.jl

@@ -135,59 +135,10 @@ function __init__()
     end
 end
 
-if !(ccall(:jl_generating_output, Cint, ()) == 1)
-    using TerminalRegressionTests
-else
-
-let
-tracecompile_file = tempname()
-
-content = """
-const OhMyREPL = Base.require(Base.PkgId(Base.UUID("5fb14364-9ced-5910-84b2-373655c76a03"), "OhMyREPL"))
-using OhMyREPL.REPL
-OhMyREPL.TerminalRegressionTests.create_automated_test(
-    tempname(),
-    [c for c in "function foo(a = b())\nend\n\x4"],
-    aggressive_yield=true) do emuterm
-    repl = REPL.LineEditREPL(emuterm, false)
-    repl.specialdisplay = REPL.REPLDisplay(repl)
-    repl.interface = REPL.setup_interface(repl)
-    OhMyREPL.Prompt.insert_keybindings(repl)
-    REPL.run_repl(repl)
+if !Sys.iswindows()
+if ccall(:jl_generating_output, Cint, ()) == 1
+    include("precompile.jl")
 end
-"""
-
-function run_with_timeout(command, timeout::Integer = 25)
-    cmd = run(command; wait=false)
-     for _ in 1:timeout
-         if !process_running(cmd) return success(cmd) end
-         sleep(1)
-     end
-     @warn "OhMyREPL precompilation not finished in time, killing"
-     kill(cmd)
-     return false
-end
-
-
-run_with_timeout(
-    pipeline(ignorestatus(`$(Base.julia_cmd()) --project=$(Base.active_project()) --trace-compile=$tracecompile_file --compiled-modules=no -e $content`); stderr=devnull)
-)
-
-num_prec = 0
-for line in eachline(tracecompile_file)
-    try
-        eval(Meta.parse(line))
-        num_prec += 1
-    catch e
-        # @warn line
-    end
-end
-
-rm(tracecompile_file; force=true)
-
-# @show num_prec ~251
 end
 
-end # end precompile block
-
 end # module

src/precompile.jl

@@ -0,0 +1,114 @@
+
+let
+
+include("FakePTYs.jl")
+import .FakePTYs: open_fake_pty
+
+repl_script = """
+const OhMyREPL = Base.require(Base.PkgId(Base.UUID("5fb14364-9ced-5910-84b2-373655c76a03"), "OhMyREPL"))
+function f(x) x end
+print("")
+[]
+?reinterpret
+"""
+
+julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename())
+
+
+JULIA_PROMPT = "julia> "
+PKG_PROMPT = "pkg> "
+SHELL_PROMPT = "shell> "
+HELP_PROMPT = "help?> "
+
+function generate_precompile_statements()
+    debug_output = devnull
+
+    # Extract the precompile statements from the precompile file
+    statements = Set{String}()
+
+    mktemp() do precompile_file, precompile_file_h
+        # Collect statements from running a REPL process and replaying our REPL script
+        pts, ptm = open_fake_pty()
+        blackhole = Sys.isunix() ? "/dev/null" : "nul"
+
+        p = withenv("JULIA_HISTORY" => blackhole,
+                    "TERM" => "") do
+            run(```$(julia_exepath()) -O0 --trace-compile=$precompile_file
+                --cpu-target=native --project=$(Base.active_project()) --compiled-modules=no --startup-file=no -i --color=yes```,
+                pts, pts, pts; wait=false)
+        end
+        Base.close_stdio(pts)
+        # Prepare a background process to copy output from process until `pts` is closed
+        output_copy = Base.BufferStream()
+        tee = @async try
+            while !eof(ptm)
+                l = readavailable(ptm)
+                write(debug_output, l)
+                Sys.iswindows() && (sleep(0.1); yield(); yield()) # workaround hang - probably a libuv issue?
+                write(output_copy, l)
+            end
+        catch ex
+            if !(ex isa Base.IOError && ex.code == Base.UV_EIO)
+                rethrow() # ignore EIO on ptm after pts dies
+            end
+        finally
+            close(output_copy)
+            close(ptm)
+        end
+        # wait for the definitive prompt before start writing to the TTY
+        readuntil(output_copy, JULIA_PROMPT)
+        sleep(0.1)
+        readavailable(output_copy)
+        # Input our script
+        precompile_lines = split(repl_script::String, '\n'; keepempty=false)
+        curr = 0
+        for l in precompile_lines
+            sleep(0.1)
+            curr += 1
+            # print("\rGenerating REPL precompile statements... $curr/$(length(precompile_lines))")
+            # consume any other output
+            bytesavailable(output_copy) > 0 && readavailable(output_copy)
+            # push our input
+            write(debug_output, "\n#### inputting statement: ####\n$(repr(l))\n####\n")
+            write(ptm, l, "\n")
+            readuntil(output_copy, "\n")
+            # wait for the next prompt-like to appear
+            readuntil(output_copy, "\n")
+            strbuf = ""
+            while !eof(output_copy)
+                strbuf *= String(readavailable(output_copy))
+                occursin(JULIA_PROMPT, strbuf) && break
+                occursin(PKG_PROMPT, strbuf) && break
+                occursin(SHELL_PROMPT, strbuf) && break
+                occursin(HELP_PROMPT, strbuf) && break
+                sleep(0.1)
+            end
+        end
+        write(ptm, "exit()\n")
+        wait(tee)
+        success(p) || Base.pipeline_error(p)
+        close(ptm)
+        write(debug_output, "\n#### FINISHED ####\n")
+
+        for statement in split(read(precompile_file, String), '\n')
+            # Main should be completely clean
+            occursin("Main.", statement) && continue
+            push!(statements, statement)
+        end
+    end
+    num_prec = 0
+    for statement in statements
+        try
+            eval(Meta.parse(statement))
+            num_prec += 1
+        catch e
+            #@show e
+            #@warn statement
+        end
+    end
+    # @show num_prec
+end
+
+generate_precompile_statements()
+
+end