From 1bc0e5b7ec9e1a9b294e0ffc5ff651a2294a69e1 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 15 Mar 2023 10:05:40 +0100 Subject: [PATCH] move to JuliaSyntax.jl over Tokenize.jl (#277) --- Project.toml | 4 +- docs/Manifest.toml | 89 +++++++++++++++++++++----------- docs/src/internals/passes.md | 73 +++++++++++++------------- src/MarkdownHighlighter.jl | 6 +-- src/OhMyREPL.jl | 3 +- src/passes/BracketHighlighter.jl | 19 +++---- src/passes/RainbowBrackets.jl | 27 +++++----- src/passes/SyntaxHighlighter.jl | 46 +++++++++-------- src/precompile.jl | 1 - src/repl.jl | 10 ++-- src/repl_pass.jl | 30 +++++------ test/test_bracketmatch.jl | 1 - test/test_custompass.jl | 9 ++-- test/test_highlighter.jl | 3 +- 14 files changed, 180 insertions(+), 141 deletions(-) diff --git a/Project.toml b/Project.toml index 3f42836..08a5b9b 100644 --- a/Project.toml +++ b/Project.toml @@ -5,16 +5,16 @@ version = "0.5.17" [deps] Crayons = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" JLFzf = "1019f520-868f-41f5-a6de-eb00f4b6a39c" +JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624" [compat] Crayons = "1, 2, 3, 4" +JuliaSyntax = "0.3.0" JLFzf = "^0.1.1" -Tokenize = "0.5" julia = "1.6" [extras] diff --git a/docs/Manifest.toml b/docs/Manifest.toml index c358061..9278090 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,7 +1,13 @@ # This file is machine-generated - editing it directly is not advised +[[ANSIColoredPrinters]] +git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" +uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9" +version = "0.0.1" + [[ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" [[Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -10,9 +16,9 @@ uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[Crayons]] -git-tree-sha1 = "3f71217b538d7aaee0b69ab47d9b7724ca8afa0d" +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.0.4" +version = "4.1.1" [[Dates]] deps = ["Printf"] @@ -20,19 +26,23 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[DocStringExtensions]] deps = ["LibGit2"] -git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f" +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.8.5" +version = "0.9.3" [[Documenter]] -deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] -git-tree-sha1 = "47f13b6305ab195edb73c86815962d84e31b0f48" +deps = ["ANSIColoredPrinters", "Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] +git-tree-sha1 = "58fea7c536acd71f3eef6be3b21c0df5f3df88fd" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.27.3" +version = "0.27.24" [[Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" [[IOCapture]] deps = ["Logging", "Random"] @@ -46,29 +56,36 @@ uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[JLFzf]] deps = ["Pipe", "REPL", "Random", "fzf_jll"] -git-tree-sha1 = "fa300b5b580549186c2b1558a199ac5615cc8928" +git-tree-sha1 = "f377670cda23b6b7c1c0b3893e37451c5c1a2185" uuid = "1019f520-868f-41f5-a6de-eb00f4b6a39c" -version = "0.1.3" +version = "0.1.5" [[JLLWrappers]] deps = ["Preferences"] -git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" +git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.3.0" +version = "1.4.1" [[JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4" +git-tree-sha1 = "3c837543ddb02250ef42f4738347454f95079d4e" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.1" +version = "0.21.3" + +[[JuliaSyntax]] +git-tree-sha1 = "db2bdeda30e452485863799be4515f6305431a46" +uuid = "70703baa-626e-46a2-a12c-08ffd08c73b4" +version = "0.3.2" [[LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" [[LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" [[LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] @@ -77,6 +94,7 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" [[Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -91,27 +109,30 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" [[NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" [[OhMyREPL]] -deps = ["Crayons", "JLFzf", "Markdown", "Pkg", "Printf", "REPL", "Tokenize"] +deps = ["Crayons", "JLFzf", "JuliaSyntax", "Markdown", "Pkg", "Printf", "REPL"] path = ".." uuid = "5fb14364-9ced-5910-84b2-373655c76a03" -version = "0.5.9" +version = "0.5.17" [[Parsers]] -deps = ["Dates"] -git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc" +deps = ["Dates", "SnoopPrecompile"] +git-tree-sha1 = "478ac6c952fddd4399e71d4779797c538d0ff2bf" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "1.1.0" +version = "2.5.8" [[Pipe]] git-tree-sha1 = "6842804e7867b115ca9de748a0cf6b364523c16d" @@ -119,14 +140,15 @@ uuid = "b98c9c47-44ae-5843-9183-064241ee97a0" version = "1.3.0" [[Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.0" [[Preferences]] deps = ["TOML"] -git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" +git-tree-sha1 = "47e5f437cc0e7ef2ce8406ce1e7e24d44915f88d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.2.2" +version = "1.3.0" [[Printf]] deps = ["Unicode"] @@ -137,35 +159,39 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[Random]] -deps = ["Serialization"] +deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +[[SnoopPrecompile]] +deps = ["Preferences"] +git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c" +uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" +version = "1.0.3" + [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" [[Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" [[Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -[[Tokenize]] -git-tree-sha1 = "37018506dc445ad7db288442fbb846105f26c43f" -uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624" -version = "0.5.17" - [[UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -176,17 +202,20 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" [[fzf_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "b2d7b205b9518f8a5ebcd05764c5536dbdf460bb" +git-tree-sha1 = "868e669ccb12ba16eaf50cb2957ee2ff61261c56" uuid = "214eeab7-80f7-51ab-84ad-2988db7cef09" -version = "0.24.4+0" +version = "0.29.0+0" [[nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" [[p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/docs/src/internals/passes.md b/docs/src/internals/passes.md index 2271889..7694603 100644 --- a/docs/src/internals/passes.md +++ b/docs/src/internals/passes.md @@ -7,10 +7,10 @@ All the passes are registered in a global pass handler. To show all the passes u ```jl julia> OhMyREPL.showpasses() ---------------------------------- - # Pass name Enabled + # Pass name Enabled ---------------------------------- - 1 BracketHighlighter true - 2 SyntaxHighlighter true + 1 BracketHighlighter true + 2 SyntaxHighlighter true ---------------------------------- ``` @@ -34,24 +34,24 @@ julia> using Tokenize julia> tokens = collect(Tokenize.tokenize(str)) 20-element Array{Tokenize.Tokens.Token,1}: 1,1-1,8: KEYWORD "function" - 1,9-1,9: WHITESPACE " " - 1,10-1,10: IDENTIFIER "f" - 1,11-1,11: LPAREN "(" - 1,12-1,12: IDENTIFIER "x" - 1,13-1,14: OP "::" + 1,9-1,9: WHITESPACE " " + 1,10-1,10: IDENTIFIER "f" + 1,11-1,11: LPAREN "(" + 1,12-1,12: IDENTIFIER "x" + 1,13-1,14: OP "::" 1,15-1,21: IDENTIFIER "Float64" - 1,22-1,22: RPAREN ")" - 1,23-1,23: WHITESPACE " " - 1,24-1,29: KEYWORD "return" - 1,30-1,30: WHITESPACE " " - 1,31-1,31: OP ":" - 1,32-1,32: IDENTIFIER "x" - 1,33-1,33: WHITESPACE " " - 1,34-1,34: OP "+" - 1,35-1,35: WHITESPACE " " - 1,36-1,38: CHAR "'a'" - 1,39-1,39: WHITESPACE " " - 1,40-1,42: KEYWORD "end" + 1,22-1,22: RPAREN ")" + 1,23-1,23: WHITESPACE " " + 1,24-1,29: KEYWORD "return" + 1,30-1,30: WHITESPACE " " + 1,31-1,31: OP ":" + 1,32-1,32: IDENTIFIER "x" + 1,33-1,33: WHITESPACE " " + 1,34-1,34: OP "+" + 1,35-1,35: WHITESPACE " " + 1,36-1,38: CHAR "'a'" + 1,39-1,39: WHITESPACE " " + 1,40-1,42: KEYWORD "end" 1,43-1,42: ENDMARKER "" ``` @@ -62,17 +62,17 @@ crayons = Vector{Crayon}(length(tokens)); fill!(crayons, Crayon()) # Crayon is a bits type so this is OK ``` -These two vectors are then sent to the syntax highlighter pass together with an integer that represent what character offset the cursor currently is located. The syntax highlighter does not use this information but the bracket highlighter does. +These two vectors and the source code are then sent to the syntax highlighter pass together with an integer that represent what character offset the cursor currently is located. The syntax highlighter does not use this information but the bracket highlighter does. ``` -OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0) +OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0, str) ``` Running this function has the effect of updating the `crayons` vector. If we print this vector we see that they have been updated: ![](ansitokens_after.png) -To print the original string with the updated vector of `Crayon`s we use the `OhMyREPL.untokenize_with_ANSI([io::IO], crayons, tokens)` function as: +To print the original string with the updated vector of `Crayon`s we use the `OhMyREPL.untokenize_with_ANSI([io::IO], crayons, tokens, str)` function as: ![](print_ansi.png) @@ -80,7 +80,7 @@ Each registered and enabled pass does this updating and the contributions from e ## Creating a pass -This section shows how to create a very pass that let the user define a `Crayon` for each typeassertion / declaration that happens to be a `Float64`. +This section shows how to create a pass that let the user define a `Crayon` for each typeassertion / declaration that happens to be a `Float64`. !!! info Please refer to the [Tokenize.jl API](https://github.com/KristofferC/Tokenize.jl#api) section and the [`Crayons.jl` documentation](https://github.com/KristofferC/Crayons.jl) while reading this section. @@ -89,7 +89,8 @@ We start off with a few imports and creating a new struct which will hold the se ```jl using Crayons -import Tokenize.Tokens: Token, untokenize, exactkind +import JuliaSyntax +import JuliaSyntax.Tokenize: Token, untokenize, kind using OhMyREPL mutable struct Float64Modifier @@ -104,11 +105,11 @@ We then use call overloading to define a function for the type. The function wil ```jl # The pass function, the cursor position is not used but it needs to be given an argument -function (float64modifier::Float64Modifier)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int) +function (float64modifier::Float64Modifier)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int, str::AbstractString) # Loop over all tokens and crayons for i in 1:length(crayons) - if untokenize(tokens[i]) == "Float64" - if i > 1 && exactkind(tokens[i-1]) == Tokenize.Tokens.DECLARATION + if untokenize(tokens[i], str) == "Float64" + if i > 1 && kind(tokens[i-1]) == JuliaSyntax.K"::" # Update the crayon crayons[i] = float64modifier.crayon end @@ -130,11 +131,11 @@ To register and start using the pass simply use `OhMyREPL.add_pass!(passname::St ```jl julia> OhMyREPL.add_pass!("Redify Float64", FLOAT64_MODIFIER) ---------------------------------- - # Pass name Enabled + # Pass name Enabled ---------------------------------- - 1 Redify Float64 true - 2 BracketHighlighter true - 3 SyntaxHighlighter true + 1 Redify Float64 true + 2 BracketHighlighter true + 3 SyntaxHighlighter true ---------------------------------- ``` @@ -151,11 +152,11 @@ The prescedence of a pass can be modified with the `OhMyREPL.prescedence!(pass:: ```jl julia> OhMyREPL.prescedence!("Redify Float64", 3) ---------------------------------- - # Pass name Enabled + # Pass name Enabled ---------------------------------- - 1 BracketHighlighter true - 2 SyntaxHighlighter true - 3 Redify Float64 true + 1 BracketHighlighter true + 2 SyntaxHighlighter true + 3 Redify Float64 true ---------------------------------- ``` diff --git a/src/MarkdownHighlighter.jl b/src/MarkdownHighlighter.jl index 1438ebc..44a9ea3 100644 --- a/src/MarkdownHighlighter.jl +++ b/src/MarkdownHighlighter.jl @@ -1,6 +1,6 @@ using Crayons -using Tokenize +using JuliaSyntax.Tokenize import Markdown import .OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS @@ -40,14 +40,14 @@ function Markdown.term(io::IO, md::Markdown.Code, columns) for (sourcecode, output) in zip(sourcecodes, outputs) tokens = collect(tokenize(sourcecode)) crayons = fill(Crayon(), length(tokens)) - SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0) + SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0, sourcecode) buff = IOBuffer() if lang == "jldoctest" || lang == "julia-repl" print(buff, Crayon(foreground = :red, bold = true), "julia> ", Crayon(reset = true)) end for (token, crayon) in zip(tokens, crayons) print(buff, crayon) - print(buff, untokenize(token)) + print(buff, untokenize(token, sourcecode)) print(buff, Crayon(reset = true)) end print(buff, output) diff --git a/src/OhMyREPL.jl b/src/OhMyREPL.jl index 564f90a..14874f9 100644 --- a/src/OhMyREPL.jl +++ b/src/OhMyREPL.jl @@ -5,7 +5,8 @@ bracket matching and other nifty features. """ module OhMyREPL -using Tokenize +import JuliaSyntax +using JuliaSyntax.Tokenize using Crayons if VERSION > v"1.3" import JLFzf diff --git a/src/passes/BracketHighlighter.jl b/src/passes/BracketHighlighter.jl index aa69aa0..2c7d25a 100644 --- a/src/passes/BracketHighlighter.jl +++ b/src/passes/BracketHighlighter.jl @@ -1,11 +1,12 @@ module BracketHighlighter -using Tokenize -using Tokenize.Tokens -import Tokenize.Tokens: Token, kind, startpos, endpos, untokenize +using JuliaSyntax.Tokenize +import .Tokenize: Token, kind, startpos, untokenize using Crayons +import JuliaSyntax.@K_str + import OhMyREPL: add_pass!, PASS_HANDLER mutable struct BracketHighlighterSettings @@ -15,8 +16,8 @@ end const BRACKETMATCHER_SETTINGS = BracketHighlighterSettings(Crayon(bold = :true, underline = :true)) -function (matcher::BracketHighlighterSettings)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int) - left_bracket_match, right_bracket_match, matched = bracket_match(tokens, cursorpos) +function (matcher::BracketHighlighterSettings)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int, str::AbstractString) + left_bracket_match, right_bracket_match, matched = bracket_match(tokens, cursorpos, str) !matched && return crayons[left_bracket_match] = matcher.crayon crayons[right_bracket_match] = matcher.crayon @@ -31,14 +32,14 @@ add_pass!(PASS_HANDLER, "BracketHighlighter", BRACKETMATCHER_SETTINGS, true) # Takes a string and a cursor index. # Returns index of left matching bracket, right matching bracket # and if there was a match at all as a 3 tuple. -const LEFT_DELIMS = [Tokens.LPAREN, Tokens.LSQUARE, Tokens.LBRACE] -const RIGHT_DELIMS = [Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE] -function bracket_match(tokens::Vector{Token}, cursoridx::Int) +const LEFT_DELIMS = [K"(", K"[", K"{"] +const RIGHT_DELIMS = [K")", K"]", K"}"] +function bracket_match(tokens::Vector{Token}, cursoridx::Int, str) enclosing_token_idx = -1 char_counter = 0 # find which token we are currently inside for (i, token) in enumerate(tokens) - char_counter += length(untokenize(token)) + char_counter += length(untokenize(token, str)) if char_counter >= cursoridx enclosing_token_idx = i break diff --git a/src/passes/RainbowBrackets.jl b/src/passes/RainbowBrackets.jl index 36fd0de..e6743e3 100644 --- a/src/passes/RainbowBrackets.jl +++ b/src/passes/RainbowBrackets.jl @@ -1,9 +1,10 @@ module RainbowBrackets using Crayons -using Tokenize -using Tokenize.Tokens -import Tokenize.Tokens: Token, kind, startpos, endpos, untokenize +import JuliaSyntax.@K_str +import JuliaSyntax.Tokenize +using .Tokenize +import .Tokenize: Token, kind, startpos, untokenize, Kind import OhMyREPL: add_pass!, PASS_HANDLER, SUPPORTS_256_COLORS @@ -22,13 +23,13 @@ mutable struct RainbowBracketsSettings active::RainBowTokens end -function get_color(rainbow::RainbowBracketsSettings, k::Tokens.Kind, i) +function get_color(rainbow::RainbowBracketsSettings, k::Kind, i) @assert i >= 0 toks = rainbow.active i == 0 && return toks.error_token - if k == Tokens.LPAREN || k == Tokens.RPAREN + if k == K"(" || k == K")" return toks.parenthesis_tokens[mod1(i, length(toks.parenthesis_tokens))] - elseif k == Tokens.LSQUARE || k == Tokens.RSQUARE + elseif k == K"[" || k == K"]" return toks.brackets_tokens[mod1(i, length(toks.brackets_tokens))] else return toks.curly_tokens[mod1(i, length(toks.curly_tokens))] @@ -63,26 +64,26 @@ function updatebracketcolors!(cs::SyntaxHighlighter.ColorScheme) RAINBOWBRACKETS_SETTINGS.active.error_token = cs.error end -function (rainbow::RainbowBracketsSettings)(ansitokens::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int) +function (rainbow::RainbowBracketsSettings)(ansitokens::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int, str::AbstractString) p, s, b = 0, 0, 0 for (i, t) in enumerate(tokens) k = kind(t) - if k == Tokens.LPAREN + if k == K"(" p += 1 ansitokens[i] = get_color(rainbow, k, p) - elseif k == Tokens.RPAREN + elseif k == K")" ansitokens[i] = get_color(rainbow, k, p) p = max(0, p - 1) - elseif k == Tokens.LSQUARE + elseif k == K"[" s += 1 ansitokens[i] = get_color(rainbow, k, s) - elseif k == Tokens.RSQUARE + elseif k == K"]" ansitokens[i] = get_color(rainbow, k, s) s = max(0, s - 1) - elseif k == Tokens.LBRACE + elseif k == K"{" b += 1 ansitokens[i] = get_color(rainbow, k, b) - elseif k == Tokens.RBRACE + elseif k == K"}" ansitokens[i] = get_color(rainbow, k, b) b = max(0, b - 1) end diff --git a/src/passes/SyntaxHighlighter.jl b/src/passes/SyntaxHighlighter.jl index c34f456..92cb684 100644 --- a/src/passes/SyntaxHighlighter.jl +++ b/src/passes/SyntaxHighlighter.jl @@ -1,8 +1,10 @@ module SyntaxHighlighter -using Tokenize -using Tokenize.Tokens -import Tokenize.Tokens: Token, kind, exactkind, iskeyword, untokenize +import JuliaSyntax +import .JuliaSyntax.Tokenize +import JuliaSyntax.@K_str +using .Tokenize +import .Tokenize: Token, kind, kind, untokenize using Crayons @@ -108,56 +110,58 @@ end add_pass!(PASS_HANDLER, "SyntaxHighlighter", SYNTAX_HIGHLIGHTER_SETTINGS, false) -function (highlighter::SyntaxHighlighterSettings)(crayons::Vector{Crayon}, tokens::Vector{Token}, ::Int) +function (highlighter::SyntaxHighlighterSettings)(crayons::Vector{Crayon}, tokens::Vector{Token}, ::Int, str::AbstractString) cscheme = highlighter.active - prev_t = Tokens.Token() - pprev_t = Tokens.Token() + prev_t = Token() + pprev_t = Token() for (i, t) in enumerate(tokens) # a::x - if exactkind(prev_t) == Tokens.DECLARATION + #= + if kind(prev_t) == Tokens.DECLARATION crayons[i-1] = cscheme.argdef crayons[i] = cscheme.argdef + =# # :foo - elseif kind(t) == Tokens.IDENTIFIER && exactkind(prev_t) == Tokens.COLON && - kind(pprev_t) ∉ (Tokens.INTEGER, Tokens.FLOAT, Tokens.IDENTIFIER, Tokens.RPAREN) + if kind(t) == K"Identifier" && kind(prev_t) == K":" && + kind(pprev_t) ∉ (K"Integer", K"Float", K"Identifier", K")") crayons[i-1] = cscheme.symbol crayons[i] = cscheme.symbol # function - elseif iskeyword(kind(t)) - if kind(t) == Tokens.TRUE || kind(t) == Tokens.FALSE + elseif JuliaSyntax.is_keyword(kind(t)) + if kind(t) == K"true" || kind(t) == K"false" crayons[i] = cscheme.symbol else crayons[i] = cscheme.keyword end # "foo" - elseif kind(t) == Tokens.STRING || kind(t) == Tokens.TRIPLE_STRING || kind(t) == Tokens.CHAR || kind(t) == Tokens.CMD || kind(t) == Tokens.TRIPLE_CMD + elseif kind(t) == K"String" || kind(t) == K"Char" || kind(t) == K"CmdString" crayons[i] = cscheme.string # * - - elseif Tokens.isoperator(kind(t)) || exactkind(t) == Tokens.TRUE || exactkind(t) == Tokens.FALSE + elseif JuliaSyntax.is_operator(kind(t)) || kind(t) == K"true" || kind(t) == K"false" crayons[i] = cscheme.op # #= foo =# - elseif kind(t) == Tokens.COMMENT + elseif kind(t) == K"Comment" crayons[i] = cscheme.comment # f(...) - elseif kind(t) == Tokens.LPAREN - if kind(prev_t) == Tokens.IDENTIFIER && !(i > 2 && exactkind(tokens[i-2]) == Tokens.AT_SIGN) + elseif kind(t) == K"(" + if kind(prev_t) == K"Identifier" && !(i > 2 && kind(tokens[i-2]) == K"@") crayons[i-1] = cscheme.call - elseif exactkind(prev_t) == Tokens.DOT && kind(pprev_t) == Tokens.IDENTIFIER + elseif kind(prev_t) == K"." && kind(pprev_t) == K"Identifier" crayons[i-1] = cscheme.call crayons[i-2] = cscheme.call end # function f(...) - if i > 3 && kind(tokens[i-2]) == Tokens.WHITESPACE && exactkind(tokens[i-3]) == Tokens.FUNCTION + if i > 3 && JuliaSyntax.is_whitespace(kind(tokens[i-2])) && kind(tokens[i-3]) == K"function" crayons[i-1] = cscheme.function_def end # @fdsafds - elseif kind(t) == Tokens.IDENTIFIER && exactkind(prev_t) == Tokens.AT_SIGN + elseif kind(t) == K"Identifier" && kind(prev_t) == K"@" crayons[i-1] = cscheme._macro crayons[i] = cscheme._macro # 2] = 32.32 - elseif kind(t) ∈ (Tokens.INTEGER, Tokens.BIN_INT, Tokens.OCT_INT, Tokens.HEX_INT, Tokens.FLOAT) || (kind(t) == Tokens.IDENTIFIER && untokenize(t) == "NaN") + elseif kind(t) ∈ (K"Integer", K"BinInt", K"OctInt", K"HexInt", K"Float") || (kind(t) == K"Identifier" && untokenize(t, str) == "NaN") crayons[i] = cscheme.number - elseif kind(t) == Tokens.WHITESPACE + elseif JuliaSyntax.is_whitespace(kind(t)) crayons[i] = Crayon() else crayons[i] = cscheme.text diff --git a/src/precompile.jl b/src/precompile.jl index 1174e1e..90caafd 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -1,4 +1,3 @@ - let include("FakePTYs.jl") diff --git a/src/repl.jl b/src/repl.jl index 85e482b..6141c18 100644 --- a/src/repl.jl +++ b/src/repl.jl @@ -11,7 +11,8 @@ import REPL: respond, return_callback import REPL.LineEdit: buffer, cmove_col, cmove_up, InputAreaState, transition, terminal, buffer, on_enter, move_input_end, add_history, state, mode, edit_insert -import Tokenize.Lexers +import JuliaSyntax.Tokenize +import .Tokenize.Lexer import REPL.Terminals: raw!, width, height, cmove, getX, TerminalBuffer, getY, clear_line, beep, disable_bracketed_paste, enable_bracketed_paste @@ -58,9 +59,10 @@ function rewrite_with_ANSI(s, cursormove::Bool = false) # Insert colorized text from running the passes seekstart(buffer(s)) - tokens = collect(Lexers.Lexer(buffer(s))) - apply_passes!(PASS_HANDLER, tokens, cursoridx, cursormove) - untokenize_with_ANSI(outbuf, PASS_HANDLER , tokens, l) + str = read(buffer(s), String) + tokens = collect(Lexer(str)) + apply_passes!(PASS_HANDLER, tokens, str, cursoridx, cursormove) + untokenize_with_ANSI(outbuf, PASS_HANDLER, tokens, str) # Reset the buffer since the Lexer messed with it (maybe the Lexer should reset it on done) seek(buffer(s), p) diff --git a/src/repl_pass.jl b/src/repl_pass.jl index aa82bfa..535613e 100644 --- a/src/repl_pass.jl +++ b/src/repl_pass.jl @@ -1,5 +1,5 @@ -import Tokenize:Tokens, Lexers -import Tokenize.Tokens: Token, kind, untokenize +import JuliaSyntax.Tokenize +import .Tokenize: Token, kind, untokenize, Lexer using Printf const RESET = Crayon(reset = true) @@ -22,9 +22,9 @@ const PASS_HANDLER = PassHandler() function test_pass(io::IO, f, str::Union{String, IO}, cursorpos::Int = 1, cursormovement::Bool = false) rpc = PassHandler() add_pass!(rpc, "TestPass", f) - tokens = collect(Lexers.Lexer(str)) - apply_passes!(rpc, tokens, cursorpos, cursormovement) - untokenize_with_ANSI(io, rpc.accum_crayons, tokens) + tokens = collect(Lexer(str)) + apply_passes!(rpc, tokens, str, cursorpos, cursormovement) + untokenize_with_ANSI(io, rpc.accum_crayons, tokens, str) end test_pass(f, str::Union{String, IOBuffer}, cursorpos::Int = 1, cursormovement::Bool = false) = @@ -32,30 +32,30 @@ test_pass(f, str::Union{String, IOBuffer}, cursorpos::Int = 1, cursormovement::B function test_passes(io::IO, rpc::PassHandler, str::Union{String, IOBuffer}, cursorpos::Int = 1, cursormovement::Bool = false) b = IOBuffer() - tokens = collect(Lexers.Lexer(str)) - apply_passes!(rpc, tokens, cursorpos, cursormovement) - untokenize_with_ANSI(io, rpc.accum_crayons, tokens) + tokens = collect(Lexer(str)) + apply_passes!(rpc, tokens, str, cursorpos, cursormovement) + untokenize_with_ANSI(io, rpc.accum_crayons, tokens, str) end test_passes(rpc::PassHandler, str::Union{String, IOBuffer}, cursorpos::Int = 1, cursormovement::Bool = false) = test_passes(stdout, rpc, str, cursorpos, cursormovement) -function untokenize_with_ANSI(io::IO, crayons::Vector{Crayon}, tokens::Vector{Token}, indent = 7) +function untokenize_with_ANSI(io::IO, crayons::Vector{Crayon}, tokens::Vector{Token}, str, indent = 7) @assert length(tokens) == length(crayons) print(io, RESET) z = 1 for (token, crayon) in zip(tokens, crayons) print(io, crayon) - for c in untokenize(token) + for c in untokenize(token, str) print(io, c) c == '\n' && print(io, " "^indent) end print(io, RESET) end end -untokenize_with_ANSI(io::IO, rpc::PassHandler, tokens::Vector{Token}, indent = 7) = untokenize_with_ANSI(io, rpc.accum_crayons, tokens, indent) -untokenize_with_ANSI(crayons::Vector{Crayon}, tokens::Vector{Token}, indent = 7) = - untokenize_with_ANSI(stdout, crayons, tokens, indent) +untokenize_with_ANSI(io::IO, rpc::PassHandler, tokens::Vector{Token}, str::AbstractString, indent = 7) = untokenize_with_ANSI(io, rpc.accum_crayons, tokens, str, indent) +untokenize_with_ANSI(crayons::Vector{Crayon}, tokens::Vector{Token}, str::AbstractString, indent = 7) = + untokenize_with_ANSI(stdout, crayons, tokens, str, indent) function merge!(t1::Vector{Crayon}, t2::Vector{Crayon}) @assert length(t1) == length(t2) @@ -65,7 +65,7 @@ function merge!(t1::Vector{Crayon}, t2::Vector{Crayon}) return t1 end -function apply_passes!(rpc::PassHandler, tokens::Vector{Token}, cursorpos::Int = 1, cursormovement::Bool = false) +function apply_passes!(rpc::PassHandler, tokens::Vector{Token}, str::AbstractString, cursorpos::Int = 1, cursormovement::Bool = false) resize!(rpc.crayons, length(tokens)) resize!(rpc.accum_crayons, length(tokens)) fill!(rpc.accum_crayons, Crayon()) @@ -77,7 +77,7 @@ function apply_passes!(rpc::PassHandler, tokens::Vector{Token}, cursorpos::Int = continue end # This is an enabled pass that should be run, so run it! - pass.f!(rpc.crayons, tokens, cursorpos) + pass.f!(rpc.crayons, tokens, cursorpos, str) merge!(rpc.accum_crayons, rpc.crayons) end end diff --git a/test/test_bracketmatch.jl b/test/test_bracketmatch.jl index d7c5ec2..3377a69 100644 --- a/test/test_bracketmatch.jl +++ b/test/test_bracketmatch.jl @@ -2,7 +2,6 @@ module TestBracketHighlighter using Test using OhMyREPL -using Tokenize b = IOBuffer() str = "(function :foobar, foobar )# foobar" diff --git a/test/test_custompass.jl b/test/test_custompass.jl index 923f264..d7a58bd 100644 --- a/test/test_custompass.jl +++ b/test/test_custompass.jl @@ -2,14 +2,15 @@ module CustomPassTest using Test using OhMyREPL -using Tokenize +import JuliaSyntax.Tokenize +using .Tokenize using Crayons -function foobar_bluify(crayons, tokens, ::Int) +function foobar_bluify(crayons, tokens, ::Int, str::AbstractString) for (i, (crayon, tok)) in enumerate(zip(crayons, tokens)) println(tok) - if (Tokenize.Tokens.kind(tok) == Tokenize.Tokens.IDENTIFIER - && Tokenize.Tokens.untokenize(tok) == "foobar") + if (Tokenize.kind(tok) == Tokenize.K"Identifier" + && Tokenize.untokenize(tok, str) == "foobar") crayons[i] = Crayon(foreground = :blue) end end diff --git a/test/test_highlighter.jl b/test/test_highlighter.jl index 2c38423..955adf4 100644 --- a/test/test_highlighter.jl +++ b/test/test_highlighter.jl @@ -5,7 +5,8 @@ using Test using OhMyREPL import OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS -using Tokenize +import JuliaSyntax.Tokenize +using .Tokenize OhMyREPL.colorscheme!("Monokai16")