Skip to content

Commit

Permalink
feat: 支持打印调试信息。
Browse files Browse the repository at this point in the history
  • Loading branch information
@CYJB
CYJB committed May 31, 2024
1 parent 8f12b97 commit f65d5c7
Show file tree
Hide file tree
Showing 8 changed files with 524 additions and 15 deletions.
77 changes: 77 additions & 0 deletions Runtime/Lexers/Core/BasicDebugCore.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
namespace Cyjb.Compilers.Lexers;

/// <summary>
/// 表示带有调试信息的基本的词法分析器核心。
/// </summary>
/// <typeparam name="T">词法单元标识符的类型，一般是一个枚举类型。</typeparam>
internal sealed class BasicDebugCore<T> : LexerCore<T>
where T : struct
{
/// <summary>
/// 词法分析器的数据。
/// </summary>
private readonly LexerData<T> lexerData;
/// <summary>
/// DFA 的状态列表。
/// </summary>
private readonly int[] states;

/// <summary>
/// 使用给定的词法分析器信息初始化 <see cref="BasicCore{T}"/> 类的新实例。
/// </summary>
/// <param name="lexerData">要使用的词法分析器的数据。</param>
/// <param name="controller">词法分析控制器。</param>
public BasicDebugCore(LexerData<T> lexerData, LexerController<T> controller) :
base(lexerData.States, lexerData.Terminals, lexerData.ContainsBeginningOfLine, controller)
{
this.lexerData = lexerData;
states = lexerData.States;
}

/// <summary>
/// 读取输入流中的下一个词法单元并提升输入流的字符位置。
/// </summary>
/// <param name="state">DFA 的起始状态。</param>
/// <param name="start">当前词法单元的起始位置。</param>
/// <returns>词法单元读入是否成功。</returns>
public override bool NextToken(int state, int start)
{
// 最后一次匹配的符号和文本索引。
int startIndex = source.Index;
int lastAccept = -1, lastIndex = source.Index;
int symbolStart = 0, symbolEnd = 0;
while (true)
{
state = lexerData.NextState(state, source.Read());
if (state == -1)
{
// 没有合适的转移，退出。
break;
}
if (lexerData.GetSymbols(state, ref symbolStart, ref symbolEnd))
{
lastAccept = states[symbolStart];
lastIndex = source.Index;
// 使用最短匹配时，可以直接返回。
if (lexerData.UseShortest && terminalData[lastAccept].UseShortest)
{
break;
}
}
}
PrintReadedText();
if (lastAccept >= 0)
{
// 将流调整到与接受状态匹配的状态。
DoAction(start, lastIndex, terminalData[lastAccept]);
Console.WriteLine(" Match {0}..{1} [{2}] {3}", startIndex, lastIndex, lastAccept,
terminalData[lastAccept].Kind);
return true;
}
else
{
Console.WriteLine(" Match none");
}
return false;
}
}
95 changes: 95 additions & 0 deletions Runtime/Lexers/Core/FixedTrailingDebugCore`1.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
namespace Cyjb.Compilers.Lexers;

/// <summary>
/// 表示带有调试信息的支持定长向前看符号的词法分析器核心。
/// </summary>
/// <typeparam name="T">词法单元标识符的类型，一般是一个枚举类型。</typeparam>
internal sealed class FixedTrailingDebugCore<T> : LexerCore<T>
where T : struct
{
/// <summary>
/// 词法分析器的数据。
/// </summary>
private readonly LexerData<T> lexerData;
/// <summary>
/// DFA 的状态列表。
/// </summary>
private readonly int[] states;

/// <summary>
/// 使用给定的词法分析器信息初始化 <see cref="FixedTrailingCore{T}"/> 类的新实例。
/// </summary>
/// <param name="lexerData">要使用的词法分析器的数据。</param>
/// <param name="controller">词法分析控制器。</param>
public FixedTrailingDebugCore(LexerData<T> lexerData, LexerController<T> controller) :
base(lexerData.States, lexerData.Terminals, lexerData.ContainsBeginningOfLine, controller)
{
this.lexerData = lexerData;
states = lexerData.States;
}

/// <summary>
/// 读取输入流中的下一个词法单元并提升输入流的字符位置。
/// </summary>
/// <param name="state">DFA 的起始状态。</param>
/// <param name="start">当前词法单元的起始位置。</param>
/// <returns>词法单元读入是否成功。</returns>
public override bool NextToken(int state, int start)
{
// 最后一次匹配的符号和文本索引。
int startIndex = source.Index;
int lastAccept = -1, lastIndex = source.Index;
int symbolStart = 0, symbolEnd = 0;
while (true)
{
state = lexerData.NextState(state, source.Read());
if (state == -1)
{
// 没有合适的转移，退出。
break;
}
// 确定不是向前看的头状态。
if (lexerData.GetSymbols(state, ref symbolStart, ref symbolEnd) && states[symbolStart] >= 0)
{
lastAccept = states[symbolStart];
lastIndex = source.Index;
// 使用最短匹配时，可以直接返回。
if (lexerData.UseShortest && terminalData[lastAccept].UseShortest)
{
break;
}
}
}
PrintReadedText();
if (lastAccept >= 0)
{
TerminalData<T> terminal = terminalData[lastAccept];
if (terminal.Trailing.HasValue)
{
// 是向前看状态。
int index = terminal.Trailing.Value;
// 将流调整到与接受状态匹配的状态。
if (index > 0)
{
// 前面长度固定。
lastIndex = start + index;
}
else
{
// 后面长度固定，注意此时 index 是负数。
lastIndex += index;
}
}
// 将流调整到与接受状态匹配的状态。
DoAction(start, lastIndex, terminal);
Console.WriteLine(" Match {0}..{1} [{2}] {3}", startIndex, lastIndex, lastAccept,
terminal.Kind);
return true;
}
else
{
Console.WriteLine(" Match none");
}
return false;
}
}
150 changes: 145 additions & 5 deletions Runtime/Lexers/Core/LexerCore`1.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,28 +15,57 @@ internal abstract class LexerCore<T>
/// </summary>
/// <param name="lexerData">词法分析器数据。</param>
/// <param name="controller">词法分析器的控制器。</param>
/// <param name="debug">是否需要打印调试信息。</param>
/// <returns>词法分析器核心。</returns>
internal static LexerCore<T> Create(LexerData<T> lexerData, LexerController<T> controller)
internal static LexerCore<T> Create(LexerData<T> lexerData, LexerController<T> controller, bool debug)
{
if (lexerData.Rejectable)
{
if (lexerData.TrailingType == TrailingType.None)
{
return new RejectableCore<T>(lexerData, controller);
if (debug)
{
return new RejectableDebugCore<T>(lexerData, controller);
}
else
{
return new RejectableCore<T>(lexerData, controller);
}
}
}
else
{
if (lexerData.TrailingType == TrailingType.None)
{
return new BasicCore<T>(lexerData, controller);
if (debug)
{
return new BasicDebugCore<T>(lexerData, controller);
}
else
{
return new BasicCore<T>(lexerData, controller);
}
}
else if (lexerData.TrailingType == TrailingType.Fixed)
{
return new FixedTrailingCore<T>(lexerData, controller);
if (debug)
{
return new FixedTrailingDebugCore<T>(lexerData, controller);
}
else
{
return new FixedTrailingCore<T>(lexerData, controller);
}
}
}
return new RejectableTrailingCore<T>(lexerData, controller);
if (debug)
{
return new RejectableTrailingDebugCore<T>(lexerData, controller);
}
else
{
return new RejectableTrailingCore<T>(lexerData, controller);
}
}

/// <summary>
Expand Down Expand Up @@ -233,6 +262,49 @@ protected bool RunTerminals(int start)
return false;
}

/// <summary>
/// 执行已添加的终结符动作，并输出调试信息。
/// </summary>
/// <param name="start">当前词法单元的起始位置。</param>
/// <returns>词法单元读入是否成功。</returns>
protected bool RunTerminalsDebug(int start)
{
rejectedTerminals.Clear();
for (; terminalStackCount > 0; terminalStackCount--)
{
LexerStateInfo info = terminalStack[terminalStackCount - 1]!;
int sourceIndex = info.SourceIndex;
while (info.TerminalStart < info.TerminalEnd)
{
int terminalIndex = terminals[info.TerminalStart];
info.TerminalStart++;
if (rejectedTerminals.Contains(terminalIndex))
{
continue;
}
// 每次都需要清空候选集合，并在使用时重新计算。
isCandidatesValid = false;
DoAction(start, sourceIndex, terminalData[terminalIndex]);
if (controller.IsReject)
{
Console.WriteLine(" Match rejected {0}..{1} [{2}] {3}",
start, sourceIndex, terminalIndex, terminalData[terminalIndex].Kind);
}
else
{
Console.WriteLine(" Match {0}..{1} [{2}] {3}",
start, sourceIndex, terminalIndex, terminalData[terminalIndex].Kind);
return true;
}
if (controller.IsRejectState)
{
rejectedTerminals.Add(terminalIndex);
}
}
}
return false;
}

/// <summary>
/// 执行已添加的终结符动作。
/// </summary>
Expand Down Expand Up @@ -276,6 +348,56 @@ protected bool RunTerminalsWithTrailing(int start, int startSourceIndex)
return false;
}

/// <summary>
/// 执行已添加的终结符动作，并输出调试信息。
/// </summary>
/// <param name="start">当前词法单元的起始位置。</param>
/// <param name="startSourceIndex">当前源码的起始位置。</param>
/// <returns>词法单元读入是否成功。</returns>
protected bool RunTerminalsDebugWithTrailing(int start, int startSourceIndex)
{
rejectedTerminals.Clear();
for (; terminalStackCount > 0; terminalStackCount--)
{
LexerStateInfo info = terminalStack[terminalStackCount - 1]!;
int sourceIndex = info.SourceIndex;
while (info.TerminalStart < info.TerminalEnd)
{
int terminalIndex = terminals[info.TerminalStart];
info.TerminalStart++;
if (terminalIndex < 0)
{
// 跳过向前看的头状态。
break;
}
if (rejectedTerminals.Contains(terminalIndex))
{
continue;
}
int endIndex = GetTrailingIndex(terminalIndex, startSourceIndex, sourceIndex);
// 每次都需要清空候选集合，并在使用时重新计算。
isCandidatesValid = false;
DoAction(start, endIndex, terminalData[terminalIndex]);
if (controller.IsReject)
{
Console.WriteLine(" Match rejected {0}..{1} [{2}] {3}",
start, endIndex, terminalIndex, terminalData[terminalIndex].Kind);
}
else
{
Console.WriteLine(" Match {0}..{1} [{2}] {3}",
start, endIndex, terminalIndex, terminalData[terminalIndex].Kind);
return true;
}
if (controller.IsRejectState)
{
rejectedTerminals.Add(terminalIndex);
}
}
}
return false;
}

/// <summary>
/// 将候选类型设置为空。
/// </summary>
Expand Down Expand Up @@ -367,4 +489,22 @@ private bool ContainsTrailingHead(LexerStateInfo state, int target)
}
return false;
}

/// <summary>
/// 打印已读取的文本。
/// </summary>
protected void PrintReadedText()
{
StringView view = source.GetReadedText();
string text;
if (view.Length <= 50)
{
text = view.ToString();
}
else
{
text = view[..24] + ".." + view[^24];
}
Console.WriteLine("read {0}..{1}: \"{2}\"", source.Index - view.Length, source.Index, text);
}
}
Loading

0 comments on commit f65d5c7

Please sign in to comment.