lexer.py

import json

import ply.lex as lex
from ply import yacc
import logging

# 定义词法分析器的tokens列表
tokens = (
    'NUMBER',
    'IDENTIFIER',
    'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'DIVISIBLE',
    'LT', 'GT', 'NEQ', 'EQ',  # < > != ==
    'LTEQ', 'GTEQ',  # <= >=
    'AND', 'OR', 'NOT',  # and or not
    'DOUBLEQUOT',  # "
    'SINGLEQUOT',  # '
    'BRACELEFT',  # {
    'BRACERIGHT',  # }
    'BRACKETLEFT',  # [
    'BRACKETRIGHT',  # ]
    'PARENTHESELEFT',  # (
    'PARENTHESERIGHT',  # )
    'COMMA',  # ,
    'SEMICOLON',  # ;
    'COLON',  # :
    'DOT',  # .
    'WHITESPACE',  # \s
    'NUMBERSIGN',  # #
    'IF',  # ?
    'ELSEIF',  # !?
    'ELSE',  # !
    # 'WHILE',  # $
    'EQUALS',  # =
    'ENDLINE',
    'INDENT',
    'DEDENT',
    'WS',
    'ENDMARKER',
    'FOR',
    'AS',
    'ADDEQUALS',
    'SUBEQUALS',
    'MULEQUALS',
    'DIVEQUALS',
    # 选择器关键词
    'CHOOSE',
    'FROM',
    'WHERE',
)

states = (
    ('ws', 'exclusive'),  # 定义 ws 状态为 exclusive 状态
)


# 正则表达式需要匹配的是至少包含一个运算符（+ - * / ^ < > !=）的表达式，
# 并且运算符前后可以有任意数量的字母数字字符。这里我们使用正则表达式的非贪婪模式来匹配尽可能少的字符，
# 直到找到第一个运算符，然后匹配该运算符和其后的任意数量的字符。

# 规则用于识别数字
def t_NUMBER(t):
    r"""\d+"""
    t.value = int(t.value)  # 将匹配的数字字符串转换为整数
    return t


# 规则用于表示计算赋值
def t_ASSIGNMENT_OPERATOR(t):
    r"""[+\-*/]="""
    t.type = {'+=': 'ADDEQUALS', '-=': 'SUBEQUALS', '*=': 'MULEQUALS', '/=': 'DIVEQUALS'}[t.value]
    return t


# 运算操作符
def t_ARITHMETIC_OPERATOR(t):
    r"""[+\-*/]/*"""
    t.type = {'+': 'PLUS', '-': 'MINUS', '*': 'TIMES', '/': 'DIVIDE', '//': 'DIVISIBLE'}[t.value]
    return t


# 比较操作符
def t_COMPARISON_OPERATOR(t):
    """\s+(<>|!=|<=|>=|<|>|==|and|or|not)\s+"""
    if t.value in ['<>', '!=']:
        t.type = 'NEQ'
    if t.value == '==':
        t.type = 'EQ'
    elif t.value == '<':
        t.type = 'LT'
    elif t.value == '>':
        t.type = 'GT'
    elif t.value == '<=':
        t.type = 'LTEQ'
    elif t.value == '>=':
        t.type = 'GTEQ'
    elif t.value == 'and':
        t.type = 'AND'
    elif t.value == 'or':
        t.type = 'OR'
    elif t.value == 'not':
        t.type = 'NOT'
    return t


# 规则用于识别标识符
def t_IDENTIFIER(t):
    r"""[a-zA-Z\u4e00-\u9fa5_][a-zA-Z0-9\u4e00-\u9fa5_]*"""
    # 处理for和as关键字
    if t.value in ['for', 'as']:
        t.type = t.value.upper()
    # 处理选择器关键词
    elif t.value == 'choose':
        t.type = 'CHOOSE'
    elif t.value == 'from':
        t.type = 'FROM'
    elif t.value == 'where':
        t.type = 'WHERE'
    return t


indent_stack = [0]


def t_WS(t):
    r'[ \t]+'
    t.lexer.ws_len = len(t.value.expandtabs())
    t.lexer.ws_pos = t.lexer.lexpos
    t.lexer.ws_line = t.lexer.lineno
    t.lexer.begin('ws')
    return t


def t_ws_default(t):
    r'.'
    # 在 ws 状态下处理任何非空白字符
    t.lexer.begin('INITIAL')  # 切回 INITIAL 状态
    t.lexer.lexpos -= 1  # 回退一步，以便后续规则可以处理这个字符

# 添加 ws 状态下的错误处理函数
def t_ws_error(t):
    print(f"Illegal character '{t.value[0]}' in ws state")
    t.lexer.skip(1)

# 正则表达式规则
t_DOUBLEQUOT = r'"'
t_SINGLEQUOT = r"'"
t_BRACELEFT = r'\{'
t_BRACERIGHT = r'\}'
t_BRACKETLEFT = r'\['
t_BRACKETRIGHT = r'\]'
t_PARENTHESELEFT = r'\('
t_PARENTHESERIGHT = r'\)'
t_COMMA = r','
t_SEMICOLON = r';'
t_COLON = r':'
t_DOT = r'\.'
t_ENDLINE = r'\n'
t_NUMBERSIGN = r'\#'
t_IF = r'\?'
t_ELSEIF = r'\!\?'
t_ELSE = r'\!'
t_EQUALS = r'='

# 忽略的字符
t_ignore = ''

with open('tokens.txt', 'w') as f:
    f.write("start\n")

# 错误处理函数
def t_error(t):
    print(f"Illegal character '{t.value[0]}'")
    t.lexer.skip(1)


def t_newline(t):
    """
    处理换行符的函数。

    该函数用于识别和处理扫描器中的换行符。每遇到一个或多个换行符，它会更新当前行号。

    参数:
    - t: 一个Token对象，包含当前匹配到的换行符。

    返回值:
    无返回值，但会更新解析器的行号信息。
    """
    r'\n+'
    # 更新行号以反映换行符的数量
    t.lexer.lineno += len(t.value)


# Compute column.
#     input is the input text string
#     token is a token instance
def find_column(inputs, token):
    line_start = inputs.rfind('\n', 0, token.lexpos) + 1
    return (token.lexpos - line_start) + 1


def _new_token(type, value=None, lineno=None, lexpos=None):
    tok = lex.LexToken()
    tok.type = type
    tok.value = value
    tok.lineno = lineno
    tok.lexpos = lexpos
    return tok


def INDENT(lineno, lexpos):
    return _new_token('INDENT', None, lineno, lexpos)


def DEDENT(lineno, lexpos):
    return _new_token('DEDENT', None, lineno, lexpos)


def filter(lexer, add_endmarker=True):
    # 初始化缩进级别堆栈
    indent_levels = [0]
    current_indent = 0
    prev_was_ws = False

    # 获取 lexer 的 token 流
    tokens = list(iter(lexer.token, None))

    # 过滤并处理 token
    for i, token in enumerate(tokens):
        if token.type == "WS":
            assert prev_was_ws is False, "连续的 WS 不被允许"
            if token.value == " ":
                continue
            prev_was_ws = True
            current_indent = len(token.value.expandtabs())
            continue
        if token.type == "ENDLINE" and abs(current_indent - indent_levels[-1]) >= 2:
            prev_was_ws = False
            # 无论缩进如何变化，都返回 ENDLINE
            yield token
            if current_indent > indent_levels[-1]:
                # 缩进增加，产生 INDENT 事件
                indent_levels.append(current_indent)
                print("---INDENT---")
                yield INDENT(token.lineno, token.lexpos)
            elif current_indent < indent_levels[-1] and abs(current_indent - indent_levels[-1]) >= 2:
                # 缩进减少，产生 DEDENT 事件直到匹配当前缩进
                while current_indent < indent_levels[-1]:
                    if i + 1 < len(tokens) and tokens[i + 1].value[0] == " ":
                        break
                    yield DEDENT(token.lineno, token.lexpos)
                    indent_levels.pop()

            current_indent = 0
            continue

        # 处理其他 token
        if prev_was_ws:
            if current_indent > indent_levels[-1] and abs(current_indent - indent_levels[-1]) >= 4:
                # 缩进增加，产生 INDENT 事件
                print("---INDENTWS---")
                indent_levels.append(current_indent)
                yield INDENT(token.lineno, token.lexpos)
            elif current_indent < indent_levels[-1] and abs(current_indent - indent_levels[-1]) >= 4:
                # 缩进减少，产生 DEDENT 事件直到匹配当前缩进
                print("---DEDENTWS---")
                while current_indent < indent_levels[-1]:
                    yield DEDENT(token.lineno, token.lexpos)
                    indent_levels.pop()
            current_indent = 0

        prev_was_ws = False
        yield token

    # 在结束时，为剩余的缩进级别产生 DEDENT 事件
    if add_endmarker:
        if len(indent_levels) > 1:
            for _ in range(1, len(indent_levels)):
                yield DEDENT(token.lineno, token.lexpos)
            yield _new_token("ENDMARKER", token.lineno if token else 1)


# 构建词法分析器
lexer = lex.lex()

# 测试数据
with open("test1.txt", "r", encoding="utf-8") as f:
    data = f.read()

# 给词法分析器输入数据
lexer.input(data)

# 打印出所有的token和它们的值
lextokens = []
filtered_tokens = filter(lexer)
for token in filtered_tokens:
    print(token.type, token.value)

# processedTokens = []
# while i < len(lextokens):
#     if i < len(lextokens)-1 and lextokens[i + 1].type == 'EXPRESSION' and lextokens[i].type == 'EXPRESSION' :
#         if find_column(data, lextokens[i]) + len(lextokens[i].value) == find_column(data, lextokens[i + 1]):
#             lextokens[i].value += lextokens[i + 1].value
#             processedTokens.append(lextokens[i])
#             i += 2
#             continue
#     processedTokens.append(lextokens[i])
#     i += 1
# print(processedTokens)
# for token in processedTokens:
#     print(token.value)

precedence = (
    ('left', 'EQUALS', 'ADDEQUALS', 'SUBEQUALS', 'MULEQUALS', 'DIVEQUALS'),
    ('left', 'OR', 'AND', 'NOT'),
    ('left', 'EQ', 'NEQ', 'GT', 'LT', 'GTEQ', 'LTEQ'),
    ('left', 'DIVISIBLE'),
    ('left', 'PLUS', 'MINUS'),
    ('left', 'TIMES', 'DIVIDE'),
    ('right', 'UMINUS'),
    # ('left', 'DEDENT')
)


def p_binaryexpression(p):
    """binaryexpression : binaryexpression PLUS expression
                        | binaryexpression MINUS expression
                        | binaryexpression TIMES expression
                        | binaryexpression DIVIDE expression
                        | binaryexpression DIVISIBLE expression
                        | binaryexpression EQ expression
                        | binaryexpression NEQ expression
                        | binaryexpression GT expression
                        | binaryexpression LT expression
                        | binaryexpression GTEQ expression
                        | binaryexpression LTEQ expression
                        | binaryexpression AND expression
                        | binaryexpression OR expression
                        | binaryexpression NOT expression
                        | expression PLUS expression
                        | expression MINUS expression
                        | expression TIMES expression
                        | expression DIVIDE expression
                        | expression DIVISIBLE expression
                        | expression EQ expression
                        | expression NEQ expression
                        | expression GT expression
                        | expression LT expression
                        | expression GTEQ expression
                        | expression LTEQ expression
                        | expression AND expression
                        | expression OR expression
                        | expression NOT expression
                        """
    if len(p) == 2:
        p[0] = p[1]
    else:
        p[0] = {"type": 'binaryexpression', "left": p[1], "right": p[3], "operator": p[2]}
    print("binaryexpression >>>", p[0])


def p_expr_uminus(p):
    """expression : MINUS expression %prec UMINUS"""
    p[0] = {"type": 'expression', "value": -p[2]["value"]}


identifiers_funcs = ['boost', 'damage', 'onBattleGround', 'inGrave', 'inDeck',
                     'addStatus', 'destroy', 'banish', 'reset', 'getPower',
                     'getBasePower', 'getArmor', 'getCommand', 'setCommand',
                     'getCount', 'setCount', 'getTimer', 'setTimer', 'getPos',
                     'jumpTo', 'create', 'random', 'reveal', 'duel', 'clash',
                     'consume', 'drain', 'infuse', 'addWeather', 'clearWeather',
                     'flip', 'getCoins', 'setCoins']
identifiers_vars = ['originalDeck', 'deck',
                    'grave', 'round', 'turn', 'allTurn']
custom_functions = []
custom_vars = []


def p_tag(p):
    """tag : SINGLEQUOT IDENTIFIER SINGLEQUOT
           | SINGLEQUOT SINGLEQUOT"""
    if len(p) == 3:
        p[0] = {"type": 'tag', "value": ""}
    else:
        p[0] = {"type": 'tag', "value": p[2]}


def p_list(p):
    """list : choice
            | BRACKETLEFT BRACKETRIGHT"""
    if len(p) == 3:
        p[0] = {"type": 'list', "value": []}
    else:
        p[0] = {"type": 'list', "value": p[2]}


def p_name(p):
    """name : DOUBLEQUOT IDENTIFIER DOUBLEQUOT
            | DOUBLEQUOT DOUBLEQUOT"""
    if len(p) == 3:
        p[0] = {"type": 'name', "value": ""}
    else:
        p[0] = {"type": 'name', "value": p[2]}


def p_choice(p):
    """choice : BRACKETLEFT args BRACKETRIGHT"""
    p[0] = {"type": 'choice', "value": p[2]}


def p_member(p):
    """member : identifier choice"""
    print("MEMBER >>>", p[1])
    p[0] = {"type": 'member', "name": p[1], "choice": p[2]}


def p_identifier(p):
    """identifier : IDENTIFIER
                 | identifier DOT IDENTIFIER"""
    print("IDENTIFIER >>>", p[1])
    if len(p) == 2:
        p[0] = {"type": 'identifier', "name": p[1], "property": None}
    else:
        p[0] = {"type": 'identifier', "name": p[1]["name"], "property": p[3]}


def p_callfunction(p):
    """callfunction : identifier PARENTHESELEFT PARENTHESERIGHT
                    | identifier PARENTHESELEFT args PARENTHESERIGHT
                    | NUMBERSIGN identifier"""
    print("CALLFUNCTION >>>", p[1])
    if len(p) == 4:
        p[0] = {"type": 'callfunction', "caller": p[1], "args": []}
    elif len(p) == 5:
        p[0] = {"type": 'callfunction', "caller": p[1], "args": p[3]}
    elif len(p) == 3:
        p[0] = {"type": 'callfunction', "caller": "len", "args": [p[2]]}


def p_args(p):
    """args : expression
            | args COMMA expression"""
    if len(p) == 2:
        p[0] = [p[1]]
    else:
        p[0] = p[1] + [p[3]]

def p_number(p):
    """number : NUMBER"""
    p[0] = {"type": 'number', "value": p[1]}

def p_expression(p):
    """expression : identifier
                  | callfunction
                  | binaryexpression
                  | number
                  | tag
                  | name
                  | list
                  | member
                  | dotidchoices"""
    print("expression >>>", p[1])
    p[0] = {"type": 'expression', "value": p[1]}


def p_assignmentexpression(p):
    """assignmentexpression : identifier EQUALS expression
                            | identifier ADDEQUALS expression
                            | identifier SUBEQUALS expression
                            | identifier MULEQUALS expression
                            | identifier DIVEQUALS expression"""
    print("assignmentexpression >>>", p[1])
    p[0] = {"type": 'assignment', "left": p[1], "right": p[3], "operator": p[2]}
    if p[1] in identifiers_vars or p[1] in custom_vars:
        return
    custom_vars.append(p[1])


def p_subscribeevent(p):
    """subscribe : identifier suite
                 | member suite"""
    print("subscribe >>>", p[1])
    if p[1].get("type") != "member":
        p[0] = {"type": 'subscribe', "name": p[1], "args": [], "body": p[2]}
    else:
        p[0] = {"type": 'subscribe', "name": p[1]["name"], "args": p[1]["choice"], "body": p[2]}

    custom_vars.append(p[1])



def p_condition(p):
    """condition : expression IF suite
                 | condition ELSEIF expression suite
                 | condition ELSE suite"""

    if len(p) == 4 and p[2] == "?":
        p[0] = {"type": 'if', "condition": p[1], "consequent": p[3], "alternate": None}
    elif p[2] == '!':
        current_condition = p[1]
        # 直到alternate为空
        while current_condition["alternate"] is not None:
            current_condition = current_condition["alternate"]
        current_condition["alternate"] = {"type": 'if', "condition": None, "consequent": p[3], "alternate": None}
        p[0] = p[1]
    else:
        current_condition = p[1]
        # 直到alternate为空
        while current_condition["alternate"] is not None:
            current_condition = current_condition["alternate"]
        current_condition["alternate"] = {"type": 'if', "condition": p[3], "consequent": p[4], "alternate": None}
        p[0] = p[1]
    print("here>>", p[0])


def p_for(p):
    """for : FOR expression AS expression suite"""
    print(f"Parsing for loop with list {p[2]}, name {p[4]}, and body {p[5]}")
    p[0] = {"type": 'for', "list": p[2], "name": p[4], "body": p[5]}

def p_suite(p):
    """suite : ENDLINE INDENT codes DEDENT"""
    print("suite >>>", p[3])
    p[0] = p[3]

def p_dotidchoices(p):
    """dotidchoices : DOT identifier choice
                   | dotidchoices AND DOT identifier choice"""
    print("dotidchoice >>>", p)
    if len(p) == 4:  # DOT identifier choice
        p[0] = {"type": 'dotidchoice', "name": p[2], "property": p[3]}
    else:  # dotidchoice AND DOT identifier choice
        p[0] = {"type": 'dotidchoice', "name": p[1]["name"], "property": p[5]}

def p_expressionswithand(p):
    """
    expressionwithand : expression
    | expression AND expression
    | expressionwithand AND expression
    """
    if len(p) == 2:
        p[0] = p[1]
    elif len(p) == 4:
        p[0] = {"type": 'expressionswithand', "left": p[1], "right": p[3], "operator": p[2]}

def p_filter(p):
    """filter : BRACELEFT expressionwithand BRACERIGHT"""
    print("filter >>>", p[2])
    p[0] = {"type": 'filter', "value": p[2]}

def p_choose(p):
    """choose : CHOOSE expression FROM expression WHERE filter AS identifier COLON suite
                        | CHOOSE expression FROM expression AS identifier COLON suite"""
    if len(p) == 11:  # 带WHERE子句的完整形式 (10个元素，索引从0开始)
        filter_processed = p[6]
        print("filter value:", [str(list(p).index(n))+": "+str(n)+"\n" if n else "" for n in p])
        filter_processed["value"]["value"]["name"]["property"] = p[4]
        p[0] = {"type": 'choose', "num": p[2], "place": p[4], "filter": filter_processed, "var": p[8], "body": p[10]}
    else:  # 简化形式 (8个元素，索引从0开始)
        p[0] = {"type": 'choose', "num": p[2], "place": p[4], "filter": None, "var": p[6], "body": p[8]}

# def p_suite_error(p):
#     """suite : ENDLINE INDENT identifier error"""
#     print("suite error >>>", p[4])
#     p[0] = p[3]

def p_code(p):
    """code : assignmentexpression ENDLINE
            | callfunction ENDLINE
            | subscribe
            | condition
            | for
            | choose"""
    print(p[1]["type"])
    p[0] = p[1]


def p_codes(p):
    """codes : code
             | codes code
             """
    if len(p) == 2:
        p[0] = {'type': 'block', 'body': [p[1]]}
    else:
        p[0] = {'type': 'block', 'body': p[1]['body'] + [p[2]]}


def p_program(p):
    """program : codes
               | program codes
               | program ENDMARKER """
    print("program >>>", p[1])
    if len(p) == 2:
        p[0] = {"type": 'program', "body": p[1]}
    else:
        print("program >>>", p[1])
        p[0] = {"type": 'program', "body": p[1]["body"] + [p[2]]}


def p_error(p):
    if p:
        print("Syntax error at '%s'" % p.type)
    else:
        print("Syntax error at EOF")


class TokenStream:
    def __init__(self, tokens):
        self.tokens = iter(tokens)
        self.current_token = None

    def token(self):
        try:
            self.current_token = next(self.tokens)
            if self.current_token.lexpos is not None and self.current_token.lineno is not None:
                return self.current_token
            else:
                self.current_token.lexpos = 1
                self.current_token.lineno = 1
                return self.current_token
        except StopIteration:
            return None


class IndentLexer(object):

    def __init__(self, debug=0, optimize=0, lextab='lextab', reflags=0):
        self.lexer = lex.lex(debug=debug, optimize=optimize,
                             lextab=lextab, reflags=reflags)
        self.token_stream = None

    def input(self, s, add_endmarker=True):
        self.lexer.paren_count = 0
        self.lexer.input(s)
        # 将 filter 函数的输出传递给 TokenStream
        self.token_stream = TokenStream(filter(self.lexer, add_endmarker))

    def token(self):
        # 使用 TokenStream 来获取下一个词法单元
        token = self.token_stream.token()
        # with open('tokens.txt', 'a') as f:
        #     f.write(str(token)+"\n")
        if not token:
            return None
        if token.lexpos is not None and token.lineno is not None:
            print(token.lexpos, token.lineno)
            print(token)
        else:
            print("got token", token.type)
        return token
# 解析
def parseGC(data):
    pseudo_lexer = IndentLexer()
    pseudo_lexer.input(data)
    logging.basicConfig(
        level=logging.DEBUG,
        filename="./parselog.txt",
        filemode="w",
        format="%(filename)10s:%(lineno)4d:%(message)s"
    )
    log = logging.getLogger()

    # 构建语法分析器
    parser = yacc.yacc(
        debug=True,
        debuglog=log,
        errorlog=log,
        start='program',
        write_tables=True,
        tabmodule='parsetab',
    )
    result = parser.parse(lexer=pseudo_lexer, debug=log)

    return result