"""Mypy parser.

Constructs a parse tree (abstract syntax tree) based on a string
representing a source file. Performs only minimal semantic checks.
"""

import re

from typing import List, Tuple, Set, cast, Union, Optional

from mypy import lex
from mypy.lex import (
    Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit,
    UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit,
    EllipsisToken
)
from mypy.nodes import (
    MypyFile, Import, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef,
    ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, Statement,
    ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt,
    DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl,
    WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, Expression,
    TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr,
    DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr,
    FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr,
    UnaryExpr, FuncExpr, PrintStmt, ImportBase, ComparisonExpr,
    StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension,
    SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument,
    BackquoteExpr
)
from mypy import defaults
from mypy import nodes
from mypy.errors import Errors, CompileError
from mypy.types import Type, CallableType, AnyType, UnboundType
from mypy.parsetype import (
    parse_type, parse_types, parse_signature, TypeParseError
)
from mypy.options import Options

from mypy import experiments


precedence = {
    '**': 16,
    '-u': 15, '+u': 15, '~': 15,   # unary operators (-, + and ~)
    '<cast>': 14,
    '*': 13, '/': 13, '//': 13, '%': 13,
    '+': 12, '-': 12,
    '>>': 11, '<<': 11,
    '&': 10,
    '^': 9,
    '|': 8,
    '==': 7, '!=': 7, '<': 7, '>': 7, '<=': 7, '>=': 7, 'is': 7, 'in': 7,
    '*u': 7,  # unary * for star expressions
    'not': 6,
    'and': 5,
    'or': 4,
    '<if>': 3,  # conditional expression
    '<for>': 2,  # list comprehension
    ',': 1}


op_assign = set([
    '+=', '-=', '*=', '/=', '//=', '%=', '**=', '|=', '&=', '^=', '>>=',
    '<<='])

op_comp = set([
    '>', '<', '==', '>=', '<=', '<>', '!=', 'is', 'is', 'in', 'not'])

none = Token('')  # Empty token


def parse(source: Union[str, bytes],
          fnam: str,
          errors: Errors,
          options: Options) -> MypyFile:
    """Parse a source file, without doing any semantic analysis.

    Return the parse tree. If errors is not provided, raise ParseError
    on failure. Otherwise, use the errors object to report parse errors.

    The python_version (major, minor) option determines the Python syntax variant.
    """
    is_stub_file = bool(fnam) and fnam.endswith('.pyi')
    if options.fast_parser:
        if options.python_version[0] >= 3 or is_stub_file:
            import mypy.fastparse
            return mypy.fastparse.parse(source,
                                        fnam=fnam,
                                        errors=errors,
                                        pyversion=options.python_version,
                                        custom_typing_module=options.custom_typing_module)
        else:
            import mypy.fastparse2
            return mypy.fastparse2.parse(source,
                                         fnam=fnam,
                                         errors=errors,
                                         pyversion=options.python_version,
                                         custom_typing_module=options.custom_typing_module)

    parser = Parser(fnam,
                    errors,
                    options.python_version,
                    options.custom_typing_module,
                    is_stub_file=is_stub_file)
    tree = parser.parse(source)
    tree.path = fnam
    tree.is_stub = is_stub_file
    return tree


class Parser:
    """Mypy parser that parses a string into an AST.

    Parses type annotations in addition to basic Python syntax. It supports both Python 2 and 3
    (though Python 2 support is incomplete).

    The AST classes are defined in mypy.nodes and mypy.types.
    """

    tok = None  # type: List[Token]
    ind = 0
    errors = None  # type: Errors
    # If True, raise an exception on any parse error. Otherwise, errors are reported via 'errors'.
    raise_on_error = False

    # Are we currently parsing the body of a class definition?
    is_class_body = False
    # All import nodes encountered so far in this parse unit.
    imports = None  # type: List[ImportBase]
    # Names imported from __future__.
    future_options = None  # type: List[str]
    # Lines to ignore (using # type: ignore).
    ignored_lines = None  # type: Set[int]

    def __init__(self, fnam: str, errors: Errors, pyversion: Tuple[int, int],
                 custom_typing_module: str = None, is_stub_file: bool = False) -> None:
        self.raise_on_error = errors is None
        self.pyversion = pyversion
        self.custom_typing_module = custom_typing_module
        self.is_stub_file = is_stub_file
        if errors is not None:
            self.errors = errors
        else:
            self.errors = Errors()
        if fnam is not None:
            self.errors.set_file(fnam)
        else:
            self.errors.set_file('<input>')

    def parse(self, s: Union[str, bytes]) -> MypyFile:
        self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion,
                                               is_stub_file=self.is_stub_file)
        self.ind = 0
        self.imports = []
        self.future_options = []
        file = self.parse_file()
        if self.raise_on_error and self.errors.is_errors():
            self.errors.raise_error()
        return file

    def parse_file(self) -> MypyFile:
        """Parse a mypy source file."""
        is_bom = self.parse_bom()
        defs = self.parse_defs()
        self.expect_type(Eof)
        node = MypyFile(defs, self.imports, is_bom, self.ignored_lines)
        return node

    # Parse the initial part

    def parse_bom(self) -> bool:
        """Parse the optional byte order mark at the beginning of a file."""
        if isinstance(self.current(), Bom):
            self.expect_type(Bom)
            if isinstance(self.current(), Break):
                self.expect_break()
            return True
        else:
            return False

    def parse_import(self) -> Import:
        self.expect('import')
        ids = []
        while True:
            id = self.parse_qualified_name()
            translated = self.translate_module_id(id)
            as_id = None
            if self.current_str() == 'as':
                self.expect('as')
                name_tok = self.expect_type(Name)
                as_id = name_tok.string
            elif translated != id:
                as_id = id
            ids.append((translated, as_id))
            if self.current_str() != ',':
                break
            self.expect(',')
        node = Import(ids)
        self.imports.append(node)
        return node

    def translate_module_id(self, id: str) -> str:
        """Return the actual, internal module id for a source text id.

        For example, translate '__builtin__' in Python 2 to 'builtins'.
        """
        if id == self.custom_typing_module:
            return 'typing'
        elif id == '__builtin__' and self.pyversion[0] == 2:
            # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation
            #   is named __builtin__.py (there is another layer of translation elsewhere).
            return 'builtins'
        return id

    def parse_import_from(self) -> ImportBase:
        self.expect('from')

        # Build the list of beginning relative tokens.
        relative = 0
        while self.current_str() in (".", "..."):
            relative += len(self.current_str())
            self.skip()

        # Parse qualified name to actually import from.
        if self.current_str() == "import":
            # Empty/default values.
            name = ""
        else:
            name = self.parse_qualified_name()

        name = self.translate_module_id(name)

        # Parse import list
        self.expect('import')
        node = None  # type: ImportBase
        if self.current_str() == '*':
            if name == '__future__':
                self.parse_error()
            # An import all from a module node:
            self.skip()
            node = ImportAll(name, relative)
        else:
            is_paren = self.current_str() == '('
            if is_paren:
                self.expect('(')
            targets = []  # type: List[Tuple[str, str]]
            while True:
                id, as_id = self.parse_import_name()
                if '%s.%s' % (name, id) == self.custom_typing_module:
                    if targets or self.current_str() == ',':
                        self.fail('You cannot import any other modules when you '
                                  'import a custom typing module',
                                  self.current().line, self.current().column)
                    node = Import([('typing', as_id)])
                    self.skip_until_break()
                    break
                targets.append((id, as_id))
                if self.current_str() != ',':
                    break
                self.expect(',')
                if is_paren and self.current_str() == ')':
                    break
            if is_paren:
                self.expect(')')
            if node is None:
                node = ImportFrom(name, relative, targets)
        self.imports.append(node)
        if name == '__future__':
            self.future_options.extend(target[0] for target in targets)
        return node

    def parse_import_name(self) -> Tuple[str, Optional[str]]:
        tok = self.expect_type(Name)
        name = tok.string
        if self.current_str() == 'as':
            self.skip()
            as_name = self.expect_type(Name)
            return name, as_name.string
        else:
            return name, None

    def parse_qualified_name(self) -> str:
        """Parse a name with an optional module qualifier.

        Return a tuple with the name as a string and a token array
        containing all the components of the name.
        """
        tok = self.expect_type(Name)
        n = tok.string
        while self.current_str() == '.':
            self.expect('.')
            tok = self.expect_type(Name)
            n += '.' + tok.string
        return n

    # Parsing global definitions

    def parse_defs(self) -> List[Statement]:
        defs = []  # type: List[Statement]
        while not self.eof():
            try:
                defn, is_simple = self.parse_statement()
                if is_simple:
                    self.expect_break()
                if defn is not None:
                    if not self.try_combine_overloads(defn, defs):
                        defs.append(defn)
            except ParseError:
                pass
        return defs

    def parse_class_def(self) -> ClassDef:
        old_is_class_body = self.is_class_body
        self.is_class_body = True

        self.expect('class')
        metaclass = None

        try:
            base_types = []  # type: List[Expression]
            try:
                name_tok = self.expect_type(Name)
                name = name_tok.string

                self.errors.push_type(name)

                if self.current_str() == '(':
                    self.skip()
                    while True:
                        if self.current_str() == ')':
                            break
                        if self.peek().string == '=':
                            metaclass = self.parse_class_keywords()
                            break
                        base_types.append(self.parse_super_type())
                        if self.current_str() != ',':
                            break
                        self.skip()
                    self.expect(')')
            except ParseError:
                pass

            defs, _ = self.parse_block()

            node = ClassDef(name, defs, None, base_types, metaclass=metaclass)
            return node
        finally:
            self.errors.pop_type()
            self.is_class_body = old_is_class_body

    def parse_class_keywords(self) -> Optional[str]:
        """Parse the class keyword arguments, keeping the metaclass but
        ignoring all others.  Returns None if the metaclass isn't found.
        """
        metaclass = None
        while True:
            key = self.expect_type(Name)
            self.expect('=')
            if key.string == 'metaclass':
                metaclass = self.parse_qualified_name()
            else:
                # skip the class value
                self.parse_expression(precedence[','])
            if self.current_str() != ',':
                break
            self.skip()
            if self.current_str() == ')':
                break
        return metaclass

    def parse_super_type(self) -> Expression:
        return self.parse_expression(precedence[','])

    def parse_decorated_function_or_class(self) -> Union[Decorator, ClassDef]:
        decorators = []
        no_type_checks = False
        while self.current_str() == '@':
            self.expect('@')
            d_exp = self.parse_expression()
            if self.is_no_type_check_decorator(d_exp):
                no_type_checks = True
            decorators.append(d_exp)
            self.expect_break()
        if self.current_str() != 'class':
            func = self.parse_function(no_type_checks)
            func.is_decorated = True
            var = Var(func.name())
            # Types of decorated functions must always be inferred.
            var.is_ready = False
            var.set_line(decorators[0].line)
            node = Decorator(func, decorators, var)
            return node
        else:
            cls = self.parse_class_def()
            cls.decorators = decorators
            return cls

    def is_no_type_check_decorator(self, expr: Expression) -> bool:
        if isinstance(expr, NameExpr):
            return expr.name == 'no_type_check'
        elif isinstance(expr, MemberExpr):
            if isinstance(expr.expr, NameExpr):
                return expr.expr.name == 'typing' and expr.name == 'no_type_check'
        else:
            return False

    def parse_function(self, no_type_checks: bool = False) -> FuncDef:
        def_tok = self.expect('def')
        is_method = self.is_class_body
        self.is_class_body = False
        try:
            (name, args, typ, is_error, extra_stmts) = self.parse_function_header(no_type_checks)

            arg_kinds = [arg.kind for arg in args]
            arg_names = [arg.variable.name() for arg in args]

            body, comment_type = self.parse_block(allow_type=True)
            # Potentially insert extra assignment statements to the beginning of the
            # body, used to decompose Python 2 tuple arguments.
            body.body[:0] = extra_stmts
            if comment_type:
                # The function has a # type: ... signature.
                if typ:
                    self.errors.report(
                        def_tok.line, def_tok.column, 'Function has duplicate type signatures')
                sig = cast(CallableType, comment_type)
                if sig.is_ellipsis_args:
                    # When we encounter an ellipsis, fill in the arg_types with
                    # a bunch of AnyTypes, emulating Callable[..., T]
                    arg_types = [AnyType()] * len(arg_kinds)  # type: List[Type]
                    typ = CallableType(
                        arg_types,
                        arg_kinds,
                        arg_names,
                        sig.ret_type,
                        None,
                        line=def_tok.line,
                        column=def_tok.column)
                elif is_method and len(sig.arg_kinds) < len(arg_kinds):
                    self.check_argument_kinds(arg_kinds,
                                              [nodes.ARG_POS] + sig.arg_kinds,
                                              def_tok.line, def_tok.column)
                    # Add implicit 'self' argument to signature.
                    first_arg = [AnyType()]  # type: List[Type]
                    typ = CallableType(
                        first_arg + sig.arg_types,
                        arg_kinds,
                        arg_names,
                        sig.ret_type,
                        None,
                        line=def_tok.line,
                        column=def_tok.column)
                else:
                    self.check_argument_kinds(arg_kinds, sig.arg_kinds,
                                              def_tok.line, def_tok.column)
                    typ = CallableType(
                        sig.arg_types,
                        arg_kinds,
                        arg_names,
                        sig.ret_type,
                        None,
                        line=def_tok.line,
                        column=def_tok.column)

            # If there was a serious error, we really cannot build a parse tree
            # node.
            if is_error:
                return None

            if typ:
                for arg, arg_type in zip(args, typ.arg_types):
                    self.set_type_optional(arg_type, arg.initializer)

            if typ and isinstance(typ.ret_type, UnboundType):
                typ.ret_type.is_ret_type = True

            node = FuncDef(name, args, body, typ)
            node.set_line(def_tok)
            if typ is not None:
                typ.definition = node
            return node
        finally:
            self.errors.pop_function()
            self.is_class_body = is_method

    def check_argument_kinds(self, funckinds: List[int], sigkinds: List[int],
                             line: int, column: int) -> None:
        """Check that arguments are consistent.

        This verifies that they have the same number and the kinds correspond.

        Arguments:
          funckinds: kinds of arguments in function definition
          sigkinds:  kinds of arguments in signature (after # type:)
        """
        if len(funckinds) != len(sigkinds):
            if len(funckinds) > len(sigkinds):
                self.fail("Type signature has too few arguments", line, column)
            else:
                self.fail("Type signature has too many arguments", line, column)
            return
        for kind, token in [(nodes.ARG_STAR, '*'),
                            (nodes.ARG_STAR2, '**')]:
            if ((funckinds.count(kind) != sigkinds.count(kind)) or
                    (kind in funckinds and sigkinds.index(kind) != funckinds.index(kind))):
                self.fail(
                    "Inconsistent use of '{}' in function "
                    "signature".format(token), line, column)

    def parse_function_header(
            self, no_type_checks: bool=False) -> Tuple[str,
                                                       List[Argument],
                                                       CallableType,
                                                       bool,
                                                       List[AssignmentStmt]]:
        """Parse function header (a name followed by arguments)

        Return a 5-tuple with the following items:
          name
          arguments
          signature (annotation)
          error flag (True if error)
          extra statements needed to decompose arguments (usually empty)

        See parse_arg_list for an explanation of the final tuple item.
        """
        name = ''

        try:
            name_tok = self.expect_type(Name)
            name = name_tok.string

            self.errors.push_function(name)

            args, typ, extra_stmts = self.parse_args(no_type_checks)
        except ParseError:
            if not isinstance(self.current(), Break):
                self.ind -= 1  # Kludge: go back to the Break token
            # Resynchronise parsing by going back over :, if present.
            if isinstance(self.tok[self.ind - 1], Colon):
                self.ind -= 1
            return (name, [], None, True, [])

        return (name, args, typ, False, extra_stmts)

    def parse_args(self, no_type_checks: bool=False) -> Tuple[List[Argument],
                                                              CallableType,
                                                              List[AssignmentStmt]]:
        """Parse a function signature (...) [-> t].

        See parse_arg_list for an explanation of the final tuple item.
        """
        lparen = self.expect('(')

        # Parse the argument list (everything within '(' and ')').
        args, extra_stmts = self.parse_arg_list(no_type_checks=no_type_checks)

        self.expect(')')

        if self.current_str() == '->':
            self.skip()
            if no_type_checks:
                self.parse_expression()
                ret_type = None
            else:
                ret_type = self.parse_type()
        else:
            ret_type = None

        arg_kinds = [arg.kind for arg in args]
        self.verify_argument_kinds(arg_kinds, lparen.line, lparen.column)

        annotation = self.build_func_annotation(
            ret_type, args, lparen.line, lparen.column)

        return args, annotation, extra_stmts

    def build_func_annotation(self, ret_type: Type, args: List[Argument],
            line: int, column: int, is_default_ret: bool = False) -> CallableType:
        arg_types = [arg.type_annotation for arg in args]
        # Are there any type annotations?
        if ((ret_type and not is_default_ret)
                or arg_types != [None] * len(arg_types)):
            # Yes. Construct a type for the function signature.
            return self.construct_function_type(args, ret_type, line, column)
        else:
            return None

    def parse_arg_list(self, allow_signature: bool = True,
            no_type_checks: bool=False) -> Tuple[List[Argument],
                                                 List[AssignmentStmt]]:
        """Parse function definition argument list.

        This includes everything between '(' and ')' (but not the
        parentheses).

        Return tuple (arguments,
                      extra statements for decomposing arguments).

        The final argument is only used for Python 2 argument lists with
        tuples; they contain destructuring assignment statements used to
        decompose tuple arguments. For example, consider a header like this:

        . def f((x, y))

        The actual (sole) argument will be __tuple_arg_1 (a generated
        name), whereas the extra statement list will contain a single
        assignment statement corresponding to this assignment:

          x, y = __tuple_arg_1
        """
        args = []  # type: List[Argument]
        extra_stmts = []
        # This is for checking duplicate argument names.
        arg_names = []  # type: List[str]
        has_tuple_arg = False

        require_named = False
        bare_asterisk_before = -1

        if self.current_str() != ')' and self.current_str() != ':':
            while self.current_str() != ')':
                if self.current_str() == '*' and self.peek().string == ',':
                    self.expect('*')
                    require_named = True
                    bare_asterisk_before = len(args)
                elif self.current_str() in ['*', '**']:
                    if bare_asterisk_before == len(args):
                        # named arguments must follow bare *
                        self.parse_error()

                    arg = self.parse_asterisk_arg(
                        allow_signature,
                        no_type_checks,
                    )
                    args.append(arg)
                    require_named = True
                elif self.current_str() == '(':
                    arg, extra_stmt, names = self.parse_tuple_arg(len(args))
                    args.append(arg)
                    if extra_stmt is not None:
                        extra_stmts.append(extra_stmt)
                        has_tuple_arg = True
                    arg_names.extend(names)
                else:
                    arg, require_named = self.parse_normal_arg(
                        require_named,
                        allow_signature,
                        no_type_checks,
                    )
                    args.append(arg)
                    arg_names.append(arg.variable.name())

                if self.current().string != ',':
                    break

                self.expect(',')

        # Non-tuple argument dupes will be checked elsewhere. Avoid
        # generating duplicate errors.
        if has_tuple_arg:
            self.check_duplicate_argument_names(arg_names)

        return args, extra_stmts

    def check_duplicate_argument_names(self, names: List[str]) -> None:
        found = set()  # type: Set[str]
        for name in names:
            if name in found:
                self.fail('Duplicate argument name "{}"'.format(name),
                          self.current().line, self.current().column)
            found.add(name)

    def parse_asterisk_arg(self,
            allow_signature: bool,
            no_type_checks: bool) -> Argument:
        asterisk = self.skip()
        name = self.expect_type(Name)
        variable = Var(name.string)
        if asterisk.string == '*':
            kind = nodes.ARG_STAR
        else:
            kind = nodes.ARG_STAR2

        type = None
        if no_type_checks:
            self.parse_parameter_annotation()
        else:
            type = self.parse_arg_type(allow_signature)

        return Argument(variable, type, None, kind)

    def parse_tuple_arg(self, index: int) -> Tuple[Argument, AssignmentStmt, List[str]]:
        """Parse a single Python 2 tuple argument.

        Example: def f(x, (y, z)): ...

        The tuple arguments gets transformed into an assignment in the
        function body (the second return value).

        Return tuple (argument, decomposing assignment, list of names defined).

        Special case: if the argument is just (x) then it's not a tuple;
        we indicate this by returning (argument, None, ['x']).
        However, if the argument is (x,) then it *is* a (singleton) tuple.
        """
        line = self.current().line
        column = self.current().column
        # Generate a new argument name that is very unlikely to clash with anything.
        arg_name = '__tuple_arg_{}'.format(index + 1)
        if self.pyversion[0] >= 3:
            self.fail('Tuples in argument lists only supported in Python 2 mode', line, column)
        paren_arg = self.parse_parentheses()
        self.verify_tuple_arg(paren_arg)
        if isinstance(paren_arg, NameExpr):
            # This isn't a tuple. Revert to a normal argument.
            arg_name = paren_arg.name
            decompose = None
        else:
            rvalue = NameExpr(arg_name)
            rvalue.set_line(line)
            decompose = AssignmentStmt([paren_arg], rvalue)
            decompose.set_line(line, column)
        kind = nodes.ARG_POS
        initializer = None
        if self.current_str() == '=':
            self.expect('=')
            initializer = self.parse_expression(precedence[','])
            kind = nodes.ARG_OPT
        var = Var(arg_name)
        arg_names = self.find_tuple_arg_argument_names(paren_arg)
        return Argument(var, None, initializer, kind), decompose, arg_names

    def verify_tuple_arg(self, paren_arg: Expression) -> None:
        if isinstance(paren_arg, TupleExpr):
            if not paren_arg.items:
                self.fail('Empty tuple not valid as an argument', paren_arg.line, paren_arg.column)
            for item in paren_arg.items:
                self.verify_tuple_arg(item)
        elif not isinstance(paren_arg, NameExpr):
            self.fail('Invalid item in tuple argument', paren_arg.line, paren_arg.column)

    def find_tuple_arg_argument_names(self, node: Expression) -> List[str]:
        result = []  # type: List[str]
        if isinstance(node, TupleExpr):
            for item in node.items:
                result.extend(self.find_tuple_arg_argument_names(item))
        elif isinstance(node, NameExpr):
            result.append(node.name)
        return result

    def parse_normal_arg(self, require_named: bool,
            allow_signature: bool,
            no_type_checks: bool) -> Tuple[Argument, bool]:
        name = self.expect_type(Name)
        variable = Var(name.string)

        type = None
        if no_type_checks:
            self.parse_parameter_annotation()
        else:
            type = self.parse_arg_type(allow_signature)

        initializer = None  # type: Expression
        if self.current_str() == '=':
            self.expect('=')
            initializer = self.parse_expression(precedence[','])
            if require_named:
                kind = nodes.ARG_NAMED
            else:
                kind = nodes.ARG_OPT
        else:
            if require_named:
                kind = nodes.ARG_NAMED
            else:
                kind = nodes.ARG_POS

        return Argument(variable, type, initializer, kind), require_named

    def set_type_optional(self, type: Type, initializer: Expression) -> None:
        if not experiments.STRICT_OPTIONAL:
            return
        # Indicate that type should be wrapped in an Optional if arg is initialized to None.
        optional = isinstance(initializer, NameExpr) and initializer.name == 'None'
        if isinstance(type, UnboundType):
            type.optional = optional

    def parse_parameter_annotation(self) -> Expression:
        if self.current_str() == ':':
            self.skip()
            return self.parse_expression(precedence[','])

    def parse_arg_type(self, allow_signature: bool) -> Type:
        if self.current_str() == ':' and allow_signature:
            self.skip()
            return self.parse_type()
        else:
            return None

    def verify_argument_kinds(self, kinds: List[int], line: int, column: int) -> None:
        found = set()  # type: Set[int]
        for i, kind in enumerate(kinds):
            if kind == nodes.ARG_POS and found & set([nodes.ARG_OPT,
                                                      nodes.ARG_STAR,
                                                      nodes.ARG_STAR2]):
                self.fail('Invalid argument list', line, column)
            elif kind == nodes.ARG_STAR and nodes.ARG_STAR in found:
                self.fail('Invalid argument list', line, column)
            elif kind == nodes.ARG_STAR2 and i != len(kinds) - 1:
                self.fail('Invalid argument list', line, column)
            found.add(kind)

    def construct_function_type(self, args: List[Argument], ret_type: Type,
                                line: int, column: int) -> CallableType:
        # Complete the type annotation by replacing omitted types with 'Any'.
        arg_types = [arg.type_annotation for arg in args]
        for i in range(len(arg_types)):
            if arg_types[i] is None:
                arg_types[i] = AnyType(implicit=True)
        if ret_type is None:
            ret_type = AnyType(implicit=True)
        arg_kinds = [arg.kind for arg in args]
        arg_names = [arg.variable.name() for arg in args]
        return CallableType(arg_types, arg_kinds, arg_names, ret_type, None, name=None,
                        variables=None, line=line, column=column)

    # Parsing statements

    def parse_block(self, allow_type: bool = False) -> Tuple[Block, Type]:
        colon = self.expect(':')
        if not isinstance(self.current(), Break):
            # Block immediately after ':'.
            nodes = []
            while True:
                ind = self.ind
                stmt, is_simple = self.parse_statement()
                if not is_simple:
                    self.parse_error_at(self.tok[ind])
                    break
                nodes.append(stmt)
                brk = self.expect_break()
                if brk.string != ';':
                    break
            node = Block(nodes)
            node.set_line(colon)
            return node, None
        else:
            # Indented block.
            brk = self.expect_break()
            type = self.parse_type_comment(brk, signature=True)
            self.expect_indent()
            stmt_list = []  # type: List[Statement]
            while (not isinstance(self.current(), Dedent) and
                   not isinstance(self.current(), Eof)):
                try:
                    stmt, is_simple = self.parse_statement()
                    if is_simple:
                        self.expect_break()
                    if stmt is not None:
                        if not self.try_combine_overloads(stmt, stmt_list):
                            stmt_list.append(stmt)
                except ParseError:
                    pass
            if isinstance(self.current(), Dedent):
                self.skip()
            node = Block(stmt_list)
            node.set_line(colon)
            return node, type

    def try_combine_overloads(self, s: Statement, stmt: List[Statement]) -> bool:
        if isinstance(s, Decorator) and stmt:
            fdef = s
            n = fdef.func.name()
            if isinstance(stmt[-1], Decorator) and stmt[-1].func.name() == n:
                stmt[-1] = OverloadedFuncDef([stmt[-1], fdef])
                return True
            elif isinstance(stmt[-1], OverloadedFuncDef) and stmt[-1].name() == n:
                stmt[-1].items.append(fdef)
                return True
        return False

    def parse_statement(self) -> Tuple[Statement, bool]:
        stmt = None  # type: Statement
        t = self.current()
        ts = self.current_str()
        is_simple = True  # Is this a non-block statement?
        if ts == 'if':
            stmt = self.parse_if_stmt()
            is_simple = False
        elif ts == 'def':
            stmt = self.parse_function()
            is_simple = False
        elif ts == 'while':
            stmt = self.parse_while_stmt()
            is_simple = False
        elif ts == 'return':
            stmt = self.parse_return_stmt()
        elif ts == 'for':
            stmt = self.parse_for_stmt()
            is_simple = False
        elif ts == 'try':
            stmt = self.parse_try_stmt()
            is_simple = False
        elif ts == 'break':
            stmt = self.parse_break_stmt()
        elif ts == 'continue':
            stmt = self.parse_continue_stmt()
        elif ts == 'pass':
            stmt = self.parse_pass_stmt()
        elif ts == 'raise':
            stmt = self.parse_raise_stmt()
        elif ts == 'import':
            stmt = self.parse_import()
        elif ts == 'from':
            stmt = self.parse_import_from()
        elif ts == 'class':
            stmt = self.parse_class_def()
            is_simple = False
        elif ts == 'global':
            stmt = self.parse_global_decl()
        elif ts == 'nonlocal' and self.pyversion[0] >= 3:
            stmt = self.parse_nonlocal_decl()
        elif ts == 'assert':
            stmt = self.parse_assert_stmt()
        elif ts == 'del':
            stmt = self.parse_del_stmt()
        elif ts == 'with':
            stmt = self.parse_with_stmt()
            is_simple = False
        elif ts == '@':
            stmt = self.parse_decorated_function_or_class()
            is_simple = False
        elif ts == 'print' and (self.pyversion[0] == 2 and
                                'print_function' not in self.future_options):
            stmt = self.parse_print_stmt()
        elif ts == 'exec' and self.pyversion[0] == 2:
            stmt = self.parse_exec_stmt()
        else:
            stmt = self.parse_expression_or_assignment()
        if ts == 'async' and self.current_str() == 'def':
            self.parse_error_at(self.current(),
                                reason='Use --fast-parser to parse code using "async def"')
            raise ParseError()
        if stmt is not None:
            stmt.set_line(t)
        return stmt, is_simple

    def parse_expression_or_assignment(self) -> Union[AssignmentStmt,
                                                      OperatorAssignmentStmt,
                                                      ExpressionStmt]:
        expr = self.parse_expression(star_expr_allowed=True)
        if self.current_str() == '=':
            return self.parse_assignment(expr)
        elif self.current_str() in op_assign:
            # Operator assignment statement.
            op = self.current_str()[:-1]
            self.skip()
            rvalue = self.parse_expression()
            return OperatorAssignmentStmt(op, expr, rvalue)
        else:
            # Expression statement.
            return ExpressionStmt(expr)

    def parse_assignment(self, lvalue: Expression) -> AssignmentStmt:
        """Parse an assignment statement.

        Assume that lvalue has been parsed already, and the current token is '='.
        Also parse an optional '# type:' comment.
        """
        self.expect('=')
        lvalues = [lvalue]
        expr = self.parse_expression(star_expr_allowed=True)
        while self.current_str() == '=':
            self.skip()
            lvalues.append(expr)
            expr = self.parse_expression(star_expr_allowed=True)
        cur = self.current()
        if isinstance(cur, Break):
            type = self.parse_type_comment(cur, signature=False)
        else:
            type = None
        return AssignmentStmt(lvalues, expr, type)

    def parse_return_stmt(self) -> ReturnStmt:
        self.expect('return')
        expr = None
        current = self.current()
        if current.string == 'yield':
            self.parse_error()
        if not isinstance(current, Break):
            expr = self.parse_expression()
        node = ReturnStmt(expr)
        return node

    def parse_raise_stmt(self) -> RaiseStmt:
        self.expect('raise')
        expr = None
        from_expr = None
        if not isinstance(self.current(), Break):
            expr = self.parse_expression()
            if self.current_str() == 'from':
                self.expect('from')
                from_expr = self.parse_expression()
        node = RaiseStmt(expr, from_expr)
        return node

    def parse_assert_stmt(self) -> AssertStmt:
        self.expect('assert')
        expr = self.parse_expression()
        node = AssertStmt(expr)
        return node

    def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]:
        self.expect("yield")
        expr = None
        node = YieldExpr(expr)  # type: Union[YieldFromExpr, YieldExpr]
        if not isinstance(self.current(), Break):
            if self.current_str() == "from":
                self.expect("from")
                expr = self.parse_expression()  # when yield from is assigned to a variable
                node = YieldFromExpr(expr)
            else:
                if self.current_str() == ')':
                    node = YieldExpr(None)
                else:
                    expr = self.parse_expression()
                    node = YieldExpr(expr)
        return node

    def parse_ellipsis(self) -> EllipsisExpr:
        self.expect('...')
        node = EllipsisExpr()
        return node

    def parse_del_stmt(self) -> DelStmt:
        self.expect('del')
        expr = self.parse_expression()
        node = DelStmt(expr)
        return node

    def parse_break_stmt(self) -> BreakStmt:
        self.expect('break')
        node = BreakStmt()
        return node

    def parse_continue_stmt(self) -> ContinueStmt:
        self.expect('continue')
        node = ContinueStmt()
        return node

    def parse_pass_stmt(self) -> PassStmt:
        self.expect('pass')
        node = PassStmt()
        return node

    def parse_global_decl(self) -> GlobalDecl:
        self.expect('global')
        names = self.parse_identifier_list()
        node = GlobalDecl(names)
        return node

    def parse_nonlocal_decl(self) -> NonlocalDecl:
        self.expect('nonlocal')
        names = self.parse_identifier_list()
        node = NonlocalDecl(names)
        return node

    def parse_identifier_list(self) -> List[str]:
        names = []
        while True:
            n = self.expect_type(Name)
            names.append(n.string)
            if self.current_str() != ',':
                break
            self.skip()
        return names

    def parse_while_stmt(self) -> WhileStmt:
        is_error = False
        self.expect('while')
        try:
            expr = self.parse_expression()
        except ParseError:
            is_error = True
        body, _ = self.parse_block()
        if self.current_str() == 'else':
            self.expect('else')
            else_body, _ = self.parse_block()
        else:
            else_body = None
        if is_error is not None:
            node = WhileStmt(expr, body, else_body)
            return node
        else:
            return None

    def parse_for_stmt(self) -> ForStmt:
        self.expect('for')
        index = self.parse_for_index_variables()
        self.expect('in')
        expr = self.parse_expression()

        body, _ = self.parse_block()

        if self.current_str() == 'else':
            self.expect('else')
            else_body, _ = self.parse_block()
        else:
            else_body = None

        node = ForStmt(index, expr, body, else_body)
        return node

    def parse_for_index_variables(self) -> Expression:
        # Parse index variables of a 'for' statement.
        index_items = []
        force_tuple = False

        while True:
            v = self.parse_expression(precedence['in'],
                                      star_expr_allowed=True)  # Prevent parsing of for stmt 'in'
            index_items.append(v)
            if self.current_str() != ',':
                break
            self.skip()
            if self.current_str() == 'in':
                force_tuple = True
                break

        if len(index_items) == 1 and not force_tuple:
            index = index_items[0]
        else:
            index = TupleExpr(index_items)
            index.set_line(index_items[0])

        return index

    def parse_if_stmt(self) -> IfStmt:
        is_error = False

        self.expect('if')
        expr = []
        try:
            expr.append(self.parse_expression())
        except ParseError:
            is_error = True

        body = [self.parse_block()[0]]

        while self.current_str() == 'elif':
            self.expect('elif')
            try:
                expr.append(self.parse_expression())
            except ParseError:
                is_error = True
            body.append(self.parse_block()[0])

        if self.current_str() == 'else':
            self.expect('else')
            else_body, _ = self.parse_block()
        else:
            else_body = None

        if not is_error:
            node = IfStmt(expr, body, else_body)
            return node
        else:
            return None

    def parse_try_stmt(self) -> TryStmt:
        self.expect('try')
        body, _ = self.parse_block()
        is_error = False
        vars = []  # type: List[NameExpr]
        types = []  # type: List[Optional[Expression]]
        handlers = []  # type: List[Block]
        while self.current_str() == 'except':
            self.expect('except')
            if not isinstance(self.current(), Colon):
                try:
                    t = self.current()
                    expr = self.parse_expression(precedence[','])
                    expr.set_line(t)
                    types.append(expr)
                    if self.current_str() == 'as':
                        self.expect('as')
                        vars.append(self.parse_name_expr())
                    elif self.pyversion[0] == 2 and self.current_str() == ',':
                        self.expect(',')
                        vars.append(self.parse_name_expr())
                    else:
                        vars.append(None)
                except ParseError:
                    is_error = True
            else:
                types.append(None)
                vars.append(None)
            handlers.append(self.parse_block()[0])
        if not is_error:
            if self.current_str() == 'else':
                self.skip()
                else_body, _ = self.parse_block()
            else:
                else_body = None
            if self.current_str() == 'finally':
                self.expect('finally')
                finally_body, _ = self.parse_block()
            else:
                finally_body = None
            node = TryStmt(body, vars, types, handlers, else_body,
                           finally_body)
            return node
        else:
            return None

    def parse_with_stmt(self) -> WithStmt:
        self.expect('with')
        exprs = []
        targets = []
        while True:
            expr = self.parse_expression(precedence[','])
            if self.current_str() == 'as':
                self.expect('as')
                target = self.parse_expression(precedence[','])
            else:
                target = None
            exprs.append(expr)
            targets.append(target)
            if self.current_str() != ',':
                break
            self.expect(',')
        body, _ = self.parse_block()
        return WithStmt(exprs, targets, body)

    def parse_print_stmt(self) -> PrintStmt:
        self.expect('print')
        args = []
        target = None
        if self.current_str() == '>>':
            self.skip()
            target = self.parse_expression(precedence[','])
            if self.current_str() == ',':
                self.skip()
                if isinstance(self.current(), Break):
                    self.parse_error()
            else:
                if not isinstance(self.current(), Break):
                    self.parse_error()
        comma = False
        while not isinstance(self.current(), Break):
            args.append(self.parse_expression(precedence[',']))
            if self.current_str() == ',':
                comma = True
                self.skip()
            else:
                comma = False
                break
        return PrintStmt(args, newline=not comma, target=target)

    def parse_exec_stmt(self) -> ExecStmt:
        self.expect('exec')
        expr = self.parse_expression(precedence['in'])
        variables1 = None
        variables2 = None
        if self.current_str() == 'in':
            self.skip()
            variables1 = self.parse_expression(precedence[','])
            if self.current_str() == ',':
                self.skip()
                variables2 = self.parse_expression(precedence[','])
        return ExecStmt(expr, variables1, variables2)

    # Parsing expressions

    def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> Expression:
        """Parse a subexpression within a specific precedence context."""
        expr = None  # type: Expression
        current = self.current()  # Remember token for setting the line number.

        # Parse a "value" expression or unary operator expression and store
        # that in expr.
        s = self.current_str()
        if s == '(':
            # Parenthesised expression or cast.
            expr = self.parse_parentheses()
        elif s == '[':
            expr = self.parse_list_expr()
        elif s in ['-', '+', 'not', '~']:
            # Unary operation.
            expr = self.parse_unary_expr()
        elif s == 'lambda':
            expr = self.parse_lambda_expr()
        elif s == '{':
            expr = self.parse_dict_or_set_expr()
        elif s == '*' and star_expr_allowed:
            expr = self.parse_star_expr()
        elif s == '`' and self.pyversion[0] == 2:
            expr = self.parse_backquote_expr()
        else:
            if isinstance(current, Name):
                # Name expression.
                expr = self.parse_name_expr()
            elif isinstance(current, IntLit):
                expr = self.parse_int_expr()
            elif isinstance(current, StrLit):
                expr = self.parse_str_expr()
            elif isinstance(current, BytesLit):
                expr = self.parse_bytes_literal()
            elif isinstance(current, UnicodeLit):
                expr = self.parse_unicode_literal()
            elif isinstance(current, FloatLit):
                expr = self.parse_float_expr()
            elif isinstance(current, ComplexLit):
                expr = self.parse_complex_expr()
            elif isinstance(current, Keyword) and s == "yield":
                # The expression yield from and yield to assign
                expr = self.parse_yield_or_yield_from_expr()
            elif isinstance(current, EllipsisToken) and (self.pyversion[0] >= 3
                                                         or self.is_stub_file):
                expr = self.parse_ellipsis()
            else:
                # Invalid expression.
                self.parse_error()

        # Set the line of the expression node, if not specified. This
        # simplifies recording the line number as not every node type needs to
        # deal with it separately.
        if expr.line < 0:
            expr.set_line(current)

        # Parse operations that require a left argument (stored in expr).
        while True:
            current = self.current()
            s = self.current_str()
            if s == '(':
                # Call expression.
                expr = self.parse_call_expr(expr)
            elif s == '.':
                # Member access expression.
                expr = self.parse_member_expr(expr)
            elif s == '[':
                # Indexing expression.
                expr = self.parse_index_expr(expr)
            elif s == ',':
                # The comma operator is used to build tuples. Comma also
                # separates array items and function arguments, but in this
                # case the precedence is too low to build a tuple.
                if precedence[','] > prec:
                    expr = self.parse_tuple_expr(expr)
                else:
                    break
            elif s == 'for':
                if precedence['<for>'] > prec:
                    # List comprehension or generator expression. Parse as
                    # generator expression; it will be converted to list
                    # comprehension if needed elsewhere.
                    expr = self.parse_generator_expr(expr)
                else:
                    break
            elif s == 'if':
                # Conditional expression.
                if precedence['<if>'] > prec:
                    expr = self.parse_conditional_expr(expr)
                else:
                    break
            else:
                # Binary operation or a special case.
                if isinstance(current, Op):
                    op = self.current_str()
                    op_prec = precedence[op]
                    if op == 'not':
                        # Either "not in" or an error.
                        op_prec = precedence['in']
                    if op_prec > prec:
                        if op in op_comp:
                            expr = self.parse_comparison_expr(expr, op_prec)
                        else:
                            expr = self.parse_bin_op_expr(expr, op_prec)
                    else:
                        # The operation cannot be associated with the
                        # current left operand due to the precedence
                        # context; let the caller handle it.
                        break
                else:
                    # Not an operation that accepts a left argument; let the
                    # caller handle the rest.
                    break

            # Set the line of the expression node, if not specified. This
            # simplifies recording the line number as not every node type
            # needs to deal with it separately.
            if expr.line < 0:
                expr.set_line(current)

        return expr

    def parse_parentheses(self) -> Expression:
        self.skip()
        if self.current_str() == ')':
            # Empty tuple ().
            expr = self.parse_empty_tuple_expr()  # type: Expression
        else:
            # Parenthesised expression.
            expr = self.parse_expression(0, star_expr_allowed=True)
            self.expect(')')
        return expr

    def parse_star_expr(self) -> StarExpr:
        star = self.expect('*')
        expr = self.parse_expression(precedence['*u'])
        expr = StarExpr(expr)
        if expr.line < 0:
            expr.set_line(star)
        return expr

    def parse_empty_tuple_expr(self) -> TupleExpr:
        self.expect(')')
        node = TupleExpr([])
        return node

    def parse_list_expr(self) -> Union[ListExpr, ListComprehension]:
        """Parse list literal or list comprehension."""
        items = []
        self.expect('[')
        while self.current_str() != ']' and not self.eol():
            items.append(self.parse_expression(precedence['<for>'], star_expr_allowed=True))
            if self.current_str() != ',':
                break
            self.expect(',')
        if self.current_str() == 'for' and len(items) == 1:
            items[0] = self.parse_generator_expr(items[0])
        self.expect(']')
        if len(items) == 1 and isinstance(items[0], GeneratorExpr):
            return ListComprehension(items[0])
        else:
            expr = ListExpr(items)
            return expr

    def parse_generator_expr(self, left_expr: Expression) -> GeneratorExpr:
        tok = self.current()
        indices, sequences, condlists = self.parse_comp_for()

        gen = GeneratorExpr(left_expr, indices, sequences, condlists)
        gen.set_line(tok)
        return gen

    def parse_comp_for(self) -> Tuple[List[Expression], List[Expression], List[List[Expression]]]:
        indices = []
        sequences = []
        condlists = []  # type: List[List[Expression]]
        while self.current_str() == 'for':
            conds = []
            self.expect('for')
            index = self.parse_for_index_variables()
            indices.append(index)
            self.expect('in')
            if self.pyversion[0] >= 3:
                sequence = self.parse_expression(precedence['<if>'])
            else:
                sequence = self.parse_expression_list()
            sequences.append(sequence)
            while self.current_str() == 'if':
                self.skip()
                conds.append(self.parse_expression(precedence['<if>']))
            condlists.append(conds)

        return indices, sequences, condlists

    def parse_expression_list(self) -> Expression:
        prec = precedence['<if>']
        expr = self.parse_expression(prec)
        if self.current_str() != ',':
            return expr
        else:
            t = self.current()
            tuple_expr = self.parse_tuple_expr(expr, prec)
            tuple_expr.set_line(t)
            return tuple_expr

    def parse_conditional_expr(self, left_expr: Expression) -> ConditionalExpr:
        self.expect('if')
        cond = self.parse_expression(precedence['<if>'])
        self.expect('else')
        else_expr = self.parse_expression(precedence['<if>'])
        return ConditionalExpr(cond, left_expr, else_expr)

    def parse_dict_or_set_expr(self) -> Union[SetComprehension, SetExpr,
                                              DictionaryComprehension, DictExpr]:
        items = []  # type: List[Tuple[Expression, Expression]]
        self.expect('{')
        while self.current_str() != '}' and not self.eol():
            key = self.parse_expression(precedence['<for>'])
            if self.current_str() in [',', '}'] and items == []:
                return self.parse_set_expr(key)
            elif self.current_str() == 'for' and items == []:
                return self.parse_set_comprehension(key)
            elif self.current_str() != ':':
                self.parse_error()
            colon = self.expect(':')
            value = self.parse_expression(precedence['<for>'])
            if self.current_str() == 'for' and items == []:
                return self.parse_dict_comprehension(key, value, colon)
            items.append((key, value))
            if self.current_str() != ',':
                break
            self.expect(',')
        self.expect('}')
        node = DictExpr(items)
        return node

    def parse_set_expr(self, first: Expression) -> SetExpr:
        items = [first]
        while self.current_str() != '}' and not self.eol():
            self.expect(',')
            if self.current_str() == '}':
                break
            items.append(self.parse_expression(precedence[',']))
        self.expect('}')
        expr = SetExpr(items)
        return expr

    def parse_set_comprehension(self, expr: Expression) -> SetComprehension:
        gen = self.parse_generator_expr(expr)
        self.expect('}')
        set_comp = SetComprehension(gen)
        return set_comp

    def parse_dict_comprehension(self, key: Expression, value: Expression,
                                 colon: Token) -> DictionaryComprehension:
        indices, sequences, condlists = self.parse_comp_for()
        dic = DictionaryComprehension(key, value, indices, sequences, condlists)
        dic.set_line(colon)
        self.expect('}')
        return dic

    def parse_tuple_expr(self, expr: Expression,
                         prec: int = precedence[',']) -> TupleExpr:
        items = [expr]
        while True:
            self.expect(',')
            if (self.current_str() in [')', ']', '=', ':'] or
                    isinstance(self.current(), Break)):
                break
            items.append(self.parse_expression(prec, star_expr_allowed=True))
            if self.current_str() != ',': break
        node = TupleExpr(items)
        return node

    def parse_backquote_expr(self) -> BackquoteExpr:
        self.expect('`')
        expr = self.parse_expression()
        self.expect('`')
        return BackquoteExpr(expr)

    def parse_name_expr(self) -> NameExpr:
        tok = self.expect_type(Name)
        node = NameExpr(tok.string)
        node.set_line(tok)
        return node

    octal_int = re.compile('0+[1-9]')

    def parse_int_expr(self) -> IntExpr:
        tok = self.expect_type(IntLit)
        string = tok.string.rstrip('lL')  # Strip L prefix (Python 2 long literals)
        if self.octal_int.match(string):
            value = int(string, 8)
        else:
            value = int(string, 0)
        node = IntExpr(value)
        return node

    def parse_str_expr(self) -> Union[UnicodeExpr, StrExpr]:
        # XXX \uxxxx literals
        token = self.expect_type(StrLit)
        value = cast(StrLit, token).parsed()
        is_unicode = False
        while isinstance(self.current(), (StrLit, UnicodeLit)):
            token = self.skip()
            if isinstance(token, StrLit):
                value += token.parsed()
            elif isinstance(token, UnicodeLit):
                value += token.parsed()
                is_unicode = True
        if is_unicode or (self.pyversion[0] == 2 and 'unicode_literals' in self.future_options):
            return UnicodeExpr(value)
        else:
            return StrExpr(value)

    def parse_bytes_literal(self) -> Union[BytesExpr, StrExpr]:
        # XXX \uxxxx literals
        tok = [self.expect_type(BytesLit)]
        value = (cast(BytesLit, tok[0])).parsed()
        while isinstance(self.current(), BytesLit):
            t = cast(BytesLit, self.skip())
            value += t.parsed()
        if self.pyversion[0] >= 3:
            return BytesExpr(value)
        else:
            return StrExpr(value)

    def parse_unicode_literal(self) -> Union[StrExpr, UnicodeExpr]:
        # XXX \uxxxx literals
        token = self.expect_type(UnicodeLit)
        value = cast(UnicodeLit, token).parsed()
        while isinstance(self.current(), (UnicodeLit, StrLit)):
            token = cast(Union[UnicodeLit, StrLit], self.skip())
            value += token.parsed()
        if self.pyversion[0] >= 3:
            # Python 3.3 supports u'...' as an alias of '...'.
            return StrExpr(value)
        else:
            return UnicodeExpr(value)

    def parse_float_expr(self) -> FloatExpr:
        tok = self.expect_type(FloatLit)
        return FloatExpr(float(tok.string))

    def parse_complex_expr(self) -> ComplexExpr:
        tok = self.expect_type(ComplexLit)
        return ComplexExpr(complex(tok.string))

    def parse_call_expr(self, callee: Expression) -> CallExpr:
        self.expect('(')
        args, kinds, names = self.parse_arg_expr()
        self.expect(')')
        return CallExpr(callee, args, kinds, names)

    def parse_arg_expr(self) -> Tuple[List[Expression], List[int], List[str]]:
        """Parse arguments in a call expression (within '(' and ')').

        Return a tuple with these items:
          argument expressions
          argument kinds
          argument names (for named arguments; None for ordinary args)
        """
        args = []   # type: List[Expression]
        kinds = []  # type: List[int]
        names = []  # type: List[str]
        var_arg = False
        dict_arg = False
        named_args = False
        while self.current_str() != ')' and not self.eol() and not dict_arg:
            if isinstance(self.current(), Name) and self.peek().string == '=':
                # Named argument
                name = self.expect_type(Name)
                self.expect('=')
                kinds.append(nodes.ARG_NAMED)
                names.append(name.string)
                named_args = True
            elif (self.current_str() == '*' and not var_arg and not dict_arg):
                # *args
                var_arg = True
                self.expect('*')
                kinds.append(nodes.ARG_STAR)
                names.append(None)
            elif self.current_str() == '**':
                # **kwargs
                self.expect('**')
                dict_arg = True
                kinds.append(nodes.ARG_STAR2)
                names.append(None)
            elif not var_arg and not named_args:
                # Ordinary argument
                kinds.append(nodes.ARG_POS)
                names.append(None)
            else:
                self.parse_error()
            args.append(self.parse_expression(precedence[',']))
            if self.current_str() != ',':
                break
            self.expect(',')
        return args, kinds, names

    def parse_member_expr(self, expr: Expression) -> Union[SuperExpr, MemberExpr]:
        self.expect('.')
        name = self.expect_type(Name)
        if (isinstance(expr, CallExpr) and isinstance(expr.callee, NameExpr)
                and expr.callee.name == 'super'):
            # super() expression
            return SuperExpr(name.string)
        else:
            return MemberExpr(expr, name.string)

    def parse_index_expr(self, base: Expression) -> IndexExpr:
        self.expect('[')
        index = self.parse_slice_item()
        if self.current_str() == ',':
            # Extended slicing such as x[1:, :2].
            items = [index]
            while self.current_str() == ',':
                self.skip()
                if self.current_str() == ']' or isinstance(self.current(), Break):
                    break
                items.append(self.parse_slice_item())
            index = TupleExpr(items)
            index.set_line(items[0])
        self.expect(']')
        node = IndexExpr(base, index)
        return node

    def parse_slice_item(self) -> Expression:
        if self.current_str() != ':':
            if self.current_str() == '...':
                # Ellipsis is valid here even in Python 2 (but not elsewhere).
                ellipsis = EllipsisExpr()
                token = self.skip()
                ellipsis.set_line(token)
                return ellipsis
            else:
                item = self.parse_expression(precedence[','])
        else:
            item = None
        if self.current_str() == ':':
            # Slice.
            index = item
            colon = self.expect(':')
            if self.current_str() not in (']', ':', ','):
                end_index = self.parse_expression(precedence[','])
            else:
                end_index = None
            stride = None
            if self.current_str() == ':':
                self.expect(':')
                if self.current_str() not in (']', ','):
                    stride = self.parse_expression(precedence[','])
            item = SliceExpr(index, end_index, stride)
            item.set_line(colon)
        return item

    def parse_bin_op_expr(self, left: Expression, prec: int) -> OpExpr:
        op = self.expect_type(Op)
        op_str = op.string
        if op_str == '~':
            self.ind -= 1
            self.parse_error()
        right = self.parse_expression(prec)
        node = OpExpr(op_str, left, right)
        return node

    def parse_comparison_expr(self, left: Expression, prec: int) -> ComparisonExpr:
        operators_str = []
        operands = [left]

        while True:
            op = self.expect_type(Op)
            op_str = op.string
            if op_str == 'not':
                if self.current_str() == 'in':
                    op_str = 'not in'
                    self.skip()
                else:
                    self.parse_error()
            elif op_str == 'is' and self.current_str() == 'not':
                op_str = 'is not'
                self.skip()

            operators_str.append(op_str)
            operand = self.parse_expression(prec)
            operands.append(operand)

            # Continue if next token is a comparison operator
            self.current()
            s = self.current_str()
            if s not in op_comp:
                break

        node = ComparisonExpr(operators_str, operands)
        return node

    def parse_unary_expr(self) -> UnaryExpr:
        op_tok = self.skip()
        op = op_tok.string
        if op == '-' or op == '+':
            prec = precedence['-u']
        else:
            prec = precedence[op]
        expr = self.parse_expression(prec)
        node = UnaryExpr(op, expr)
        return node

    def parse_lambda_expr(self) -> FuncExpr:
        lambda_tok = self.expect('lambda')

        args, extra_stmts = self.parse_arg_list(allow_signature=False)

        # Use 'object' as the placeholder return type; it will be inferred
        # later. We can't use 'Any' since it could make type inference results
        # less precise.
        ret_type = UnboundType('__builtins__.object')
        typ = self.build_func_annotation(ret_type, args,
                                         lambda_tok.line, lambda_tok.column, is_default_ret=True)

        colon = self.expect(':')

        expr = self.parse_expression(precedence[','])

        return_stmt = ReturnStmt(expr)
        return_stmt.set_line(lambda_tok)
        nodes = [return_stmt]  # type: List[Statement]
        # Potentially insert extra assignment statements to the beginning of the
        # body, used to decompose Python 2 tuple arguments.
        nodes[:0] = extra_stmts
        body = Block(nodes)
        body.set_line(colon)

        return FuncExpr(args, body, typ)

    # Helper methods

    def skip(self) -> Token:
        self.ind += 1
        return self.tok[self.ind - 1]

    def expect(self, string: str) -> Token:
        if self.current_str() == string:
            self.ind += 1
            return self.tok[self.ind - 1]
        else:
            self.parse_error()

    def expect_indent(self) -> Token:
        if isinstance(self.current(), Indent):
            return self.expect_type(Indent)
        else:
            self.fail('Expected an indented block', self.current().line, self.current().column)
            return none

    def fail(self, msg: str, line: int, column: int) -> None:
        self.errors.report(line, column, msg)

    def expect_type(self, typ: type) -> Token:
        current = self.current()
        if isinstance(current, typ):
            self.ind += 1
            return current
        else:
            self.parse_error()

    def expect_colon_and_break(self) -> Tuple[Token, Token]:
        return self.expect_type(Colon), self.expect_type(Break)

    def expect_break(self) -> Token:
        return self.expect_type(Break)

    def current(self) -> Token:
        return self.tok[self.ind]

    def current_str(self) -> str:
        return self.current().string

    def peek(self) -> Token:
        return self.tok[self.ind + 1]

    def parse_error(self) -> None:
        self.parse_error_at(self.current())
        raise ParseError()

    def parse_error_at(self, tok: Token, skip: bool = True, reason: Optional[str] = None) -> None:
        msg = ''
        if isinstance(tok, LexError):
            msg = token_repr(tok)
            msg = msg[0].upper() + msg[1:]
        elif isinstance(tok, Indent) or isinstance(tok, Dedent):
            msg = 'Inconsistent indentation'
        else:
            formatted_reason = ": {}".format(reason) if reason else ""
            msg = 'Parse error before {}{}'.format(token_repr(tok), formatted_reason)

        self.errors.report(tok.line, tok.column, msg)

        if skip:
            self.skip_until_next_line()

    def skip_until_break(self) -> None:
        n = 0
        while (not isinstance(self.current(), Break)
               and not isinstance(self.current(), Eof)):
            self.skip()
            n += 1
        if isinstance(self.tok[self.ind - 1], Colon) and n > 1:
            self.ind -= 1

    def skip_until_next_line(self) -> None:
        self.skip_until_break()
        if isinstance(self.current(), Break):
            self.skip()

    def eol(self) -> bool:
        return isinstance(self.current(), Break) or self.eof()

    def eof(self) -> bool:
        return isinstance(self.current(), Eof)

    # Type annotation related functionality

    def parse_type(self) -> Type:
        try:
            typ, self.ind = parse_type(self.tok, self.ind)
        except TypeParseError as e:
            self.parse_error_at(e.token, reason=e.message)
            raise ParseError()
        return typ

    annotation_prefix_re = re.compile(r'#\s*type:')
    ignore_prefix_re = re.compile(r'ignore\b')

    def parse_type_comment(self, token: Token, signature: bool) -> Type:
        """Parse a '# type: ...' annotation.

        Return None if no annotation found. If signature is True, expect
        a type signature of form (...) -> t.
        """
        whitespace_or_comments = token.rep().strip()
        if self.annotation_prefix_re.match(whitespace_or_comments):
            type_as_str = whitespace_or_comments.split(':', 1)[1].strip()
            if self.ignore_prefix_re.match(type_as_str):
                # Actually a "# type: ignore" annotation -> not a type.
                return None
            tokens = lex.lex(type_as_str, token.line)[0]
            if len(tokens) < 2:
                # Empty annotation (only Eof token)
                self.errors.report(token.line, token.column, 'Empty type annotation')
                return None
            try:
                if not signature:
                    type, index = parse_types(tokens, 0)
                else:
                    type, index = parse_signature(tokens)
            except TypeParseError as e:
                self.parse_error_at(e.token, skip=False, reason=e.message)
                return None
            if index < len(tokens) - 2:
                self.parse_error_at(tokens[index], skip=False)
                return None
            return type
        else:
            return None


class ParseError(Exception): pass


def token_repr(tok: Token) -> str:
    """Return a representation of a token for use in parse error messages."""
    if isinstance(tok, Break):
        return 'end of line'
    elif isinstance(tok, Eof):
        return 'end of file'
    elif isinstance(tok, Keyword) or isinstance(tok, Name):
        return '"{}"'.format(tok.string)
    elif isinstance(tok, IntLit) or isinstance(tok, FloatLit) or isinstance(tok, ComplexLit):
        return 'numeric literal'
    elif isinstance(tok, StrLit) or isinstance(tok, UnicodeLit):
        return 'string literal'
    elif (isinstance(tok, Punct) or isinstance(tok, Op)
          or isinstance(tok, Colon)):
        return tok.string
    elif isinstance(tok, Bom):
        return 'byte order mark'
    elif isinstance(tok, Indent):
        return 'indent'
    elif isinstance(tok, Dedent):
        return 'dedent'
    elif isinstance(tok, EllipsisToken):
        return '...'
    else:
        if isinstance(tok, LexError):
            t = tok.type
            if t == lex.NUMERIC_LITERAL_ERROR:
                return 'invalid numeric literal'
            elif t == lex.UNTERMINATED_STRING_LITERAL:
                return 'unterminated string literal'
            elif t == lex.INVALID_CHARACTER:
                msg = 'unrecognized character'
                if ord(tok.string) in range(33, 127):
                    msg += ' ' + tok.string
                return msg
            elif t == lex.INVALID_DEDENT:
                return 'inconsistent indentation'
            elif t == lex.DECODE_ERROR:
                return tok.message
        raise ValueError('Unknown token {}'.format(repr(tok)))


if __name__ == '__main__':
    # Parse a file and dump the AST (or display errors).
    import sys

    def usage() -> None:
        print('Usage: parse.py [--py2] [--quiet] FILE [...]', file=sys.stderr)
        sys.exit(2)

    args = sys.argv[1:]
    pyversion = defaults.PYTHON3_VERSION
    quiet = False
    while args and args[0].startswith('--'):
        if args[0] == '--py2':
            pyversion = defaults.PYTHON2_VERSION
        elif args[0] == '--quiet':
            quiet = True
        else:
            usage()
        args = args[1:]
    if len(args) < 1:
        usage()
    status = 0
    for fnam in args:
        s = open(fnam, 'rb').read()
        errors = Errors()
        try:
            options = Options()
            options.python_version = pyversion
            tree = parse(s, fnam, None, options=options)
            if not quiet:
                print(tree)
        except CompileError as e:
            for msg in e.messages:
                sys.stderr.write('%s\n' % msg)
            status = 1
    sys.exit(status)
