Adding new stuff

vim-plugins/bundle/jedi-vim/jedi/test/test_parser/test_fast_parser.py

@@ -0,0 +1,496 @@
+from textwrap import dedent
+
+import pytest
+
+import jedi
+from jedi._compatibility import u
+from jedi import cache
+from jedi.parser import load_grammar
+from jedi.parser.fast import FastParser
+from jedi.parser.utils import save_parser
+
+
+def test_add_to_end():
+    """
+    fast_parser doesn't parse everything again. It just updates with the
+    help of caches, this is an example that didn't work.
+    """
+
+    a = dedent("""
+    class Abc():
+        def abc(self):
+            self.x = 3
+
+    class Two(Abc):
+        def h(self):
+            self
+    """)      # ^ here is the first completion
+
+    b = "    def g(self):\n" \
+        "        self."
+    assert jedi.Script(a, 8, 12, 'example.py').completions()
+    assert jedi.Script(a + b, path='example.py').completions()
+
+    a = a[:-1] + '.\n'
+    assert jedi.Script(a, 8, 13, 'example.py').completions()
+    assert jedi.Script(a + b, path='example.py').completions()
+
+
+def test_class_in_docstr():
+    """
+    Regression test for a problem with classes in docstrings.
+    """
+    a = '"\nclasses\n"'
+    jedi.Script(a, 1, 0)._get_module()
+
+    b = a + '\nimport os'
+    assert jedi.Script(b, 4, 8).goto_assignments()
+
+
+def test_carriage_return_splitting():
+    source = u(dedent('''
+
+
+
+        "string"
+
+        class Foo():
+            pass
+        '''))
+    source = source.replace('\n', '\r\n')
+    p = FastParser(load_grammar(), source)
+    assert [n.value for lst in p.module.names_dict.values() for n in lst] == ['Foo']
+
+
+def test_split_parts():
+    cache.parser_cache.pop(None, None)
+
+    def splits(source):
+        class Mock(FastParser):
+            def __init__(self, *args):
+                self.number_of_splits = 0
+
+        return tuple(FastParser._split_parts(Mock(None, None), source))
+
+    def test(*parts):
+        assert splits(''.join(parts)) == parts
+
+    test('a\n\n', 'def b(): pass\n', 'c\n')
+    test('a\n', 'def b():\n pass\n', 'c\n')
+
+    test('from x\\\n')
+    test('a\n\\\n')
+
+
+def check_fp(src, number_parsers_used, number_of_splits=None, number_of_misses=0):
+    if number_of_splits is None:
+        number_of_splits = number_parsers_used
+
+    p = FastParser(load_grammar(), u(src))
+    save_parser(None, p, pickling=False)
+
+    assert src == p.module.get_code()
+    assert p.number_of_splits == number_of_splits
+    assert p.number_parsers_used == number_parsers_used
+    assert p.number_of_misses == number_of_misses
+    return p.module
+
+
+def test_change_and_undo():
+    # Empty the parser cache for the path None.
+    cache.parser_cache.pop(None, None)
+    func_before = 'def func():\n    pass\n'
+    # Parse the function and a.
+    check_fp(func_before + 'a', 2)
+    # Parse just b.
+    check_fp(func_before + 'b', 1, 2)
+    # b has changed to a again, so parse that.
+    check_fp(func_before + 'a', 1, 2)
+    # Same as before no parsers should be used.
+    check_fp(func_before + 'a', 0, 2)
+
+    # Getting rid of an old parser: Still no parsers used.
+    check_fp('a', 0, 1)
+    # Now the file has completely change and we need to parse.
+    check_fp('b', 1, 1)
+    # And again.
+    check_fp('a', 1, 1)
+
+
+def test_positions():
+    # Empty the parser cache for the path None.
+    cache.parser_cache.pop(None, None)
+
+    func_before = 'class A:\n pass\n'
+    m = check_fp(func_before + 'a', 2)
+    assert m.start_pos == (1, 0)
+    assert m.end_pos == (3, 1)
+
+    m = check_fp('a', 0, 1)
+    assert m.start_pos == (1, 0)
+    assert m.end_pos == (1, 1)
+
+
+def test_if():
+    src = dedent('''\
+    def func():
+        x = 3
+        if x:
+            def y():
+                return x
+        return y()
+
+    func()
+    ''')
+
+    # Two parsers needed, one for pass and one for the function.
+    check_fp(src, 2)
+    assert [d.name for d in jedi.Script(src, 8, 6).goto_definitions()] == ['int']
+
+
+def test_if_simple():
+    src = dedent('''\
+    if 1:
+        a = 3
+    ''')
+    check_fp(src + 'a', 1)
+    check_fp(src + "else:\n    a = ''\na", 1)
+
+
+def test_for():
+    src = dedent("""\
+    for a in [1,2]:
+        a
+
+    for a1 in 1,"":
+        a1
+    """)
+    check_fp(src, 1)
+
+
+def test_class_with_class_var():
+    src = dedent("""\
+    class SuperClass:
+        class_super = 3
+        def __init__(self):
+            self.foo = 4
+    pass
+    """)
+    check_fp(src, 3)
+
+
+def test_func_with_if():
+    src = dedent("""\
+    def recursion(a):
+        if foo:
+            return recursion(a)
+        else:
+            if bar:
+                return inexistent
+            else:
+                return a
+    """)
+    check_fp(src, 1)
+
+
+def test_decorator():
+    src = dedent("""\
+    class Decorator():
+        @memoize
+        def dec(self, a):
+            return a
+    """)
+    check_fp(src, 2)
+
+
+def test_nested_funcs():
+    src = dedent("""\
+    def memoize(func):
+        def wrapper(*args, **kwargs):
+            return func(*args, **kwargs)
+        return wrapper
+    """)
+    check_fp(src, 3)
+
+
+def test_class_and_if():
+    src = dedent("""\
+    class V:
+        def __init__(self):
+            pass
+
+        if 1:
+            c = 3
+
+    def a_func():
+        return 1
+
+    # COMMENT
+    a_func()""")
+    check_fp(src, 5, 5)
+    assert [d.name for d in jedi.Script(src).goto_definitions()] == ['int']
+
+
+def test_func_with_for_and_comment():
+    # The first newline is important, leave it. It should not trigger another
+    # parser split.
+    src = dedent("""\
+
+    def func():
+        pass
+
+
+    for a in [1]:
+        # COMMENT
+        a""")
+    check_fp(src, 2)
+    # We don't need to parse the for loop, but we need to parse the other two,
+    # because the split is in a different place.
+    check_fp('a\n' + src, 2, 3)
+
+
+def test_multi_line_params():
+    src = dedent("""\
+    def x(a,
+          b):
+        pass
+
+    foo = 1
+    """)
+    check_fp(src, 2)
+
+
+def test_one_statement_func():
+    src = dedent("""\
+    first
+    def func(): a
+    """)
+    check_fp(src + 'second', 3)
+    # Empty the parser cache, because we're not interested in modifications
+    # here.
+    cache.parser_cache.pop(None, None)
+    check_fp(src + 'def second():\n a', 3)
+
+
+def test_class_func_if():
+    src = dedent("""\
+    class Class:
+        def func(self):
+            if 1:
+                a
+            else:
+                b
+
+    pass
+    """)
+    check_fp(src, 3)
+
+
+def test_for_on_one_line():
+    src = dedent("""\
+    foo = 1
+    for x in foo: pass
+
+    def hi():
+        pass
+    """)
+    check_fp(src, 2)
+
+    src = dedent("""\
+    def hi():
+        for x in foo: pass
+        pass
+
+    pass
+    """)
+    check_fp(src, 2)
+
+    src = dedent("""\
+    def hi():
+        for x in foo: pass
+
+        def nested():
+            pass
+    """)
+    check_fp(src, 2)
+
+
+def test_multi_line_for():
+    src = dedent("""\
+    for x in [1,
+              2]:
+        pass
+
+    pass
+    """)
+    check_fp(src, 1)
+
+
+def test_wrong_indentation():
+    src = dedent("""\
+    def func():
+        a
+         b
+        a
+    """)
+    #check_fp(src, 1)
+
+    src = dedent("""\
+    def complex():
+        def nested():
+            a
+             b
+            a
+
+        def other():
+            pass
+    """)
+    check_fp(src, 3)
+
+
+def test_open_parentheses():
+    func = 'def func():\n a'
+    code = u('isinstance(\n\n' + func)
+    p = FastParser(load_grammar(), code)
+    # As you can see, the part that was failing is still there in the get_code
+    # call. It is not relevant for evaluation, but still available as an
+    # ErrorNode.
+    assert p.module.get_code() == code
+    assert p.number_of_splits == 2
+    assert p.number_parsers_used == 2
+    save_parser(None, p, pickling=False)
+
+    # Now with a correct parser it should work perfectly well.
+    check_fp('isinstance()\n' + func, 1, 2)
+
+
+def test_strange_parentheses():
+    src = dedent("""
+    class X():
+        a = (1
+    if 1 else 2)
+        def x():
+            pass
+    """)
+    check_fp(src, 2)
+
+
+def test_backslash():
+    src = dedent(r"""
+    a = 1\
+        if 1 else 2
+    def x():
+        pass
+    """)
+    check_fp(src, 2)
+
+    src = dedent(r"""
+    def x():
+        a = 1\
+    if 1 else 2
+        def y():
+            pass
+    """)
+    # The dangling if leads to not splitting where we theoretically could
+    # split.
+    check_fp(src, 2)
+
+    src = dedent(r"""
+    def first():
+        if foo \
+                and bar \
+                or baz:
+            pass
+    def second():
+        pass
+    """)
+    check_fp(src, 2)
+
+
+
+def test_fake_parentheses():
+    """
+    The fast parser splitting counts parentheses, but not as correct tokens.
+    Therefore parentheses in string tokens are included as well. This needs to
+    be accounted for.
+    """
+    src = dedent(r"""
+    def x():
+        a = (')'
+    if 1 else 2)
+        def y():
+            pass
+        def z():
+            pass
+    """)
+    check_fp(src, 3, 2, 1)
+
+
+def test_additional_indent():
+    source = dedent('''\
+    int(
+      def x():
+          pass
+    ''')
+
+    check_fp(source, 2)
+
+
+def test_incomplete_function():
+    source = '''return ImportErr'''
+
+    script = jedi.Script(dedent(source), 1, 3)
+    assert script.completions()
+
+
+def test_string_literals():
+    """Simplified case of jedi-vim#377."""
+    source = dedent("""
+    x = ur''' 
+
+    def foo():
+        pass
+    """)
+
+    script = jedi.Script(dedent(source))
+    script._get_module().end_pos == (6, 0)
+    assert script.completions()
+
+
+def test_decorator_string_issue():
+    """
+    Test case from #589
+    """
+    source = dedent('''\
+    """
+      @"""
+    def bla():
+      pass
+
+    bla.''')
+
+    s = jedi.Script(source)
+    assert s.completions()
+    assert s._get_module().get_code() == source
+
+
+def test_round_trip():
+    source = dedent('''
+    def x():
+        """hahaha"""
+    func''')
+
+    f = FastParser(load_grammar(), u(source))
+    assert f.get_parsed_node().get_code() == source
+
+
+@pytest.mark.xfail()
+def test_parentheses_in_string():
+    code = dedent('''
+    def x():
+        '('
+
+    import abc
+
+    abc.''')
+    check_fp(code, 2, 1, 1)

vim-plugins/bundle/jedi-vim/jedi/test/test_parser/test_get_code.py

@@ -0,0 +1,107 @@
+import difflib
+
+import pytest
+
+from jedi._compatibility import u
+from jedi.parser import ParserWithRecovery, load_grammar
+
+code_basic_features = u('''
+"""A mod docstring"""
+
+def a_function(a_argument, a_default = "default"):
+    """A func docstring"""
+
+    a_result = 3 * a_argument
+    print(a_result)  # a comment
+    b = """
+from
+to""" + "huhu"
+
+
+    if a_default == "default":
+        return str(a_result)
+    else
+        return None
+''')
+
+
+def diff_code_assert(a, b, n=4):
+    if a != b:
+        diff = "\n".join(difflib.unified_diff(
+            a.splitlines(),
+            b.splitlines(),
+            n=n,
+            lineterm=""
+        ))
+        assert False, "Code does not match:\n%s\n\ncreated code:\n%s" % (
+            diff,
+            b
+        )
+    pass
+
+
+@pytest.mark.skipif('True', reason='Refactor a few parser things first.')
+def test_basic_parsing():
+    """Validate the parsing features"""
+
+    prs = ParserWithRecovery(load_grammar(), code_basic_features)
+    diff_code_assert(
+        code_basic_features,
+        prs.module.get_code()
+    )
+
+
+def test_operators():
+    src = u('5  * 3')
+    prs = ParserWithRecovery(load_grammar(), src)
+    diff_code_assert(src, prs.module.get_code())
+
+
+def test_get_code():
+    """Use the same code that the parser also generates, to compare"""
+    s = u('''"""a docstring"""
+class SomeClass(object, mixin):
+    def __init__(self):
+        self.xy = 3.0
+        """statement docstr"""
+    def some_method(self):
+        return 1
+    def yield_method(self):
+        while hasattr(self, 'xy'):
+            yield True
+        for x in [1, 2]:
+            yield x
+    def empty(self):
+        pass
+class Empty:
+    pass
+class WithDocstring:
+    """class docstr"""
+    pass
+def method_with_docstring():
+    """class docstr"""
+    pass
+''')
+    assert ParserWithRecovery(load_grammar(), s).module.get_code() == s
+
+
+def test_end_newlines():
+    """
+    The Python grammar explicitly needs a newline at the end. Jedi though still
+    wants to be able, to return the exact same code without the additional new
+    line the parser needs.
+    """
+    def test(source, end_pos):
+        module = ParserWithRecovery(load_grammar(), u(source)).module
+        assert module.get_code() == source
+        assert module.end_pos == end_pos
+
+    test('a', (1, 1))
+    test('a\n', (2, 0))
+    test('a\nb', (2, 1))
+    test('a\n#comment\n', (3, 0))
+    test('a\n#comment', (2, 8))
+    test('a#comment', (1, 9))
+    test('def a():\n pass', (2, 5))
+
+    test('def a(', (1, 6))

vim-plugins/bundle/jedi-vim/jedi/test/test_parser/test_param_splitting.py

@@ -0,0 +1,34 @@
+'''
+To make the life of any analysis easier, we are generating Param objects
+instead of simple parser objects.
+'''
+
+from textwrap import dedent
+
+from jedi.parser import Parser, load_grammar
+
+
+def assert_params(param_string, **wanted_dct):
+    source = dedent('''
+    def x(%s):
+        pass
+    ''') % param_string
+
+    parser = Parser(load_grammar(), dedent(source))
+    funcdef = parser.get_parsed_node().subscopes[0]
+    dct = dict((p.name.value, p.default and p.default.get_code())
+               for p in funcdef.params)
+    assert dct == wanted_dct
+    assert parser.get_parsed_node().get_code() == source
+
+
+def test_split_params_with_separation_star():
+    assert_params(u'x, y=1, *, z=3', x=None, y='1', z='3')
+    assert_params(u'*, x', x=None)
+    assert_params(u'*')
+
+
+def test_split_params_with_stars():
+    assert_params(u'x, *args', x=None, args=None)
+    assert_params(u'**kwargs', kwargs=None)
+    assert_params(u'*args, **kwargs', args=None, kwargs=None)

vim-plugins/bundle/jedi-vim/jedi/test/test_parser/test_parser.py

@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+import sys
+from textwrap import dedent
+
+import jedi
+from jedi._compatibility import u, is_py3
+from jedi.parser import ParserWithRecovery, load_grammar
+from jedi.parser import tree as pt
+
+
+def test_user_statement_on_import():
+    """github #285"""
+    s = u("from datetime import (\n"
+          "    time)")
+
+    for pos in [(2, 1), (2, 4)]:
+        p = ParserWithRecovery(load_grammar(), s)
+        stmt = p.module.get_statement_for_position(pos)
+        assert isinstance(stmt, pt.Import)
+        assert [str(n) for n in stmt.get_defined_names()] == ['time']
+
+
+class TestCallAndName():
+    def get_call(self, source):
+        # Get the simple_stmt and then the first one.
+        simple_stmt = ParserWithRecovery(load_grammar(), u(source)).module.children[0]
+        return simple_stmt.children[0]
+
+    def test_name_and_call_positions(self):
+        name = self.get_call('name\nsomething_else')
+        assert str(name) == 'name'
+        assert name.start_pos == (1, 0)
+        assert name.end_pos == (1, 4)
+
+        leaf = self.get_call('1.0\n')
+        assert leaf.value == '1.0'
+        assert leaf.eval() == 1.0
+        assert leaf.start_pos == (1, 0)
+        assert leaf.end_pos == (1, 3)
+
+    def test_call_type(self):
+        call = self.get_call('hello')
+        assert isinstance(call, pt.Name)
+
+    def test_literal_type(self):
+        literal = self.get_call('1.0')
+        assert isinstance(literal, pt.Literal)
+        assert type(literal.eval()) == float
+
+        literal = self.get_call('1')
+        assert isinstance(literal, pt.Literal)
+        assert type(literal.eval()) == int
+
+        literal = self.get_call('"hello"')
+        assert isinstance(literal, pt.Literal)
+        assert literal.eval() == 'hello'
+
+
+class TestSubscopes():
+    def get_sub(self, source):
+        return ParserWithRecovery(load_grammar(), u(source)).module.subscopes[0]
+
+    def test_subscope_names(self):
+        name = self.get_sub('class Foo: pass').name
+        assert name.start_pos == (1, len('class '))
+        assert name.end_pos == (1, len('class Foo'))
+        assert str(name) == 'Foo'
+
+        name = self.get_sub('def foo(): pass').name
+        assert name.start_pos == (1, len('def '))
+        assert name.end_pos == (1, len('def foo'))
+        assert str(name) == 'foo'
+
+
+class TestImports():
+    def get_import(self, source):
+        return ParserWithRecovery(load_grammar(), source).module.imports[0]
+
+    def test_import_names(self):
+        imp = self.get_import(u('import math\n'))
+        names = imp.get_defined_names()
+        assert len(names) == 1
+        assert str(names[0]) == 'math'
+        assert names[0].start_pos == (1, len('import '))
+        assert names[0].end_pos == (1, len('import math'))
+
+        assert imp.start_pos == (1, 0)
+        assert imp.end_pos == (1, len('import math'))
+
+
+def test_module():
+    module = ParserWithRecovery(load_grammar(), u('asdf'), 'example.py').module
+    name = module.name
+    assert str(name) == 'example'
+    assert name.start_pos == (1, 0)
+    assert name.end_pos == (1, 7)
+
+    module = ParserWithRecovery(load_grammar(), u('asdf')).module
+    name = module.name
+    assert str(name) == ''
+    assert name.start_pos == (1, 0)
+    assert name.end_pos == (1, 0)
+
+
+def test_end_pos():
+    s = u(dedent('''
+                 x = ['a', 'b', 'c']
+                 def func():
+                     y = None
+                 '''))
+    parser = ParserWithRecovery(load_grammar(), s)
+    scope = parser.module.subscopes[0]
+    assert scope.start_pos == (3, 0)
+    assert scope.end_pos == (5, 0)
+
+
+def test_carriage_return_statements():
+    source = u(dedent('''
+        foo = 'ns1!'
+
+        # this is a namespace package
+    '''))
+    source = source.replace('\n', '\r\n')
+    stmt = ParserWithRecovery(load_grammar(), source).module.statements[0]
+    assert '#' not in stmt.get_code()
+
+
+def test_incomplete_list_comprehension():
+    """ Shouldn't raise an error, same bug as #418. """
+    # With the old parser this actually returned a statement. With the new
+    # parser only valid statements generate one.
+    assert ParserWithRecovery(load_grammar(), u('(1 for def')).module.statements == []
+
+
+def test_hex_values_in_docstring():
+    source = r'''
+        def foo(object):
+            """
+             \xff
+            """
+            return 1
+        '''
+
+    doc = ParserWithRecovery(load_grammar(), dedent(u(source))).module.subscopes[0].raw_doc
+    if is_py3:
+        assert doc == '\xff'
+    else:
+        assert doc == u('<EFBFBD>')
+
+
+def test_error_correction_with():
+    source = """
+    with open() as f:
+        try:
+            f."""
+    comps = jedi.Script(source).completions()
+    assert len(comps) > 30
+    # `open` completions have a closed attribute.
+    assert [1 for c in comps if c.name == 'closed']
+
+
+def test_newline_positions():
+    endmarker = ParserWithRecovery(load_grammar(), u('a\n')).module.children[-1]
+    assert endmarker.end_pos == (2, 0)
+    new_line = endmarker.get_previous_leaf()
+    assert new_line.start_pos == (1, 1)
+    assert new_line.end_pos == (2, 0)
+
+
+def test_end_pos_error_correction():
+    """
+    Source code without ending newline are given one, because the Python
+    grammar needs it. However, they are removed again. We still want the right
+    end_pos, even if something breaks in the parser (error correction).
+    """
+    s = u('def x():\n .')
+    m = ParserWithRecovery(load_grammar(), s).module
+    func = m.children[0]
+    assert func.type == 'funcdef'
+    # This is not exactly correct, but ok, because it doesn't make a difference
+    # at all. We just want to make sure that the module end_pos is correct!
+    assert func.end_pos == (3, 0)
+    assert m.end_pos == (2, 2)
+
+
+def test_param_splitting():
+    """
+    Jedi splits parameters into params, this is not what the grammar does,
+    but Jedi does this to simplify argument parsing.
+    """
+    def check(src, result):
+        # Python 2 tuple params should be ignored for now.
+        grammar = load_grammar('%s.%s' % sys.version_info[:2])
+        m = ParserWithRecovery(grammar, u(src)).module
+        if is_py3:
+            assert not m.subscopes
+        else:
+            # We don't want b and c to be a part of the param enumeration. Just
+            # ignore them, because it's not what we want to support in the
+            # future.
+            assert [str(param.name) for param in m.subscopes[0].params] == result
+
+    check('def x(a, (b, c)):\n pass', ['a'])
+    check('def x((b, c)):\n pass', [])
+
+
+def test_unicode_string():
+    s = pt.String(None, u('bö'), (0, 0))
+    assert repr(s)  # Should not raise an Error!
+
+
+def test_backslash_dos_style():
+    grammar = load_grammar()
+    m = ParserWithRecovery(grammar, u('\\\r\n')).module
+    assert m
+
+
+def test_started_lambda_stmt():
+    p = ParserWithRecovery(load_grammar(), u'lambda a, b: a i')
+    assert p.get_parsed_node().children[0].type == 'error_node'
+
+
+def test_python2_octal():
+    parser = ParserWithRecovery(load_grammar(), u'0660')
+    first = parser.get_parsed_node().children[0]
+    if is_py3:
+        assert first.type == 'error_node'
+    else:
+        assert first.children[0].type == 'number'
+
+
+def test_python3_octal():
+    parser = ParserWithRecovery(load_grammar(), u'0o660')
+    module = parser.get_parsed_node()
+    if is_py3:
+        assert module.children[0].children[0].type == 'number'
+    else:
+        assert module.children[0].type == 'error_node'

vim-plugins/bundle/jedi-vim/jedi/test/test_parser/test_parser_tree.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8    # This file contains Unicode characters.
+
+from textwrap import dedent
+
+import pytest
+
+from jedi._compatibility import u, unicode
+from jedi.parser import ParserWithRecovery, load_grammar
+from jedi.parser import tree as pt
+
+
+class TestsFunctionAndLambdaParsing(object):
+
+    FIXTURES = [
+        ('def my_function(x, y, z) -> str:\n    return x + y * z\n', {
+            'name': 'my_function',
+            'call_sig': 'my_function(x, y, z)',
+            'params': ['x', 'y', 'z'],
+            'annotation': "str",
+        }),
+        ('lambda x, y, z: x + y * z\n', {
+            'name': '<lambda>',
+            'call_sig': '<lambda>(x, y, z)',
+            'params': ['x', 'y', 'z'],
+        }),
+    ]
+
+    @pytest.fixture(params=FIXTURES)
+    def node(self, request):
+        parsed = ParserWithRecovery(load_grammar(), dedent(u(request.param[0])))
+        request.keywords['expected'] = request.param[1]
+        return parsed.module.subscopes[0]
+
+    @pytest.fixture()
+    def expected(self, request, node):
+        return request.keywords['expected']
+    
+    def test_name(self, node, expected):
+        assert isinstance(node.name, pt.Name)
+        assert unicode(node.name) == u(expected['name'])
+    
+    def test_params(self, node, expected):
+        assert isinstance(node.params, list)
+        assert all(isinstance(x, pt.Param) for x in node.params)
+        assert [unicode(x.name) for x in node.params] == [u(x) for x in expected['params']]
+
+    def test_is_generator(self, node, expected):
+        assert node.is_generator() is expected.get('is_generator', False)
+
+    def test_yields(self, node, expected):
+        # TODO: There's a comment in the code noting that the current implementation is incorrect.  This returns an
+        # empty list at the moment (not e.g. False).
+        if expected.get('yields', False):
+            assert node.yields
+        else:
+            assert not node.yields
+
+    def test_annotation(self, node, expected):
+        expected_annotation = expected.get('annotation', None)
+        if expected_annotation is None:
+            assert node.annotation() is None
+        else:
+            assert node.annotation().value == expected_annotation
+
+    def test_get_call_signature(self, node, expected):
+        assert node.get_call_signature() == expected['call_sig']
+
+    def test_doc(self, node, expected):
+        assert node.doc == expected.get('doc') or (expected['call_sig'] + '\n\n')

vim-plugins/bundle/jedi-vim/jedi/test/test_parser/test_pgen2.py

@@ -0,0 +1,279 @@
+"""Test suite for 2to3's parser and grammar files.
+
+This is the place to add tests for changes to 2to3's grammar, such as those
+merging the grammars for Python 2 and 3. In addition to specific tests for
+parts of the grammar we've changed, we also make sure we can parse the
+test_grammar.py files from both Python 2 and Python 3.
+"""
+
+from textwrap import dedent
+
+
+from jedi._compatibility import unicode, is_py3
+from jedi.parser import Parser, load_grammar, ParseError
+import pytest
+
+from test.helpers import TestCase
+
+
+def parse(code, version='3.4'):
+    code = dedent(code) + "\n\n"
+    grammar = load_grammar(version=version)
+    return Parser(grammar, unicode(code), 'file_input').get_parsed_node()
+
+
+class TestDriver(TestCase):
+
+    def test_formfeed(self):
+        s = """print 1\n\x0Cprint 2\n"""
+        t = parse(s, '2.7')
+        self.assertEqual(t.children[0].children[0].type, 'print_stmt')
+        self.assertEqual(t.children[1].children[0].type, 'print_stmt')
+        s = """1\n\x0C\x0C2\n"""
+        t = parse(s, '2.7')
+
+
+class GrammarTest(TestCase):
+    def invalid_syntax(self, code, **kwargs):
+        try:
+            parse(code, **kwargs)
+        except ParseError:
+            pass
+        else:
+            raise AssertionError("Syntax shouldn't have been valid")
+
+
+class TestMatrixMultiplication(GrammarTest):
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    def test_matrix_multiplication_operator(self):
+        parse("a @ b", "3.5")
+        parse("a @= b", "3.5")
+
+
+class TestYieldFrom(GrammarTest):
+    def test_yield_from(self):
+        parse("yield from x")
+        parse("(yield from x) + y")
+        self.invalid_syntax("yield from")
+
+
+class TestAsyncAwait(GrammarTest):
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    def test_await_expr(self):
+        parse("""async def foo():
+                             await x
+                      """, "3.5")
+
+        parse("""async def foo():
+
+            def foo(): pass
+
+            def foo(): pass
+
+            await x
+        """, "3.5")
+
+        parse("""async def foo(): return await a""", "3.5")
+
+        parse("""def foo():
+            def foo(): pass
+            async def foo(): await x
+        """, "3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_await_expr_invalid(self):
+        self.invalid_syntax("await x", version="3.5")
+        self.invalid_syntax("""def foo():
+                                   await x""", version="3.5")
+
+        self.invalid_syntax("""def foo():
+            def foo(): pass
+            async def foo(): pass
+            await x
+        """, version="3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_async_var(self):
+        parse("""async = 1""", "3.5")
+        parse("""await = 1""", "3.5")
+        parse("""def async(): pass""", "3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    def test_async_for(self):
+        parse("""async def foo():
+                             async for a in b: pass""", "3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_async_for_invalid(self):
+        self.invalid_syntax("""def foo():
+                                   async for a in b: pass""", version="3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    def test_async_with(self):
+        parse("""async def foo():
+                             async with a: pass""", "3.5")
+
+    @pytest.mark.skipif('sys.version_info[:2] < (3, 5)')
+    @pytest.mark.xfail(reason="acting like python 3.7")
+    def test_async_with_invalid(self):
+        self.invalid_syntax("""def foo():
+                                   async with a: pass""", version="3.5")
+
+
+class TestRaiseChanges(GrammarTest):
+    def test_2x_style_1(self):
+        parse("raise")
+
+    def test_2x_style_2(self):
+        parse("raise E, V", version='2.7')
+
+    def test_2x_style_3(self):
+        parse("raise E, V, T", version='2.7')
+
+    def test_2x_style_invalid_1(self):
+        self.invalid_syntax("raise E, V, T, Z", version='2.7')
+
+    def test_3x_style(self):
+        parse("raise E1 from E2")
+
+    def test_3x_style_invalid_1(self):
+        self.invalid_syntax("raise E, V from E1")
+
+    def test_3x_style_invalid_2(self):
+        self.invalid_syntax("raise E from E1, E2")
+
+    def test_3x_style_invalid_3(self):
+        self.invalid_syntax("raise from E1, E2")
+
+    def test_3x_style_invalid_4(self):
+        self.invalid_syntax("raise E from")
+
+
+# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
+class TestFunctionAnnotations(GrammarTest):
+    def test_1(self):
+        parse("""def f(x) -> list: pass""")
+
+    def test_2(self):
+        parse("""def f(x:int): pass""")
+
+    def test_3(self):
+        parse("""def f(*x:str): pass""")
+
+    def test_4(self):
+        parse("""def f(**x:float): pass""")
+
+    def test_5(self):
+        parse("""def f(x, y:1+2): pass""")
+
+    def test_6(self):
+        self.invalid_syntax("""def f(a, (b:1, c:2, d)): pass""")
+
+    def test_7(self):
+        self.invalid_syntax("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""")
+
+    def test_8(self):
+        s = """def f(a, (b:1, c:2, d), e:3=4, f=5,
+                        *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass"""
+        self.invalid_syntax(s)
+
+
+class TestExcept(GrammarTest):
+    def test_new(self):
+        s = """
+            try:
+                x
+            except E as N:
+                y"""
+        parse(s)
+
+    def test_old(self):
+        s = """
+            try:
+                x
+            except E, N:
+                y"""
+        parse(s, version='2.7')
+
+
+# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
+class TestSetLiteral(GrammarTest):
+    def test_1(self):
+        parse("""x = {'one'}""")
+
+    def test_2(self):
+        parse("""x = {'one', 1,}""")
+
+    def test_3(self):
+        parse("""x = {'one', 'two', 'three'}""")
+
+    def test_4(self):
+        parse("""x = {2, 3, 4,}""")
+
+
+class TestNumericLiterals(GrammarTest):
+    def test_new_octal_notation(self):
+        code = """0o7777777777777"""
+        if is_py3:
+            parse(code)
+        else:
+            self.invalid_syntax(code)
+        self.invalid_syntax("""0o7324528887""")
+
+    def test_new_binary_notation(self):
+        parse("""0b101010""")
+        self.invalid_syntax("""0b0101021""")
+
+
+class TestClassDef(GrammarTest):
+    def test_new_syntax(self):
+        parse("class B(t=7): pass")
+        parse("class B(t, *args): pass")
+        parse("class B(t, **kwargs): pass")
+        parse("class B(t, *args, **kwargs): pass")
+        parse("class B(t, y=9, *args, **kwargs): pass")
+
+
+class TestParserIdempotency(TestCase):
+    """A cut-down version of pytree_idempotency.py."""
+    def test_extended_unpacking(self):
+        parse("a, *b, c = x\n")
+        parse("[*a, b] = x\n")
+        parse("(z, *y, w) = m\n")
+        parse("for *z, m in d: pass\n")
+
+
+class TestLiterals(GrammarTest):
+    # It's not possible to get the same result when using \xaa in Python 2/3,
+    # because it's treated differently.
+    @pytest.mark.skipif('sys.version_info[0] < 3')
+    def test_multiline_bytes_literals(self):
+        s = """
+            md5test(b"\xaa" * 80,
+                    (b"Test Using Larger Than Block-Size Key "
+                     b"and Larger Than One Block-Size Data"),
+                    "6f630fad67cda0ee1fb1f562db3aa53e")
+            """
+        parse(s)
+
+    def test_multiline_bytes_tripquote_literals(self):
+        s = '''
+            b"""
+            <?xml version="1.0" encoding="UTF-8"?>
+            <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN">
+            """
+            '''
+        parse(s)
+
+    @pytest.mark.skipif('sys.version_info[0] < 3')
+    def test_multiline_str_literals(self):
+        s = """
+            md5test("\xaa" * 80,
+                    ("Test Using Larger Than Block-Size Key "
+                     "and Larger Than One Block-Size Data"),
+                    "6f630fad67cda0ee1fb1f562db3aa53e")
+            """
+        parse(s)

vim-plugins/bundle/jedi-vim/jedi/test/test_parser/test_tokenize.py

@@ -0,0 +1,152 @@
+# -*- coding: utf-8    # This file contains Unicode characters.
+
+from io import StringIO
+from textwrap import dedent
+
+import pytest
+
+from jedi._compatibility import u, is_py3
+from jedi.parser.token import NAME, OP, NEWLINE, STRING, INDENT
+from jedi.parser import ParserWithRecovery, load_grammar, tokenize
+
+
+from ..helpers import unittest
+
+
+class TokenTest(unittest.TestCase):
+    def test_end_pos_one_line(self):
+        parsed = ParserWithRecovery(load_grammar(), dedent(u('''
+        def testit():
+            a = "huhu"
+        ''')))
+        tok = parsed.module.subscopes[0].statements[0].children[2]
+        assert tok.end_pos == (3, 14)
+
+    def test_end_pos_multi_line(self):
+        parsed = ParserWithRecovery(load_grammar(), dedent(u('''
+        def testit():
+            a = """huhu
+        asdfasdf""" + "h"
+        ''')))
+        tok = parsed.module.subscopes[0].statements[0].children[2].children[0]
+        assert tok.end_pos == (4, 11)
+
+    def test_simple_no_whitespace(self):
+        # Test a simple one line string, no preceding whitespace
+        simple_docstring = u('"""simple one line docstring"""')
+        simple_docstring_io = StringIO(simple_docstring)
+        tokens = tokenize.generate_tokens(simple_docstring_io.readline)
+        token_list = list(tokens)
+        _, value, _, prefix = token_list[0]
+        assert prefix == ''
+        assert value == '"""simple one line docstring"""'
+
+    def test_simple_with_whitespace(self):
+        # Test a simple one line string with preceding whitespace and newline
+        simple_docstring = u('  """simple one line docstring""" \r\n')
+        simple_docstring_io = StringIO(simple_docstring)
+        tokens = tokenize.generate_tokens(simple_docstring_io.readline)
+        token_list = list(tokens)
+        assert token_list[0][0] == INDENT
+        typ, value, start_pos, prefix = token_list[1]
+        assert prefix == '  '
+        assert value == '"""simple one line docstring"""'
+        assert typ == STRING
+        typ, value, start_pos, prefix = token_list[2]
+        assert prefix == ' '
+        assert typ == NEWLINE
+
+    def test_function_whitespace(self):
+        # Test function definition whitespace identification
+        fundef = dedent(u('''
+        def test_whitespace(*args, **kwargs):
+            x = 1
+            if x > 0:
+                print(True)
+        '''))
+        fundef_io = StringIO(fundef)
+        tokens = tokenize.generate_tokens(fundef_io.readline)
+        token_list = list(tokens)
+        for _, value, _, prefix in token_list:
+            if value == 'test_whitespace':
+                assert prefix == ' '
+            if value == '(':
+                assert prefix == ''
+            if value == '*':
+                assert prefix == ''
+            if value == '**':
+                assert prefix == ' '
+            if value == 'print':
+                assert prefix == '        '
+            if value == 'if':
+                assert prefix == '    '
+
+    def test_identifier_contains_unicode(self):
+        fundef = dedent(u('''
+        def 我あφ():
+            pass
+        '''))
+        fundef_io = StringIO(fundef)
+        tokens = tokenize.generate_tokens(fundef_io.readline)
+        token_list = list(tokens)
+        unicode_token = token_list[1]
+        if is_py3:
+            assert unicode_token[0] == NAME
+        else:
+            # Unicode tokens in Python 2 seem to be identified as operators.
+            # They will be ignored in the parser, that's ok.
+            assert unicode_token[0] == OP
+
+    def test_quoted_strings(self):
+
+        string_tokens = [
+            'u"test"',
+            'u"""test"""',
+            'U"""test"""',
+            "u'''test'''",
+            "U'''test'''",
+        ]
+
+        for s in string_tokens:
+            parsed = ParserWithRecovery(load_grammar(), u('''a = %s\n''' % s))
+            simple_stmt = parsed.module.children[0]
+            expr_stmt = simple_stmt.children[0]
+            assert len(expr_stmt.children) == 3
+            string_tok = expr_stmt.children[2]
+            assert string_tok.type == 'string'
+            assert string_tok.value == s
+            assert string_tok.eval() == 'test'
+
+
+def test_tokenizer_with_string_literal_backslash():
+    import jedi
+    c = jedi.Script("statement = u'foo\\\n'; statement").goto_definitions()
+    assert c[0]._name.parent.obj == 'foo'
+
+
+def test_ur_literals():
+    """
+    Decided to parse `u''` literals regardless of Python version. This makes
+    probably sense:
+
+    - Python 3.2 doesn't support it and is still supported by Jedi, but might
+      not be. While this is incorrect, it's just incorrect for one "old" and in
+      the future not very important version.
+    - All the other Python versions work very well with it.
+    """
+    def check(literal):
+        io = StringIO(u(literal))
+        tokens = tokenize.generate_tokens(io.readline)
+        token_list = list(tokens)
+        typ, result_literal, _, _ = token_list[0]
+        assert typ == STRING
+        assert result_literal == literal
+
+    check('u""')
+    check('ur""')
+    check('Ur""')
+    check('UR""')
+    check('bR""')
+    # Must be in the right order.
+    with pytest.raises(AssertionError):
+        check('Rb""')

vim-plugins/bundle/jedi-vim/jedi/test/test_parser/test_user_context.py

@@ -0,0 +1,6 @@
+import jedi
+
+
+def test_form_feed_characters():
+    s = "\f\nclass Test(object):\n    pass"
+    jedi.Script(s, line=2, column=18).call_signatures()