Adding new stuff

vim-plugins/bundle/jedi-vim/jedi/jedi/__init__.py

@@ -0,0 +1,43 @@
+"""
+Jedi is a static analysis tool for Python that can be used in IDEs/editors. Its
+historic focus is autocompletion, but does static analysis for now as well.
+Jedi is fast and is very well tested. It understands Python on a deeper level
+than all other static analysis frameworks for Python.
+
+Jedi has support for two different goto functions. It's possible to search for
+related names and to list all names in a Python file and infer them. Jedi
+understands docstrings and you can use Jedi autocompletion in your REPL as
+well.
+
+Jedi uses a very simple API to connect with IDE's. There's a reference
+implementation as a `VIM-Plugin <https://github.com/davidhalter/jedi-vim>`_,
+which uses Jedi's autocompletion.  We encourage you to use Jedi in your IDEs.
+It's really easy.
+
+To give you a simple example how you can use the Jedi library, here is an
+example for the autocompletion feature:
+
+>>> import jedi
+>>> source = '''
+... import datetime
+... datetime.da'''
+>>> script = jedi.Script(source, 3, len('datetime.da'), 'example.py')
+>>> script
+<Script: 'example.py'>
+>>> completions = script.completions()
+>>> completions                                         #doctest: +ELLIPSIS
+[<Completion: date>, <Completion: datetime>, ...]
+>>> print(completions[0].complete)
+te
+>>> print(completions[0].name)
+date
+
+As you see Jedi is pretty simple and allows you to concentrate on writing a
+good text editor, while still having very good IDE features for Python.
+"""
+
+__version__ = '0.10.0'
+
+from jedi.api import Script, Interpreter, NotFoundError, set_debug_function
+from jedi.api import preload_module, defined_names, names
+from jedi import settings

vim-plugins/bundle/jedi-vim/jedi/jedi/__main__.py

@@ -0,0 +1,48 @@
+import sys
+from os.path import join, dirname, abspath, isdir
+
+
+def _start_linter():
+    """
+    This is a pre-alpha API. You're not supposed to use it at all, except for
+    testing. It will very likely change.
+    """
+    import jedi
+
+    if '--debug' in sys.argv:
+        jedi.set_debug_function()
+
+    for path in sys.argv[2:]:
+        if path.startswith('--'):
+            continue
+        if isdir(path):
+            import fnmatch
+            import os
+
+            paths = []
+            for root, dirnames, filenames in os.walk(path):
+                for filename in fnmatch.filter(filenames, '*.py'):
+                    paths.append(os.path.join(root, filename))
+        else:
+            paths = [path]
+
+        try:
+            for path in paths:
+                for error in jedi.Script(path=path)._analysis():
+                    print(error)
+        except Exception:
+            if '--pdb' in sys.argv:
+                import traceback
+                traceback.print_exc()
+                import pdb
+                pdb.post_mortem()
+            else:
+                raise
+
+
+if len(sys.argv) == 2 and sys.argv[1] == 'repl':
+    # don't want to use __main__ only for repl yet, maybe we want to use it for
+    # something else. So just use the keyword ``repl`` for now.
+    print(join(dirname(abspath(__file__)), 'api', 'replstartup.py'))
+elif len(sys.argv) > 1 and sys.argv[1] == 'linter':
+    _start_linter()

vim-plugins/bundle/jedi-vim/jedi/jedi/_compatibility.py

@@ -0,0 +1,263 @@
+"""
+To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been
+created. Clearly there is huge need to use conforming syntax.
+"""
+import sys
+import imp
+import os
+import re
+import pkgutil
+try:
+    import importlib
+except ImportError:
+    pass
+
+is_py3 = sys.version_info[0] >= 3
+is_py33 = is_py3 and sys.version_info.minor >= 3
+is_py34 = is_py3 and sys.version_info.minor >= 4
+is_py35 = is_py3 and sys.version_info.minor >= 5
+is_py26 = not is_py3 and sys.version_info[1] < 7
+
+
+class DummyFile(object):
+    def __init__(self, loader, string):
+        self.loader = loader
+        self.string = string
+
+    def read(self):
+        return self.loader.get_source(self.string)
+
+    def close(self):
+        del self.loader
+
+
+def find_module_py33(string, path=None):
+    loader = importlib.machinery.PathFinder.find_module(string, path)
+
+    if loader is None and path is None:  # Fallback to find builtins
+        try:
+            loader = importlib.find_loader(string)
+        except ValueError as e:
+            # See #491. Importlib might raise a ValueError, to avoid this, we
+            # just raise an ImportError to fix the issue.
+            raise ImportError("Originally  " + repr(e))
+
+    if loader is None:
+        raise ImportError("Couldn't find a loader for {0}".format(string))
+
+    try:
+        is_package = loader.is_package(string)
+        if is_package:
+            if hasattr(loader, 'path'):
+                module_path = os.path.dirname(loader.path)
+            else:
+                # At least zipimporter does not have path attribute
+                module_path = os.path.dirname(loader.get_filename(string))
+            if hasattr(loader, 'archive'):
+                module_file = DummyFile(loader, string)
+            else:
+                module_file = None
+        else:
+            module_path = loader.get_filename(string)
+            module_file = DummyFile(loader, string)
+    except AttributeError:
+        # ExtensionLoader has not attribute get_filename, instead it has a
+        # path attribute that we can use to retrieve the module path
+        try:
+            module_path = loader.path
+            module_file = DummyFile(loader, string)
+        except AttributeError:
+            module_path = string
+            module_file = None
+        finally:
+            is_package = False
+
+    if hasattr(loader, 'archive'):
+        module_path = loader.archive
+
+    return module_file, module_path, is_package
+
+
+def find_module_pre_py33(string, path=None):
+    try:
+        module_file, module_path, description = imp.find_module(string, path)
+        module_type = description[2]
+        return module_file, module_path, module_type is imp.PKG_DIRECTORY
+    except ImportError:
+        pass
+
+    if path is None:
+        path = sys.path
+    for item in path:
+        loader = pkgutil.get_importer(item)
+        if loader:
+            try:
+                loader = loader.find_module(string)
+                if loader:
+                    is_package = loader.is_package(string)
+                    is_archive = hasattr(loader, 'archive')
+                    try:
+                        module_path = loader.get_filename(string)
+                    except AttributeError:
+                        # fallback for py26
+                        try:
+                            module_path = loader._get_filename(string)
+                        except AttributeError:
+                            continue
+                    if is_package:
+                        module_path = os.path.dirname(module_path)
+                    if is_archive:
+                        module_path = loader.archive
+                    file = None
+                    if not is_package or is_archive:
+                        file = DummyFile(loader, string)
+                    return (file, module_path, is_package)
+            except ImportError:
+                pass
+    raise ImportError("No module named {0}".format(string))
+
+
+find_module = find_module_py33 if is_py33 else find_module_pre_py33
+find_module.__doc__ = """
+Provides information about a module.
+
+This function isolates the differences in importing libraries introduced with
+python 3.3 on; it gets a module name and optionally a path. It will return a
+tuple containin an open file for the module (if not builtin), the filename
+or the name of the module if it is a builtin one and a boolean indicating
+if the module is contained in a package.
+"""
+
+
+# unicode function
+try:
+    unicode = unicode
+except NameError:
+    unicode = str
+
+if is_py3:
+    u = lambda s: s
+else:
+    u = lambda s: s.decode('utf-8')
+
+u.__doc__ = """
+Decode a raw string into unicode object.  Do nothing in Python 3.
+"""
+
+# exec function
+if is_py3:
+    def exec_function(source, global_map):
+        exec(source, global_map)
+else:
+    eval(compile("""def exec_function(source, global_map):
+                        exec source in global_map """, 'blub', 'exec'))
+
+# re-raise function
+if is_py3:
+    def reraise(exception, traceback):
+        raise exception.with_traceback(traceback)
+else:
+    eval(compile("""
+def reraise(exception, traceback):
+    raise exception, None, traceback
+""", 'blub', 'exec'))
+
+reraise.__doc__ = """
+Re-raise `exception` with a `traceback` object.
+
+Usage::
+
+    reraise(Exception, sys.exc_info()[2])
+
+"""
+
+class Python3Method(object):
+    def __init__(self, func):
+        self.func = func
+
+    def __get__(self, obj, objtype):
+        if obj is None:
+            return lambda *args, **kwargs: self.func(*args, **kwargs)
+        else:
+            return lambda *args, **kwargs: self.func(obj, *args, **kwargs)
+
+
+def use_metaclass(meta, *bases):
+    """ Create a class with a metaclass. """
+    if not bases:
+        bases = (object,)
+    return meta("HackClass", bases, {})
+
+
+try:
+    encoding = sys.stdout.encoding
+    if encoding is None:
+        encoding = 'utf-8'
+except AttributeError:
+    encoding = 'ascii'
+
+
+def u(string):
+    """Cast to unicode DAMMIT!
+    Written because Python2 repr always implicitly casts to a string, so we
+    have to cast back to a unicode (and we now that we always deal with valid
+    unicode, because we check that in the beginning).
+    """
+    if is_py3:
+        return str(string)
+    elif not isinstance(string, unicode):
+        return unicode(str(string), 'UTF-8')
+    return string
+
+try:
+    import builtins  # module name in python 3
+except ImportError:
+    import __builtin__ as builtins
+
+
+import ast
+
+
+def literal_eval(string):
+    # py3.0, py3.1 and py32 don't support unicode literals. Support those, I
+    # don't want to write two versions of the tokenizer.
+    if is_py3 and sys.version_info.minor < 3:
+        if re.match('[uU][\'"]', string):
+            string = string[1:]
+    return ast.literal_eval(string)
+
+
+try:
+    from itertools import zip_longest
+except ImportError:
+    from itertools import izip_longest as zip_longest  # Python 2
+
+
+def no_unicode_pprint(dct):
+    """
+    Python 2/3 dict __repr__ may be different, because of unicode differens
+    (with or without a `u` prefix). Normally in doctests we could use `pprint`
+    to sort dicts and check for equality, but here we have to write a separate
+    function to do that.
+    """
+    import pprint
+    s = pprint.pformat(dct)
+    print(re.sub("u'", "'", s))
+
+
+def utf8_repr(func):
+    """
+    ``__repr__`` methods in Python 2 don't allow unicode objects to be
+    returned. Therefore cast them to utf-8 bytes in this decorator.
+    """
+    def wrapper(self):
+        result = func(self)
+        if isinstance(result, unicode):
+            return result.encode('utf-8')
+        else:
+            return result
+
+    if is_py3:
+        return func
+    else:
+        return wrapper

vim-plugins/bundle/jedi-vim/jedi/jedi/api/__init__.py

@@ -0,0 +1,463 @@
+"""
+The API basically only provides one class. You can create a :class:`Script` and
+use its methods.
+
+Additionally you can add a debug function with :func:`set_debug_function`.
+Alternatively, if you don't need a custom function and are happy with printing
+debug messages to stdout, simply call :func:`set_debug_function` without
+arguments.
+
+.. warning:: Please, note that Jedi is **not thread safe**.
+"""
+import os
+import warnings
+import sys
+
+from jedi._compatibility import unicode
+from jedi.parser import load_grammar
+from jedi.parser import tree
+from jedi.parser.fast import FastParser
+from jedi.parser.utils import save_parser
+from jedi import debug
+from jedi import settings
+from jedi import common
+from jedi import cache
+from jedi.api import classes
+from jedi.api import interpreter
+from jedi.api import usages
+from jedi.api import helpers
+from jedi.api.completion import Completion
+from jedi.evaluate import Evaluator
+from jedi.evaluate import representation as er
+from jedi.evaluate import imports
+from jedi.evaluate.param import try_iter_content
+from jedi.evaluate.helpers import get_module_names
+from jedi.evaluate.sys_path import get_venv_path
+from jedi.evaluate.iterable import unpack_tuple_to_dict
+
+# Jedi uses lots and lots of recursion. By setting this a little bit higher, we
+# can remove some "maximum recursion depth" errors.
+sys.setrecursionlimit(2000)
+
+
+class NotFoundError(Exception):
+    """A custom error to avoid catching the wrong exceptions.
+
+    .. deprecated:: 0.9.0
+       Not in use anymore, Jedi just returns no goto result if you're not on a
+       valid name.
+    .. todo:: Remove!
+    """
+
+
+class Script(object):
+    """
+    A Script is the base for completions, goto or whatever you want to do with
+    |jedi|.
+
+    You can either use the ``source`` parameter or ``path`` to read a file.
+    Usually you're going to want to use both of them (in an editor).
+
+    The script might be analyzed in a different ``sys.path`` than |jedi|:
+
+    - if `sys_path` parameter is not ``None``, it will be used as ``sys.path``
+      for the script;
+
+    - if `sys_path` parameter is ``None`` and ``VIRTUAL_ENV`` environment
+      variable is defined, ``sys.path`` for the specified environment will be
+      guessed (see :func:`jedi.evaluate.sys_path.get_venv_path`) and used for
+      the script;
+
+    - otherwise ``sys.path`` will match that of |jedi|.
+
+    :param source: The source code of the current file, separated by newlines.
+    :type source: str
+    :param line: The line to perform actions on (starting with 1).
+    :type line: int
+    :param column: The column of the cursor (starting with 0).
+    :type column: int
+    :param path: The path of the file in the file system, or ``''`` if
+        it hasn't been saved yet.
+    :type path: str or None
+    :param encoding: The encoding of ``source``, if it is not a
+        ``unicode`` object (default ``'utf-8'``).
+    :type encoding: str
+    :param source_encoding: The encoding of ``source``, if it is not a
+        ``unicode`` object (default ``'utf-8'``).
+    :type encoding: str
+    :param sys_path: ``sys.path`` to use during analysis of the script
+    :type sys_path: list
+
+    """
+    def __init__(self, source=None, line=None, column=None, path=None,
+                 encoding='utf-8', source_path=None, source_encoding=None,
+                 sys_path=None):
+        if source_path is not None:
+            warnings.warn("Use path instead of source_path.", DeprecationWarning)
+            path = source_path
+        if source_encoding is not None:
+            warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning)
+            encoding = source_encoding
+
+        self._orig_path = path
+        # An empty path (also empty string) should always result in no path.
+        self.path = os.path.abspath(path) if path else None
+
+        if source is None:
+            # TODO add a better warning than the traceback!
+            with open(path, 'rb') as f:
+                source = f.read()
+
+        self._source = common.source_to_unicode(source, encoding)
+        self._code_lines = common.splitlines(self._source)
+        line = max(len(self._code_lines), 1) if line is None else line
+        if not (0 < line <= len(self._code_lines)):
+            raise ValueError('`line` parameter is not in a valid range.')
+
+        line_len = len(self._code_lines[line - 1])
+        column = line_len if column is None else column
+        if not (0 <= column <= line_len):
+            raise ValueError('`column` parameter is not in a valid range.')
+        self._pos = line, column
+        self._path = path
+
+        cache.clear_time_caches()
+        debug.reset_time()
+        self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2])
+        if sys_path is None:
+            venv = os.getenv('VIRTUAL_ENV')
+            if venv:
+                sys_path = list(get_venv_path(venv))
+        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
+        debug.speed('init')
+
+    def _get_module(self):
+        cache.invalidate_star_import_cache(self._path)
+        parser = FastParser(self._grammar, self._source, self.path)
+        save_parser(self.path, parser, pickling=False)
+
+        module = self._evaluator.wrap(parser.module)
+        imports.add_module(self._evaluator, unicode(module.name), module)
+        return parser.module
+
+    @property
+    def source_path(self):
+        """
+        .. deprecated:: 0.7.0
+           Use :attr:`.path` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use path instead of source_path.", DeprecationWarning)
+        return self.path
+
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, repr(self._orig_path))
+
+    def completions(self):
+        """
+        Return :class:`classes.Completion` objects. Those objects contain
+        information about the completions, more than just names.
+
+        :return: Completion objects, sorted by name and __ comes last.
+        :rtype: list of :class:`classes.Completion`
+        """
+        debug.speed('completions start')
+        completion = Completion(
+            self._evaluator, self._get_module(), self._code_lines,
+            self._pos, self.call_signatures
+        )
+        completions = completion.completions()
+        debug.speed('completions end')
+        return completions
+
+    def goto_definitions(self):
+        """
+        Return the definitions of a the path under the cursor.  goto function!
+        This follows complicated paths and returns the end, not the first
+        definition. The big difference between :meth:`goto_assignments` and
+        :meth:`goto_definitions` is that :meth:`goto_assignments` doesn't
+        follow imports and statements. Multiple objects may be returned,
+        because Python itself is a dynamic language, which means depending on
+        an option you can have two different versions of a function.
+
+        :rtype: list of :class:`classes.Definition`
+        """
+        leaf = self._get_module().name_for_position(self._pos)
+        if leaf is None:
+            leaf = self._get_module().get_leaf_for_position(self._pos)
+            if leaf is None:
+                return []
+        definitions = helpers.evaluate_goto_definition(self._evaluator, leaf)
+
+        names = [s.name for s in definitions]
+        defs = [classes.Definition(self._evaluator, name) for name in names]
+        # The additional set here allows the definitions to become unique in an
+        # API sense. In the internals we want to separate more things than in
+        # the API.
+        return helpers.sorted_definitions(set(defs))
+
+    def goto_assignments(self, follow_imports=False):
+        """
+        Return the first definition found, while optionally following imports.
+        Multiple objects may be returned, because Python itself is a
+        dynamic language, which means depending on an option you can have two
+        different versions of a function.
+
+        :rtype: list of :class:`classes.Definition`
+        """
+        def filter_follow_imports(names):
+            for name in names:
+                definition = name.get_definition()
+                if definition.type in ('import_name', 'import_from'):
+                    imp = imports.ImportWrapper(self._evaluator, name)
+                    for name in filter_follow_imports(imp.follow(is_goto=True)):
+                        yield name
+                else:
+                    yield name
+
+        names = self._goto()
+        if follow_imports:
+            names = filter_follow_imports(names)
+
+        defs = [classes.Definition(self._evaluator, d) for d in set(names)]
+        return helpers.sorted_definitions(defs)
+
+    def _goto(self):
+        """
+        Used for goto_assignments and usages.
+        """
+        name = self._get_module().name_for_position(self._pos)
+        if name is None:
+            return []
+        return list(self._evaluator.goto(name))
+
+    def usages(self, additional_module_paths=()):
+        """
+        Return :class:`classes.Definition` objects, which contain all
+        names that point to the definition of the name under the cursor. This
+        is very useful for refactoring (renaming), or to show all usages of a
+        variable.
+
+        .. todo:: Implement additional_module_paths
+
+        :rtype: list of :class:`classes.Definition`
+        """
+        temp, settings.dynamic_flow_information = \
+            settings.dynamic_flow_information, False
+        try:
+            user_stmt = self._get_module().get_statement_for_position(self._pos)
+            definitions = self._goto()
+            if not definitions and isinstance(user_stmt, tree.Import):
+                # For not defined imports (goto doesn't find something, we take
+                # the name as a definition. This is enough, because every name
+                # points to it.
+                name = user_stmt.name_for_position(self._pos)
+                if name is None:
+                    # Must be syntax
+                    return []
+                definitions = [name]
+
+            if not definitions:
+                # Without a definition for a name we cannot find references.
+                return []
+
+            if not isinstance(user_stmt, tree.Import):
+                # import case is looked at with add_import_name option
+                definitions = usages.usages_add_import_modules(self._evaluator,
+                                                               definitions)
+
+            module = set([d.get_parent_until() for d in definitions])
+            module.add(self._get_module())
+            names = usages.usages(self._evaluator, definitions, module)
+
+            for d in set(definitions):
+                names.append(classes.Definition(self._evaluator, d))
+        finally:
+            settings.dynamic_flow_information = temp
+
+        return helpers.sorted_definitions(set(names))
+
+    def call_signatures(self):
+        """
+        Return the function object of the call you're currently in.
+
+        E.g. if the cursor is here::
+
+            abs(# <-- cursor is here
+
+        This would return the ``abs`` function. On the other hand::
+
+            abs()# <-- cursor is here
+
+        This would return an empty list..
+
+        :rtype: list of :class:`classes.CallSignature`
+        """
+        call_signature_details = \
+            helpers.get_call_signature_details(self._get_module(), self._pos)
+        if call_signature_details is None:
+            return []
+
+        with common.scale_speed_settings(settings.scale_call_signatures):
+            definitions = helpers.cache_call_signatures(
+                self._evaluator,
+                call_signature_details.bracket_leaf,
+                self._code_lines,
+                self._pos
+            )
+        debug.speed('func_call followed')
+
+        return [classes.CallSignature(self._evaluator, d.name,
+                                      call_signature_details.bracket_leaf.start_pos,
+                                      call_signature_details.call_index,
+                                      call_signature_details.keyword_name_str)
+                for d in definitions if hasattr(d, 'py__call__')]
+
+    def _analysis(self):
+        self._evaluator.is_analysis = True
+        self._evaluator.analysis_modules = [self._get_module()]
+        try:
+            for node in self._get_module().nodes_to_execute():
+                if node.type in ('funcdef', 'classdef'):
+                    if node.type == 'classdef':
+                        continue
+                        raise NotImplementedError
+                    er.Function(self._evaluator, node).get_decorated_func()
+                elif isinstance(node, tree.Import):
+                    import_names = set(node.get_defined_names())
+                    if node.is_nested():
+                        import_names |= set(path[-1] for path in node.paths())
+                    for n in import_names:
+                        imports.ImportWrapper(self._evaluator, n).follow()
+                elif node.type == 'expr_stmt':
+                    types = self._evaluator.eval_element(node)
+                    for testlist in node.children[:-1:2]:
+                        # Iterate tuples.
+                        unpack_tuple_to_dict(self._evaluator, types, testlist)
+                else:
+                    try_iter_content(self._evaluator.goto_definitions(node))
+                self._evaluator.reset_recursion_limitations()
+
+            ana = [a for a in self._evaluator.analysis if self.path == a.path]
+            return sorted(set(ana), key=lambda x: x.line)
+        finally:
+            self._evaluator.is_analysis = False
+
+
+class Interpreter(Script):
+    """
+    Jedi API for Python REPLs.
+
+    In addition to completion of simple attribute access, Jedi
+    supports code completion based on static code analysis.
+    Jedi can complete attributes of object which is not initialized
+    yet.
+
+    >>> from os.path import join
+    >>> namespace = locals()
+    >>> script = Interpreter('join("").up', [namespace])
+    >>> print(script.completions()[0].name)
+    upper
+    """
+
+    def __init__(self, source, namespaces, **kwds):
+        """
+        Parse `source` and mixin interpreted Python objects from `namespaces`.
+
+        :type source: str
+        :arg  source: Code to parse.
+        :type namespaces: list of dict
+        :arg  namespaces: a list of namespace dictionaries such as the one
+                          returned by :func:`locals`.
+
+        Other optional arguments are same as the ones for :class:`Script`.
+        If `line` and `column` are None, they are assumed be at the end of
+        `source`.
+        """
+        try:
+            namespaces = [dict(n) for n in namespaces]
+        except Exception:
+            raise TypeError("namespaces must be a non-empty list of dicts.")
+
+        super(Interpreter, self).__init__(source, **kwds)
+        self.namespaces = namespaces
+
+        parser_module = super(Interpreter, self)._get_module()
+        self._module = interpreter.MixedModule(self._evaluator, parser_module, self.namespaces)
+
+    def _get_module(self):
+        return self._module
+
+
+def defined_names(source, path=None, encoding='utf-8'):
+    """
+    Get all definitions in `source` sorted by its position.
+
+    This functions can be used for listing functions, classes and
+    data defined in a file.  This can be useful if you want to list
+    them in "sidebar".  Each element in the returned list also has
+    `defined_names` method which can be used to get sub-definitions
+    (e.g., methods in class).
+
+    :rtype: list of classes.Definition
+
+    .. deprecated:: 0.9.0
+       Use :func:`names` instead.
+    .. todo:: Remove!
+    """
+    warnings.warn("Use call_signatures instead.", DeprecationWarning)
+    return names(source, path, encoding)
+
+
+def names(source=None, path=None, encoding='utf-8', all_scopes=False,
+          definitions=True, references=False):
+    """
+    Returns a list of `Definition` objects, containing name parts.
+    This means you can call ``Definition.goto_assignments()`` and get the
+    reference of a name.
+    The parameters are the same as in :py:class:`Script`, except or the
+    following ones:
+
+    :param all_scopes: If True lists the names of all scopes instead of only
+        the module namespace.
+    :param definitions: If True lists the names that have been defined by a
+        class, function or a statement (``a = b`` returns ``a``).
+    :param references: If True lists all the names that are not listed by
+        ``definitions=True``. E.g. ``a = b`` returns ``b``.
+    """
+    def def_ref_filter(_def):
+        is_def = _def.is_definition()
+        return definitions and is_def or references and not is_def
+
+    # Set line/column to a random position, because they don't matter.
+    script = Script(source, line=1, column=0, path=path, encoding=encoding)
+    defs = [classes.Definition(script._evaluator, name_part)
+            for name_part in get_module_names(script._get_module(), all_scopes)]
+    return sorted(filter(def_ref_filter, defs), key=lambda x: (x.line, x.column))
+
+
+def preload_module(*modules):
+    """
+    Preloading modules tells Jedi to load a module now, instead of lazy parsing
+    of modules. Usful for IDEs, to control which modules to load on startup.
+
+    :param modules: different module names, list of string.
+    """
+    for m in modules:
+        s = "import %s as x; x." % m
+        Script(s, 1, len(s), None).completions()
+
+
+def set_debug_function(func_cb=debug.print_to_stdout, warnings=True,
+                       notices=True, speed=True):
+    """
+    Define a callback debug function to get all the debug messages.
+
+    If you don't specify any arguments, debug messages will be printed to stdout.
+
+    :param func_cb: The callback function for debug messages, with n params.
+    """
+    debug.debug_function = func_cb
+    debug.enable_warning = warnings
+    debug.enable_notice = notices
+    debug.enable_speed = speed

vim-plugins/bundle/jedi-vim/jedi/jedi/api/classes.py

@@ -0,0 +1,764 @@
+"""
+The :mod:`jedi.api.classes` module contains the return classes of the API.
+These classes are the much bigger part of the whole API, because they contain
+the interesting information about completion and goto operations.
+"""
+import warnings
+from itertools import chain
+import re
+
+from jedi._compatibility import unicode, use_metaclass
+from jedi import settings
+from jedi import common
+from jedi.parser import tree
+from jedi.parser.utils import load_parser
+from jedi.evaluate.cache import memoize_default, CachedMetaClass
+from jedi.evaluate import representation as er
+from jedi.evaluate import iterable
+from jedi.evaluate import imports
+from jedi.evaluate import compiled
+from jedi.evaluate.compiled import mixed
+from jedi.api import keywords
+from jedi.evaluate.finder import filter_definition_names
+
+
+def defined_names(evaluator, scope):
+    """
+    List sub-definitions (e.g., methods in class).
+
+    :type scope: Scope
+    :rtype: list of Definition
+    """
+    dct = scope.names_dict
+    names = list(chain.from_iterable(dct.values()))
+    names = filter_definition_names(names, scope)
+    return [Definition(evaluator, d) for d in sorted(names, key=lambda s: s.start_pos)]
+
+
+class BaseDefinition(object):
+    _mapping = {
+        'posixpath': 'os.path',
+        'riscospath': 'os.path',
+        'ntpath': 'os.path',
+        'os2emxpath': 'os.path',
+        'macpath': 'os.path',
+        'genericpath': 'os.path',
+        'posix': 'os',
+        '_io': 'io',
+        '_functools': 'functools',
+        '_sqlite3': 'sqlite3',
+        '__builtin__': '',
+        'builtins': '',
+    }
+
+    _tuple_mapping = dict((tuple(k.split('.')), v) for (k, v) in {
+        'argparse._ActionsContainer': 'argparse.ArgumentParser',
+        '_sre.SRE_Match': 're.MatchObject',
+        '_sre.SRE_Pattern': 're.RegexObject',
+    }.items())
+
+    def __init__(self, evaluator, name):
+        self._evaluator = evaluator
+        self._name = name
+        """
+        An instance of :class:`jedi.parser.reprsentation.Name` subclass.
+        """
+        self._definition = evaluator.wrap(self._name.get_definition())
+        self.is_keyword = isinstance(self._definition, keywords.Keyword)
+
+        # generate a path to the definition
+        self._module = name.get_parent_until()
+        if self.in_builtin_module():
+            self.module_path = None
+        else:
+            self.module_path = self._module.path
+            """Shows the file path of a module. e.g. ``/usr/lib/python2.7/os.py``"""
+
+    @property
+    def name(self):
+        """
+        Name of variable/function/class/module.
+
+        For example, for ``x = None`` it returns ``'x'``.
+
+        :rtype: str or None
+        """
+        return unicode(self._name)
+
+    @property
+    def start_pos(self):
+        """
+        .. deprecated:: 0.7.0
+           Use :attr:`.line` and :attr:`.column` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use line/column instead.", DeprecationWarning)
+        return self._name.start_pos
+
+    @property
+    def type(self):
+        """
+        The type of the definition.
+
+        Here is an example of the value of this attribute.  Let's consider
+        the following source.  As what is in ``variable`` is unambiguous
+        to Jedi, :meth:`jedi.Script.goto_definitions` should return a list of
+        definition for ``sys``, ``f``, ``C`` and ``x``.
+
+        >>> from jedi import Script
+        >>> source = '''
+        ... import keyword
+        ...
+        ... class C:
+        ...     pass
+        ...
+        ... class D:
+        ...     pass
+        ...
+        ... x = D()
+        ...
+        ... def f():
+        ...     pass
+        ...
+        ... for variable in [keyword, f, C, x]:
+        ...     variable'''
+
+        >>> script = Script(source)
+        >>> defs = script.goto_definitions()
+
+        Before showing what is in ``defs``, let's sort it by :attr:`line`
+        so that it is easy to relate the result to the source code.
+
+        >>> defs = sorted(defs, key=lambda d: d.line)
+        >>> defs                           # doctest: +NORMALIZE_WHITESPACE
+        [<Definition module keyword>, <Definition class C>,
+         <Definition class D>, <Definition def f>]
+
+        Finally, here is what you can get from :attr:`type`:
+
+        >>> defs[0].type
+        'module'
+        >>> defs[1].type
+        'class'
+        >>> defs[2].type
+        'instance'
+        >>> defs[3].type
+        'function'
+
+        """
+        stripped = self._definition
+        if isinstance(stripped, er.InstanceElement):
+            stripped = stripped.var
+
+        if isinstance(stripped, (compiled.CompiledObject, mixed.MixedObject)):
+            return stripped.api_type()
+        elif isinstance(stripped, iterable.Array):
+            return 'instance'
+        elif isinstance(stripped, tree.Import):
+            return 'import'
+
+        string = type(stripped).__name__.lower().replace('wrapper', '')
+        if string == 'exprstmt':
+            return 'statement'
+        else:
+            return string
+
+    def _path(self):
+        """The path to a module/class/function definition."""
+        path = []
+        par = self._definition
+        while par is not None:
+            if isinstance(par, tree.Import):
+                path += imports.ImportWrapper(self._evaluator, self._name).import_path
+                break
+            try:
+                name = par.name
+            except AttributeError:
+                pass
+            else:
+                if isinstance(par, er.ModuleWrapper):
+                    # TODO just make the path dotted from the beginning, we
+                    # shouldn't really split here.
+                    path[0:0] = par.py__name__().split('.')
+                    break
+                else:
+                    path.insert(0, unicode(name))
+            par = par.parent
+        return path
+
+    @property
+    def module_name(self):
+        """
+        The module name.
+
+        >>> from jedi import Script
+        >>> source = 'import json'
+        >>> script = Script(source, path='example.py')
+        >>> d = script.goto_definitions()[0]
+        >>> print(d.module_name)                       # doctest: +ELLIPSIS
+        json
+        """
+        return str(self._module.name)
+
+    def in_builtin_module(self):
+        """Whether this is a builtin module."""
+        return isinstance(self._module, compiled.CompiledObject)
+
+    @property
+    def line(self):
+        """The line where the definition occurs (starting with 1)."""
+        if self.in_builtin_module():
+            return None
+        return self._name.start_pos[0]
+
+    @property
+    def column(self):
+        """The column where the definition occurs (starting with 0)."""
+        if self.in_builtin_module():
+            return None
+        return self._name.start_pos[1]
+
+    def docstring(self, raw=False):
+        r"""
+        Return a document string for this completion object.
+
+        Example:
+
+        >>> from jedi import Script
+        >>> source = '''\
+        ... def f(a, b=1):
+        ...     "Document for function f."
+        ... '''
+        >>> script = Script(source, 1, len('def f'), 'example.py')
+        >>> doc = script.goto_definitions()[0].docstring()
+        >>> print(doc)
+        f(a, b=1)
+        <BLANKLINE>
+        Document for function f.
+
+        Notice that useful extra information is added to the actual
+        docstring.  For function, it is call signature.  If you need
+        actual docstring, use ``raw=True`` instead.
+
+        >>> print(script.goto_definitions()[0].docstring(raw=True))
+        Document for function f.
+
+        """
+        if raw:
+            return _Help(self._definition).raw()
+        else:
+            return _Help(self._definition).full()
+
+    @property
+    def doc(self):
+        """
+        .. deprecated:: 0.8.0
+           Use :meth:`.docstring` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use docstring() instead.", DeprecationWarning)
+        return self.docstring()
+
+    @property
+    def raw_doc(self):
+        """
+        .. deprecated:: 0.8.0
+           Use :meth:`.docstring` instead.
+        .. todo:: Remove!
+        """
+        warnings.warn("Use docstring() instead.", DeprecationWarning)
+        return self.docstring(raw=True)
+
+    @property
+    def description(self):
+        """A textual description of the object."""
+        return unicode(self._name)
+
+    @property
+    def full_name(self):
+        """
+        Dot-separated path of this object.
+
+        It is in the form of ``<module>[.<submodule>[...]][.<object>]``.
+        It is useful when you want to look up Python manual of the
+        object at hand.
+
+        Example:
+
+        >>> from jedi import Script
+        >>> source = '''
+        ... import os
+        ... os.path.join'''
+        >>> script = Script(source, 3, len('os.path.join'), 'example.py')
+        >>> print(script.goto_definitions()[0].full_name)
+        os.path.join
+
+        Notice that it correctly returns ``'os.path.join'`` instead of
+        (for example) ``'posixpath.join'``.
+
+        """
+        path = [unicode(p) for p in self._path()]
+        # TODO add further checks, the mapping should only occur on stdlib.
+        if not path:
+            return None  # for keywords the path is empty
+
+        with common.ignored(KeyError):
+            path[0] = self._mapping[path[0]]
+        for key, repl in self._tuple_mapping.items():
+            if tuple(path[:len(key)]) == key:
+                path = [repl] + path[len(key):]
+
+        return '.'.join(path if path[0] else path[1:])
+
+    def goto_assignments(self):
+        defs = self._evaluator.goto(self._name)
+        return [Definition(self._evaluator, d) for d in defs]
+
+    @memoize_default()
+    def _follow_statements_imports(self):
+        """
+        Follow both statements and imports, as far as possible.
+        """
+        if self._definition.isinstance(tree.ExprStmt):
+            return self._evaluator.eval_statement(self._definition)
+        elif self._definition.isinstance(tree.Import):
+            return imports.ImportWrapper(self._evaluator, self._name).follow()
+        else:
+            return set([self._definition])
+
+    @property
+    @memoize_default()
+    def params(self):
+        """
+        Raises an ``AttributeError``if the definition is not callable.
+        Otherwise returns a list of `Definition` that represents the params.
+        """
+        followed = list(self._follow_statements_imports())
+        if not followed or not hasattr(followed[0], 'py__call__'):
+            raise AttributeError()
+        followed = followed[0]  # only check the first one.
+
+        if followed.type in ('funcdef', 'lambda'):
+            if isinstance(followed, er.InstanceElement):
+                params = followed.params[1:]
+            else:
+                params = followed.params
+        elif followed.isinstance(er.compiled.CompiledObject):
+            params = followed.params
+        elif isinstance(followed, er.Class):
+            try:
+                sub = followed.get_subscope_by_name('__init__')
+                params = sub.params[1:]  # ignore self
+            except KeyError:
+                return []
+        elif isinstance(followed, er.Instance):
+            try:
+                sub = followed.get_subscope_by_name('__call__')
+                params = sub.params[1:]  # ignore self
+            except KeyError:
+                return []
+        else:
+            return []
+        return [_Param(self._evaluator, p.name) for p in params]
+
+    def parent(self):
+        scope = self._definition.get_parent_scope()
+        scope = self._evaluator.wrap(scope)
+        return Definition(self._evaluator, scope.name)
+
+    def __repr__(self):
+        return "<%s %s>" % (type(self).__name__, self.description)
+
+    def get_line_code(self, before=0, after=0):
+        """
+        Returns the line of code where this object was defined.
+
+        :param before: Add n lines before the current line to the output.
+        :param after: Add n lines after the current line to the output.
+
+        :return str: Returns the line(s) of code or an empty string if it's a
+                     builtin.
+        """
+        if self.in_builtin_module():
+            return ''
+
+        path = self._definition.get_parent_until().path
+        parser = load_parser(path)
+        lines = common.splitlines(parser.source)
+
+        line_nr = self._name.start_pos[0]
+        start_line_nr = line_nr - before
+        return '\n'.join(lines[start_line_nr:line_nr + after + 1])
+
+
+class Completion(BaseDefinition):
+    """
+    `Completion` objects are returned from :meth:`api.Script.completions`. They
+    provide additional information about a completion.
+    """
+    def __init__(self, evaluator, name, stack, like_name_length):
+        super(Completion, self).__init__(evaluator, name)
+
+        self._like_name_length = like_name_length
+        self._stack = stack
+
+        # Completion objects with the same Completion name (which means
+        # duplicate items in the completion)
+        self._same_name_completions = []
+
+    def _complete(self, like_name):
+        append = ''
+        if settings.add_bracket_after_function \
+                and self.type == 'Function':
+            append = '('
+
+        if isinstance(self._definition, tree.Param) and self._stack is not None:
+            node_names = list(self._stack.get_node_names(self._evaluator.grammar))
+            if 'trailer' in node_names and 'argument' not in node_names:
+                append += '='
+
+        name = str(self._name)
+        if like_name:
+            name = name[self._like_name_length:]
+        return name + append
+
+    @property
+    def complete(self):
+        """
+        Return the rest of the word, e.g. completing ``isinstance``::
+
+            isinstan# <-- Cursor is here
+
+        would return the string 'ce'. It also adds additional stuff, depending
+        on your `settings.py`.
+        """
+        return self._complete(True)
+
+    @property
+    def name_with_symbols(self):
+        """
+        Similar to :attr:`name`, but like :attr:`name`
+        returns also the symbols, for example::
+
+            list()
+
+        would return ``.append`` and others (which means it adds a dot).
+        """
+        return self._complete(False)
+
+    @property
+    def description(self):
+        """Provide a description of the completion object."""
+        if self._definition is None:
+            return ''
+        t = self.type
+        if t == 'statement' or t == 'import':
+            desc = self._definition.get_code()
+        else:
+            desc = '.'.join(unicode(p) for p in self._path())
+
+        line = '' if self.in_builtin_module else '@%s' % self.line
+        return '%s: %s%s' % (t, desc, line)
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, self._name)
+
+    def docstring(self, raw=False, fast=True):
+        """
+        :param fast: Don't follow imports that are only one level deep like
+            ``import foo``, but follow ``from foo import bar``. This makes
+            sense for speed reasons. Completing `import a` is slow if you use
+            the ``foo.docstring(fast=False)`` on every object, because it
+            parses all libraries starting with ``a``.
+        """
+        definition = self._definition
+        if isinstance(definition, tree.Import):
+            i = imports.ImportWrapper(self._evaluator, self._name)
+            if len(i.import_path) > 1 or not fast:
+                followed = self._follow_statements_imports()
+                if followed:
+                    # TODO: Use all of the followed objects as input to Documentation.
+                    definition = list(followed)[0]
+
+        if raw:
+            return _Help(definition).raw()
+        else:
+            return _Help(definition).full()
+
+    @property
+    def type(self):
+        """
+        The type of the completion objects. Follows imports. For a further
+        description, look at :attr:`jedi.api.classes.BaseDefinition.type`.
+        """
+        if isinstance(self._definition, tree.Import):
+            i = imports.ImportWrapper(self._evaluator, self._name)
+            if len(i.import_path) <= 1:
+                return 'module'
+
+            followed = self.follow_definition()
+            if followed:
+                # Caveat: Only follows the first one, ignore the other ones.
+                # This is ok, since people are almost never interested in
+                # variations.
+                return followed[0].type
+        return super(Completion, self).type
+
+    @memoize_default()
+    def _follow_statements_imports(self):
+        # imports completion is very complicated and needs to be treated
+        # separately in Completion.
+        definition = self._definition
+        if definition.isinstance(tree.Import):
+            i = imports.ImportWrapper(self._evaluator, self._name)
+            return i.follow()
+        return super(Completion, self)._follow_statements_imports()
+
+    @memoize_default()
+    def follow_definition(self):
+        """
+        Return the original definitions. I strongly recommend not using it for
+        your completions, because it might slow down |jedi|. If you want to
+        read only a few objects (<=20), it might be useful, especially to get
+        the original docstrings. The basic problem of this function is that it
+        follows all results. This means with 1000 completions (e.g.  numpy),
+        it's just PITA-slow.
+        """
+        defs = self._follow_statements_imports()
+        return [Definition(self._evaluator, d.name) for d in defs]
+
+
+class Definition(use_metaclass(CachedMetaClass, BaseDefinition)):
+    """
+    *Definition* objects are returned from :meth:`api.Script.goto_assignments`
+    or :meth:`api.Script.goto_definitions`.
+    """
+    def __init__(self, evaluator, definition):
+        super(Definition, self).__init__(evaluator, definition)
+
+    @property
+    def description(self):
+        """
+        A description of the :class:`.Definition` object, which is heavily used
+        in testing. e.g. for ``isinstance`` it returns ``def isinstance``.
+
+        Example:
+
+        >>> from jedi import Script
+        >>> source = '''
+        ... def f():
+        ...     pass
+        ...
+        ... class C:
+        ...     pass
+        ...
+        ... variable = f if random.choice([0,1]) else C'''
+        >>> script = Script(source, column=3)  # line is maximum by default
+        >>> defs = script.goto_definitions()
+        >>> defs = sorted(defs, key=lambda d: d.line)
+        >>> defs
+        [<Definition def f>, <Definition class C>]
+        >>> str(defs[0].description)  # strip literals in python2
+        'def f'
+        >>> str(defs[1].description)
+        'class C'
+
+        """
+        d = self._definition
+        if isinstance(d, er.InstanceElement):
+            d = d.var
+
+        if isinstance(d, compiled.CompiledObject):
+            typ = d.api_type()
+            if typ == 'instance':
+                typ = 'class'  # The description should be similar to Py objects.
+            d = typ + ' ' + d.name.get_code()
+        elif isinstance(d, iterable.Array):
+            d = 'class ' + d.type
+        elif isinstance(d, (tree.Class, er.Class, er.Instance)):
+            d = 'class ' + unicode(d.name)
+        elif isinstance(d, (er.Function, tree.Function)):
+            d = 'def ' + unicode(d.name)
+        elif isinstance(d, tree.Module):
+            # only show module name
+            d = 'module %s' % self.module_name
+        elif isinstance(d, tree.Param):
+            d = d.get_code().strip()
+            if d.endswith(','):
+                d = d[:-1]  # Remove the comma.
+        else:  # ExprStmt
+            try:
+                first_leaf = d.first_leaf()
+            except AttributeError:
+                # `d` is already a Leaf (Name).
+                first_leaf = d
+            # Remove the prefix, because that's not what we want for get_code
+            # here.
+            old, first_leaf.prefix = first_leaf.prefix, ''
+            try:
+                d = d.get_code()
+            finally:
+                first_leaf.prefix = old
+        # Delete comments:
+        d = re.sub('#[^\n]+\n', ' ', d)
+        # Delete multi spaces/newlines
+        return re.sub('\s+', ' ', d).strip()
+
+    @property
+    def desc_with_module(self):
+        """
+        In addition to the definition, also return the module.
+
+        .. warning:: Don't use this function yet, its behaviour may change. If
+            you really need it, talk to me.
+
+        .. todo:: Add full path. This function is should return a
+            `module.class.function` path.
+        """
+        position = '' if self.in_builtin_module else '@%s' % (self.line)
+        return "%s:%s%s" % (self.module_name, self.description, position)
+
+    @memoize_default()
+    def defined_names(self):
+        """
+        List sub-definitions (e.g., methods in class).
+
+        :rtype: list of Definition
+        """
+        defs = self._follow_statements_imports()
+        # For now we don't want base classes or evaluate decorators.
+        defs = [d.base if isinstance(d, (er.Class, er.Function)) else d for d in defs]
+        iterable = (defined_names(self._evaluator, d) for d in defs)
+        iterable = list(iterable)
+        return list(chain.from_iterable(iterable))
+
+    def is_definition(self):
+        """
+        Returns True, if defined as a name in a statement, function or class.
+        Returns False, if it's a reference to such a definition.
+        """
+        return self._name.is_definition()
+
+    def __eq__(self, other):
+        return self._name.start_pos == other._name.start_pos \
+            and self.module_path == other.module_path \
+            and self.name == other.name \
+            and self._evaluator == other._evaluator
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return hash((self._name.start_pos, self.module_path, self.name, self._evaluator))
+
+
+class CallSignature(Definition):
+    """
+    `CallSignature` objects is the return value of `Script.function_definition`.
+    It knows what functions you are currently in. e.g. `isinstance(` would
+    return the `isinstance` function. without `(` it would return nothing.
+    """
+    def __init__(self, evaluator, executable_name, bracket_start_pos, index, key_name_str):
+        super(CallSignature, self).__init__(evaluator, executable_name)
+        self._index = index
+        self._key_name_str = key_name_str
+        self._bracket_start_pos = bracket_start_pos
+
+    @property
+    def index(self):
+        """
+        The Param index of the current call.
+        Returns None if the index cannot be found in the curent call.
+        """
+        if self._key_name_str is not None:
+            for i, param in enumerate(self.params):
+                if self._key_name_str == param.name:
+                    return i
+            if self.params and self.params[-1]._name.get_definition().stars == 2:
+                return i
+            else:
+                return None
+
+        if self._index >= len(self.params):
+
+            for i, param in enumerate(self.params):
+                # *args case
+                if param._name.get_definition().stars == 1:
+                    return i
+            return None
+        return self._index
+
+    @property
+    def bracket_start(self):
+        """
+        The indent of the bracket that is responsible for the last function
+        call.
+        """
+        return self._bracket_start_pos
+
+    @property
+    def call_name(self):
+        """
+        .. deprecated:: 0.8.0
+           Use :attr:`.name` instead.
+        .. todo:: Remove!
+
+        The name (e.g. 'isinstance') as a string.
+        """
+        warnings.warn("Use name instead.", DeprecationWarning)
+        return unicode(self.name)
+
+    @property
+    def module(self):
+        """
+        .. deprecated:: 0.8.0
+           Use :attr:`.module_name` for the module name.
+        .. todo:: Remove!
+        """
+        return self._executable.get_parent_until()
+
+    def __repr__(self):
+        return '<%s: %s index %s>' % (type(self).__name__, self._name,
+                                      self.index)
+
+
+class _Param(Definition):
+    """
+    Just here for backwards compatibility.
+    """
+    def get_code(self):
+        """
+        .. deprecated:: 0.8.0
+           Use :attr:`.description` and :attr:`.name` instead.
+        .. todo:: Remove!
+
+        A function to get the whole code of the param.
+        """
+        warnings.warn("Use description instead.", DeprecationWarning)
+        return self.description
+
+
+class _Help(object):
+    """
+    Temporary implementation, will be used as `Script.help() or something in
+    the future.
+    """
+    def __init__(self, definition):
+        self._name = definition
+
+    def full(self):
+        try:
+            return self._name.doc
+        except AttributeError:
+            return self.raw()
+
+    def raw(self):
+        """
+        The raw docstring ``__doc__`` for any object.
+
+        See :attr:`doc` for example.
+        """
+        try:
+            return self._name.raw_doc
+        except AttributeError:
+            return ''

vim-plugins/bundle/jedi-vim/jedi/jedi/api/completion.py

@@ -0,0 +1,256 @@
+from itertools import chain
+
+from jedi.parser import token
+from jedi.parser import tree
+from jedi import debug
+from jedi import settings
+from jedi.api import classes
+from jedi.api import helpers
+from jedi.evaluate import imports
+from jedi.api import keywords
+from jedi.evaluate import compiled
+from jedi.evaluate.helpers import call_of_leaf
+from jedi.evaluate.finder import global_names_dict_generator, filter_definition_names
+
+
+def get_call_signature_param_names(call_signatures):
+    # add named params
+    for call_sig in call_signatures:
+        # Allow protected access, because it's a public API.
+        module = call_sig._name.get_parent_until()
+        # Compiled modules typically don't allow keyword arguments.
+        if not isinstance(module, compiled.CompiledObject):
+            for p in call_sig.params:
+                # Allow access on _definition here, because it's a
+                # public API and we don't want to make the internal
+                # Name object public.
+                if p._definition.stars == 0:  # no *args/**kwargs
+                    yield p._name
+
+
+def filter_names(evaluator, completion_names, stack, like_name):
+    comp_dct = {}
+    for name in set(completion_names):
+        if settings.case_insensitive_completion \
+                and str(name).lower().startswith(like_name.lower()) \
+                or str(name).startswith(like_name):
+
+            if isinstance(name.parent, (tree.Function, tree.Class)):
+                # TODO I think this is a hack. It should be an
+                #   er.Function/er.Class before that.
+                name = evaluator.wrap(name.parent).name
+            new = classes.Completion(
+                evaluator,
+                name,
+                stack,
+                len(like_name)
+            )
+            k = (new.name, new.complete)  # key
+            if k in comp_dct and settings.no_completion_duplicates:
+                comp_dct[k]._same_name_completions.append(new)
+            else:
+                comp_dct[k] = new
+                yield new
+
+
+def get_user_scope(module, position):
+    """
+    Returns the scope in which the user resides. This includes flows.
+    """
+    user_stmt = module.get_statement_for_position(position)
+    if user_stmt is None:
+        def scan(scope):
+            for s in scope.children:
+                if s.start_pos <= position <= s.end_pos:
+                    if isinstance(s, (tree.Scope, tree.Flow)):
+                        return scan(s) or s
+                    elif s.type in ('suite', 'decorated'):
+                        return scan(s)
+            return None
+
+        return scan(module) or module
+    else:
+        return user_stmt.get_parent_scope(include_flows=True)
+
+
+class Completion:
+    def __init__(self, evaluator, module, code_lines, position, call_signatures_method):
+        self._evaluator = evaluator
+        self._module = evaluator.wrap(module)
+        self._code_lines = code_lines
+
+        # The first step of completions is to get the name
+        self._like_name = helpers.get_on_completion_name(module, code_lines, position)
+        # The actual cursor position is not what we need to calculate
+        # everything. We want the start of the name we're on.
+        self._position = position[0], position[1] - len(self._like_name)
+        self._call_signatures_method = call_signatures_method
+
+    def completions(self):
+        completion_names = self._get_context_completions()
+
+        completions = filter_names(self._evaluator, completion_names,
+                                   self.stack, self._like_name)
+
+        return sorted(completions, key=lambda x: (x.name.startswith('__'),
+                                                  x.name.startswith('_'),
+                                                  x.name.lower()))
+
+    def _get_context_completions(self):
+        """
+        Analyzes the context that a completion is made in and decides what to
+        return.
+
+        Technically this works by generating a parser stack and analysing the
+        current stack for possible grammar nodes.
+
+        Possible enhancements:
+        - global/nonlocal search global
+        - yield from / raise from <- could be only exceptions/generators
+        - In args: */**: no completion
+        - In params (also lambda): no completion before =
+        """
+
+        grammar = self._evaluator.grammar
+
+        try:
+            self.stack = helpers.get_stack_at_position(
+                grammar, self._code_lines, self._module, self._position
+            )
+        except helpers.OnErrorLeaf as e:
+            self.stack = None
+            if e.error_leaf.value == '.':
+                # After ErrorLeaf's that are dots, we will not do any
+                # completions since this probably just confuses the user.
+                return []
+            # If we don't have a context, just use global completion.
+
+            return self._global_completions()
+
+        allowed_keywords, allowed_tokens = \
+            helpers.get_possible_completion_types(grammar, self.stack)
+
+        completion_names = list(self._get_keyword_completion_names(allowed_keywords))
+
+        if token.NAME in allowed_tokens:
+            # This means that we actually have to do type inference.
+
+            symbol_names = list(self.stack.get_node_names(grammar))
+
+            nodes = list(self.stack.get_nodes())
+
+            if "import_stmt" in symbol_names:
+                level = 0
+                only_modules = True
+                level, names = self._parse_dotted_names(nodes)
+                if "import_from" in symbol_names:
+                    if 'import' in nodes:
+                        only_modules = False
+                else:
+                    assert "import_name" in symbol_names
+
+                completion_names += self._get_importer_names(
+                    names,
+                    level,
+                    only_modules
+                )
+            elif nodes and nodes[-1] in ('as', 'def', 'class'):
+                # No completions for ``with x as foo`` and ``import x as foo``.
+                # Also true for defining names as a class or function.
+                return list(self._get_class_context_completions(is_function=True))
+            elif symbol_names[-1] == 'trailer' and nodes[-1] == '.':
+                dot = self._module.get_leaf_for_position(self._position)
+                atom_expr = call_of_leaf(dot.get_previous_leaf())
+                completion_names += self._trailer_completions(atom_expr)
+            else:
+                completion_names += self._global_completions()
+                completion_names += self._get_class_context_completions(is_function=False)
+
+            if 'trailer' in symbol_names:
+                call_signatures = self._call_signatures_method()
+                completion_names += get_call_signature_param_names(call_signatures)
+
+        return completion_names
+
+    def _get_keyword_completion_names(self, keywords_):
+        for k in keywords_:
+            yield keywords.keyword(self._evaluator, k).name
+
+    def _global_completions(self):
+        scope = get_user_scope(self._module, self._position)
+        if not scope.is_scope():  # Might be a flow (if/while/etc).
+            scope = scope.get_parent_scope()
+        scope = self._evaluator.wrap(scope)
+        debug.dbg('global completion scope: %s', scope)
+        names_dicts = global_names_dict_generator(
+            self._evaluator,
+            scope,
+            self._position
+        )
+        completion_names = []
+        for names_dict, pos in names_dicts:
+            names = list(chain.from_iterable(names_dict.values()))
+            if not names:
+                continue
+            completion_names += filter_definition_names(
+                names, self._module.get_statement_for_position(self._position), pos
+            )
+        return completion_names
+
+    def _trailer_completions(self, atom_expr):
+        scopes = self._evaluator.eval_element(atom_expr)
+        completion_names = []
+        debug.dbg('trailer completion scopes: %s', scopes)
+        for s in scopes:
+            names = []
+            for names_dict in s.names_dicts(search_global=False):
+                names += chain.from_iterable(names_dict.values())
+
+            completion_names += filter_definition_names(
+                names, self._module.get_statement_for_position(self._position)
+            )
+        return completion_names
+
+    def _parse_dotted_names(self, nodes):
+        level = 0
+        names = []
+        for node in nodes[1:]:
+            if node in ('.', '...'):
+                if not names:
+                    level += len(node.value)
+            elif node.type == 'dotted_name':
+                names += node.children[::2]
+            elif node.type == 'name':
+                names.append(node)
+            else:
+                break
+        return level, names
+
+    def _get_importer_names(self, names, level=0, only_modules=True):
+        names = [str(n) for n in names]
+        i = imports.Importer(self._evaluator, names, self._module, level)
+        return i.completion_names(self._evaluator, only_modules=only_modules)
+
+    def _get_class_context_completions(self, is_function=True):
+        """
+        Autocomplete inherited methods when overriding in child class.
+        """
+        leaf = self._module.get_leaf_for_position(self._position, include_prefixes=True)
+        cls = leaf.get_parent_until(tree.Class)
+        if isinstance(cls, (tree.Class, tree.Function)):
+            # Complete the methods that are defined in the super classes.
+            cls = self._evaluator.wrap(cls)
+        else:
+            return
+
+        if cls.start_pos[1] >= leaf.start_pos[1]:
+            return
+
+        names_dicts = cls.names_dicts(search_global=False, is_instance=True)
+        # The first dict is the dictionary of class itself.
+        next(names_dicts)
+        for names_dict in names_dicts:
+            for values in names_dict.values():
+                for value in values:
+                    if (value.parent.type == 'funcdef') == is_function:
+                        yield value

vim-plugins/bundle/jedi-vim/jedi/jedi/api/helpers.py

@@ -0,0 +1,315 @@
+"""
+Helpers for the API
+"""
+import re
+from collections import namedtuple
+
+from jedi._compatibility import u
+from jedi.evaluate.helpers import call_of_leaf
+from jedi import parser
+from jedi.parser import tokenize
+from jedi.cache import time_cache
+from jedi import common
+
+
+CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
+
+
+def sorted_definitions(defs):
+    # Note: `or ''` below is required because `module_path` could be
+    return sorted(defs, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0))
+
+
+def get_on_completion_name(module, lines, position):
+    leaf = module.get_leaf_for_position(position)
+    if leaf is None or leaf.type in ('string', 'error_leaf'):
+        # Completions inside strings are a bit special, we need to parse the
+        # string. The same is true for comments and error_leafs.
+        line = lines[position[0] - 1]
+        # The first step of completions is to get the name
+        return re.search(r'(?!\d)\w+$|$', line[:position[1]]).group(0)
+    elif leaf.type not in ('name', 'keyword'):
+        return ''
+
+    return leaf.value[:position[1] - leaf.start_pos[1]]
+
+
+def _get_code(code_lines, start_pos, end_pos):
+    # Get relevant lines.
+    lines = code_lines[start_pos[0] - 1:end_pos[0]]
+    # Remove the parts at the end of the line.
+    lines[-1] = lines[-1][:end_pos[1]]
+    # Remove first line indentation.
+    lines[0] = lines[0][start_pos[1]:]
+    return '\n'.join(lines)
+
+
+class OnErrorLeaf(Exception):
+    @property
+    def error_leaf(self):
+        return self.args[0]
+
+
+def _is_on_comment(leaf, position):
+    # We might be on a comment.
+    if leaf.type == 'endmarker':
+        try:
+            dedent = leaf.get_previous_leaf()
+            if dedent.type == 'dedent' and dedent.prefix:
+                # TODO This is needed because the fast parser uses multiple
+                # endmarker tokens within a file which is obviously ugly.
+                # This is so ugly that I'm not even commenting how it exactly
+                # happens, but let me tell you that I want to get rid of it.
+                leaf = dedent
+        except IndexError:
+            pass
+
+    comment_lines = common.splitlines(leaf.prefix)
+    difference = leaf.start_pos[0] - position[0]
+    prefix_start_pos = leaf.get_start_pos_of_prefix()
+    if difference == 0:
+        indent = leaf.start_pos[1]
+    elif position[0] == prefix_start_pos[0]:
+        indent = prefix_start_pos[1]
+    else:
+        indent = 0
+    line = comment_lines[-difference - 1][:position[1] - indent]
+    return '#' in line
+
+
+def _get_code_for_stack(code_lines, module, position):
+    leaf = module.get_leaf_for_position(position, include_prefixes=True)
+    # It might happen that we're on whitespace or on a comment. This means
+    # that we would not get the right leaf.
+    if leaf.start_pos >= position:
+        if _is_on_comment(leaf, position):
+            return u('')
+
+        # If we're not on a comment simply get the previous leaf and proceed.
+        try:
+            leaf = leaf.get_previous_leaf()
+        except IndexError:
+            return u('')  # At the beginning of the file.
+
+    is_after_newline = leaf.type == 'newline'
+    while leaf.type == 'newline':
+        try:
+            leaf = leaf.get_previous_leaf()
+        except IndexError:
+            return u('')
+
+    if leaf.type in ('indent', 'dedent'):
+        return u('')
+    elif leaf.type == 'error_leaf' or leaf.type == 'string':
+        # Error leafs cannot be parsed, completion in strings is also
+        # impossible.
+        raise OnErrorLeaf(leaf)
+    else:
+        if leaf == ';':
+            user_stmt = leaf.parent
+        else:
+            user_stmt = leaf.get_definition()
+        if user_stmt.parent.type == 'simple_stmt':
+            user_stmt = user_stmt.parent
+
+        if is_after_newline:
+            if user_stmt.start_pos[1] > position[1]:
+                # This means that it's actually a dedent and that means that we
+                # start without context (part of a suite).
+                return u('')
+
+        # This is basically getting the relevant lines.
+        return _get_code(code_lines, user_stmt.get_start_pos_of_prefix(), position)
+
+
+def get_stack_at_position(grammar, code_lines, module, pos):
+    """
+    Returns the possible node names (e.g. import_from, xor_test or yield_stmt).
+    """
+    class EndMarkerReached(Exception):
+        pass
+
+    def tokenize_without_endmarker(code):
+        tokens = tokenize.source_tokens(code, use_exact_op_types=True)
+        for token_ in tokens:
+            if token_.string == safeword:
+                raise EndMarkerReached()
+            else:
+                yield token_
+
+    code = _get_code_for_stack(code_lines, module, pos)
+    # We use a word to tell Jedi when we have reached the start of the
+    # completion.
+    # Use Z as a prefix because it's not part of a number suffix.
+    safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI'
+    # Remove as many indents from **all** code lines as possible.
+    code = code + safeword
+
+    p = parser.ParserWithRecovery(grammar, code, start_parsing=False)
+    try:
+        p.parse(tokenizer=tokenize_without_endmarker(code))
+    except EndMarkerReached:
+        return Stack(p.stack)
+    raise SystemError("This really shouldn't happen. There's a bug in Jedi.")
+
+
+class Stack(list):
+    def get_node_names(self, grammar):
+        for dfa, state, (node_number, nodes) in self:
+            yield grammar.number2symbol[node_number]
+
+    def get_nodes(self):
+        for dfa, state, (node_number, nodes) in self:
+            for node in nodes:
+                yield node
+
+
+def get_possible_completion_types(grammar, stack):
+    def add_results(label_index):
+        try:
+            grammar_labels.append(inversed_tokens[label_index])
+        except KeyError:
+            try:
+                keywords.append(inversed_keywords[label_index])
+            except KeyError:
+                t, v = grammar.labels[label_index]
+                assert t >= 256
+                # See if it's a symbol and if we're in its first set
+                inversed_keywords
+                itsdfa = grammar.dfas[t]
+                itsstates, itsfirst = itsdfa
+                for first_label_index in itsfirst.keys():
+                    add_results(first_label_index)
+
+    inversed_keywords = dict((v, k) for k, v in grammar.keywords.items())
+    inversed_tokens = dict((v, k) for k, v in grammar.tokens.items())
+
+    keywords = []
+    grammar_labels = []
+
+    def scan_stack(index):
+        dfa, state, node = stack[index]
+        states, first = dfa
+        arcs = states[state]
+
+        for label_index, new_state in arcs:
+            if label_index == 0:
+                # An accepting state, check the stack below.
+                scan_stack(index - 1)
+            else:
+                add_results(label_index)
+
+    scan_stack(-1)
+
+    return keywords, grammar_labels
+
+
+def evaluate_goto_definition(evaluator, leaf):
+    if leaf.type == 'name':
+        # In case of a name we can just use goto_definition which does all the
+        # magic itself.
+        return evaluator.goto_definitions(leaf)
+
+    node = None
+    parent = leaf.parent
+    if parent.type == 'atom':
+        node = leaf.parent
+    elif parent.type == 'trailer':
+        node = call_of_leaf(leaf)
+
+    if node is None:
+        return []
+    return evaluator.eval_element(node)
+
+
+CallSignatureDetails = namedtuple(
+    'CallSignatureDetails',
+    ['bracket_leaf', 'call_index', 'keyword_name_str']
+)
+
+
+def _get_index_and_key(nodes, position):
+    """
+    Returns the amount of commas and the keyword argument string.
+    """
+    nodes_before = [c for c in nodes if c.start_pos < position]
+    if nodes_before[-1].type == 'arglist':
+        nodes_before = [c for c in nodes_before[-1].children if c.start_pos < position]
+
+    key_str = None
+
+    if nodes_before:
+        last = nodes_before[-1]
+        if last.type == 'argument' and last.children[1].end_pos <= position:
+            # Checked if the argument
+            key_str = last.children[0].value
+        elif last == '=':
+            key_str = nodes_before[-2].value
+
+    return nodes_before.count(','), key_str
+
+
+def _get_call_signature_details_from_error_node(node, position):
+    for index, element in reversed(list(enumerate(node.children))):
+        # `index > 0` means that it's a trailer and not an atom.
+        if element == '(' and element.end_pos <= position and index > 0:
+            # It's an error node, we don't want to match too much, just
+            # until the parentheses is enough.
+            children = node.children[index:]
+            name = element.get_previous_leaf()
+            if name.type == 'name' or name.parent.type in ('trailer', 'atom'):
+                return CallSignatureDetails(
+                    element,
+                    *_get_index_and_key(children, position)
+                )
+
+
+def get_call_signature_details(module, position):
+    leaf = module.get_leaf_for_position(position, include_prefixes=True)
+    if leaf == ')':
+        if leaf.end_pos == position:
+            leaf = leaf.get_next_leaf()
+    # Now that we know where we are in the syntax tree, we start to look at
+    # parents for possible function definitions.
+    node = leaf.parent
+    while node is not None:
+        if node.type in ('funcdef', 'classdef'):
+            # Don't show call signatures if there's stuff before it that just
+            # makes it feel strange to have a call signature.
+            return None
+
+        for n in node.children[::-1]:
+            if n.start_pos < position and n.type == 'error_node':
+                result = _get_call_signature_details_from_error_node(n, position)
+                if result is not None:
+                    return result
+
+        if node.type == 'trailer' and node.children[0] == '(':
+            leaf = node.get_previous_leaf()
+            return CallSignatureDetails(
+                node.children[0], *_get_index_and_key(node.children, position))
+
+        node = node.parent
+
+    return None
+
+
+@time_cache("call_signatures_validity")
+def cache_call_signatures(evaluator, bracket_leaf, code_lines, user_pos):
+    """This function calculates the cache key."""
+    index = user_pos[0] - 1
+
+    before_cursor = code_lines[index][:user_pos[1]]
+    other_lines = code_lines[bracket_leaf.start_pos[0]:index]
+    whole = '\n'.join(other_lines + [before_cursor])
+    before_bracket = re.match(r'.*\(', whole, re.DOTALL)
+
+    module_path = bracket_leaf.get_parent_until().path
+    if module_path is None:
+        yield None  # Don't cache!
+    else:
+        yield (module_path, before_bracket, bracket_leaf.start_pos)
+    yield evaluate_goto_definition(
+        evaluator,
+        bracket_leaf.get_previous_leaf()
+    )

vim-plugins/bundle/jedi-vim/jedi/jedi/api/interpreter.py

@@ -0,0 +1,67 @@
+"""
+TODO Some parts of this module are still not well documented.
+"""
+import copy
+
+from jedi.cache import underscore_memoization
+from jedi.evaluate import helpers
+from jedi.evaluate.representation import ModuleWrapper
+from jedi.evaluate.compiled import mixed
+
+
+class MixedModule(object):
+    resets_positions = True
+    type = 'mixed_module'
+
+    def __init__(self, evaluator, parser_module, namespaces):
+        self._evaluator = evaluator
+        self._namespaces = namespaces
+
+        self._namespace_objects = [type('jedi_namespace', (), n) for n in namespaces]
+        self._wrapped_module = ModuleWrapper(evaluator, parser_module)
+        # Usually we are dealing with very small code sizes when it comes to
+        # interpreter modules. In this case we just copy the whole syntax tree
+        # to be able to modify it.
+        self._parser_module = copy.deepcopy(parser_module)
+
+        for child in self._parser_module.children:
+            child.parent = self
+
+    def names_dicts(self, search_global):
+        for names_dict in self._wrapped_module.names_dicts(search_global):
+            yield names_dict
+
+        for namespace_obj in self._namespace_objects:
+            m = mixed.MixedObject(self._evaluator, namespace_obj, self._parser_module.name)
+            for names_dict in m.names_dicts(False):
+                yield names_dict
+
+    def __getattr__(self, name):
+        return getattr(self._parser_module, name)
+
+
+class LazyName(helpers.FakeName):
+    def __init__(self, evaluator, module, name, value):
+        super(LazyName, self).__init__(name)
+        self._module = module
+        self._evaluator = evaluator
+        self._value = value
+        self._name = name
+
+    def is_definition(self):
+        return True
+
+    @property
+    @underscore_memoization
+    def parent(self):
+        """
+        Creating fake statements for the interpreter.
+
+        Here we are trying to link back to Python code, if possible. This means
+        we try to find the python module for a name (not the builtin).
+        """
+        return mixed.create(self._evaluator, self._value)
+
+    @parent.setter
+    def parent(self, value):
+        """Needed because the super class tries to set parent."""

vim-plugins/bundle/jedi-vim/jedi/jedi/api/keywords.py

@@ -0,0 +1,132 @@
+import pydoc
+import keyword
+
+from jedi._compatibility import is_py3, is_py35
+from jedi import common
+from jedi.evaluate.helpers import FakeName
+from jedi.parser.tree import Leaf
+try:
+    from pydoc_data import topics as pydoc_topics
+except ImportError:
+    # Python 2
+    try:
+        import pydoc_topics
+    except ImportError:
+        # This is for Python 3 embeddable version, which dont have
+        # pydoc_data module in its file python3x.zip.
+        pydoc_topics = None
+
+if is_py3:
+    if is_py35:
+        # in python 3.5 async and await are not proper keywords, but for
+        # completion pursposes should as as though they are
+        keys = keyword.kwlist + ["async", "await"]
+    else:
+        keys = keyword.kwlist
+else:
+    keys = keyword.kwlist + ['None', 'False', 'True']
+
+
+def has_inappropriate_leaf_keyword(pos, module):
+    relevant_errors = filter(
+        lambda error: error.first_pos[0] == pos[0],
+        module.error_statement_stacks)
+
+    for error in relevant_errors:
+        if error.next_token in keys:
+            return True
+
+    return False
+
+
+def completion_names(evaluator, stmt, pos, module):
+    keyword_list = all_keywords(evaluator)
+
+    if not isinstance(stmt, Leaf) or has_inappropriate_leaf_keyword(pos, module):
+        keyword_list = filter(
+            lambda keyword: not keyword.only_valid_as_leaf,
+            keyword_list
+        )
+    return [keyword.name for keyword in keyword_list]
+
+
+def all_keywords(evaluator, pos=(0, 0)):
+    return set([Keyword(evaluator, k, pos) for k in keys])
+
+
+def keyword(evaluator, string, pos=(0, 0)):
+    if string in keys:
+        return Keyword(evaluator, string, pos)
+    else:
+        return None
+
+
+def get_operator(evaluator, string, pos):
+    return Keyword(evaluator, string, pos)
+
+
+keywords_only_valid_as_leaf = (
+    'continue',
+    'break',
+)
+
+
+class Keyword(object):
+    type = 'completion_keyword'
+
+    def __init__(self, evaluator, name, pos):
+        self.name = FakeName(name, self, pos)
+        self.start_pos = pos
+        self.parent = evaluator.BUILTINS
+
+    def get_parent_until(self):
+        return self.parent
+
+    @property
+    def only_valid_as_leaf(self):
+        return self.name.value in keywords_only_valid_as_leaf
+
+    @property
+    def names(self):
+        """ For a `parsing.Name` like comparision """
+        return [self.name]
+
+    @property
+    def docstr(self):
+        return imitate_pydoc(self.name)
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, self.name)
+
+
+def imitate_pydoc(string):
+    """
+    It's not possible to get the pydoc's without starting the annoying pager
+    stuff.
+    """
+    if pydoc_topics is None:
+        return ''
+
+    # str needed because of possible unicode stuff in py2k (pydoc doesn't work
+    # with unicode strings)
+    string = str(string)
+    h = pydoc.help
+    with common.ignored(KeyError):
+        # try to access symbols
+        string = h.symbols[string]
+        string, _, related = string.partition(' ')
+
+    get_target = lambda s: h.topics.get(s, h.keywords.get(s))
+    while isinstance(string, str):
+        string = get_target(string)
+
+    try:
+        # is a tuple now
+        label, related = string
+    except TypeError:
+        return ''
+
+    try:
+        return pydoc_topics.topics[label] if pydoc_topics else ''
+    except KeyError:
+        return ''

vim-plugins/bundle/jedi-vim/jedi/jedi/api/replstartup.py

@@ -0,0 +1,27 @@
+"""
+To use Jedi completion in Python interpreter, add the following in your shell
+setup (e.g., ``.bashrc``)::
+
+    export PYTHONSTARTUP="$(python -m jedi repl)"
+
+Then you will be able to use Jedi completer in your Python interpreter::
+
+    $ python
+    Python 2.7.2+ (default, Jul 20 2012, 22:15:08)
+    [GCC 4.6.1] on linux2
+    Type "help", "copyright", "credits" or "license" for more information.
+    >>> import os
+    >>> os.path.join().split().in<TAB>                     # doctest: +SKIP
+    os.path.join().split().index   os.path.join().split().insert
+
+"""
+import jedi.utils
+from jedi import __version__ as __jedi_version__
+
+print('REPL completion using Jedi %s' % __jedi_version__)
+jedi.utils.setup_readline()
+
+del jedi
+
+# Note: try not to do many things here, as it will contaminate global
+# namespace of the interpreter.

vim-plugins/bundle/jedi-vim/jedi/jedi/api/usages.py

@@ -0,0 +1,49 @@
+from jedi._compatibility import unicode
+from jedi.api import classes
+from jedi.parser import tree
+from jedi.evaluate import imports
+
+
+def usages(evaluator, definition_names, mods):
+    """
+    :param definitions: list of Name
+    """
+    def compare_array(definitions):
+        """ `definitions` are being compared by module/start_pos, because
+        sometimes the id's of the objects change (e.g. executions).
+        """
+        result = []
+        for d in definitions:
+            module = d.get_parent_until()
+            result.append((module, d.start_pos))
+        return result
+
+    search_name = unicode(list(definition_names)[0])
+    compare_definitions = compare_array(definition_names)
+    mods |= set([d.get_parent_until() for d in definition_names])
+    definitions = []
+    for m in imports.get_modules_containing_name(evaluator, mods, search_name):
+        try:
+            check_names = m.used_names[search_name]
+        except KeyError:
+            continue
+        for name in check_names:
+
+            result = evaluator.goto(name)
+            if [c for c in compare_array(result) if c in compare_definitions]:
+                definitions.append(classes.Definition(evaluator, name))
+                # Previous definitions might be imports, so include them
+                # (because goto might return that import name).
+                compare_definitions += compare_array([name])
+    return definitions
+
+
+def usages_add_import_modules(evaluator, definitions):
+    """ Adds the modules of the imports """
+    new = set()
+    for d in definitions:
+        imp_or_stmt = d.get_definition()
+        if isinstance(imp_or_stmt, tree.Import):
+            s = imports.ImportWrapper(evaluator, d)
+            new |= set(s.follow(is_goto=True))
+    return set(definitions) | new

vim-plugins/bundle/jedi-vim/jedi/jedi/cache.py

@@ -0,0 +1,127 @@
+"""
+This caching is very important for speed and memory optimizations. There's
+nothing really spectacular, just some decorators. The following cache types are
+available:
+
+- module caching (`load_parser` and `save_parser`), which uses pickle and is
+  really important to assure low load times of modules like ``numpy``.
+- ``time_cache`` can be used to cache something for just a limited time span,
+  which can be useful if there's user interaction and the user cannot react
+  faster than a certain time.
+
+This module is one of the reasons why |jedi| is not thread-safe. As you can see
+there are global variables, which are holding the cache information. Some of
+these variables are being cleaned after every API usage.
+"""
+import time
+
+from jedi import settings
+from jedi.parser.utils import parser_cache
+from jedi.parser.utils import underscore_memoization
+
+_time_caches = {}
+
+
+def clear_time_caches(delete_all=False):
+    """ Jedi caches many things, that should be completed after each completion
+    finishes.
+
+    :param delete_all: Deletes also the cache that is normally not deleted,
+        like parser cache, which is important for faster parsing.
+    """
+    global _time_caches
+
+    if delete_all:
+        for cache in _time_caches.values():
+            cache.clear()
+        parser_cache.clear()
+    else:
+        # normally just kill the expired entries, not all
+        for tc in _time_caches.values():
+            # check time_cache for expired entries
+            for key, (t, value) in list(tc.items()):
+                if t < time.time():
+                    # delete expired entries
+                    del tc[key]
+
+
+def time_cache(time_add_setting):
+    """
+    This decorator works as follows: Call it with a setting and after that
+    use the function with a callable that returns the key.
+    But: This function is only called if the key is not available. After a
+    certain amount of time (`time_add_setting`) the cache is invalid.
+
+    If the given key is None, the function will not be cached.
+    """
+    def _temp(key_func):
+        dct = {}
+        _time_caches[time_add_setting] = dct
+
+        def wrapper(*args, **kwargs):
+            generator = key_func(*args, **kwargs)
+            key = next(generator)
+            try:
+                expiry, value = dct[key]
+                if expiry > time.time():
+                    return value
+            except KeyError:
+                pass
+
+            value = next(generator)
+            time_add = getattr(settings, time_add_setting)
+            if key is not None:
+                dct[key] = time.time() + time_add, value
+            return value
+        return wrapper
+    return _temp
+
+
+def memoize_method(method):
+    """A normal memoize function."""
+    def wrapper(self, *args, **kwargs):
+        dct = self.__dict__.setdefault('_memoize_method_dct', {})
+        key = (args, frozenset(kwargs.items()))
+        try:
+            return dct[key]
+        except KeyError:
+            result = method(self, *args, **kwargs)
+            dct[key] = result
+            return result
+    return wrapper
+
+
+def cache_star_import(func):
+    @time_cache("star_import_cache_validity")
+    def wrapper(self):
+        yield self.base  # The cache key
+        yield func(self)
+    return wrapper
+
+
+def _invalidate_star_import_cache_module(module, only_main=False):
+    """ Important if some new modules are being reparsed """
+    try:
+        t, modules = _time_caches['star_import_cache_validity'][module]
+    except KeyError:
+        pass
+    else:
+        del _time_caches['star_import_cache_validity'][module]
+
+        # This stuff was part of load_parser. However since we're most likely
+        # not going to use star import caching anymore, just ignore it.
+        #else:
+            # In case there is already a module cached and this module
+            # has to be reparsed, we also need to invalidate the import
+            # caches.
+        #    _invalidate_star_import_cache_module(parser_cache_item.parser.module)
+
+
+def invalidate_star_import_cache(path):
+    """On success returns True."""
+    try:
+        parser_cache_item = parser_cache[path]
+    except KeyError:
+        pass
+    else:
+        _invalidate_star_import_cache_module(parser_cache_item.parser.module)

vim-plugins/bundle/jedi-vim/jedi/jedi/common.py

@@ -0,0 +1,169 @@
+""" A universal module with functions / classes without dependencies. """
+import sys
+import contextlib
+import functools
+import re
+from itertools import chain
+from ast import literal_eval
+
+from jedi._compatibility import unicode, reraise
+from jedi import settings
+
+
+class UncaughtAttributeError(Exception):
+    """
+    Important, because `__getattr__` and `hasattr` catch AttributeErrors
+    implicitly. This is really evil (mainly because of `__getattr__`).
+    `hasattr` in Python 2 is even more evil, because it catches ALL exceptions.
+    Therefore this class originally had to be derived from `BaseException`
+    instead of `Exception`.  But because I removed relevant `hasattr` from
+    the code base, we can now switch back to `Exception`.
+
+    :param base: return values of sys.exc_info().
+    """
+
+
+def safe_property(func):
+    return property(reraise_uncaught(func))
+
+
+def reraise_uncaught(func):
+    """
+    Re-throw uncaught `AttributeError`.
+
+    Usage:  Put ``@rethrow_uncaught`` in front of the function
+    which does **not** suppose to raise `AttributeError`.
+
+    AttributeError is easily get caught by `hasattr` and another
+    ``except AttributeError`` clause.  This becomes problem when you use
+    a lot of "dynamic" attributes (e.g., using ``@property``) because you
+    can't distinguish if the property does not exist for real or some code
+    inside of the "dynamic" attribute through that error.  In a well
+    written code, such error should not exist but getting there is very
+    difficult.  This decorator is to help us getting there by changing
+    `AttributeError` to `UncaughtAttributeError` to avoid unexpected catch.
+    This helps us noticing bugs earlier and facilitates debugging.
+
+    .. note:: Treating StopIteration here is easy.
+              Add that feature when needed.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwds):
+        try:
+            return func(*args, **kwds)
+        except AttributeError:
+            exc_info = sys.exc_info()
+            reraise(UncaughtAttributeError(exc_info[1]), exc_info[2])
+    return wrapper
+
+
+class PushBackIterator(object):
+    def __init__(self, iterator):
+        self.pushes = []
+        self.iterator = iterator
+        self.current = None
+
+    def push_back(self, value):
+        self.pushes.append(value)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        """ Python 2 Compatibility """
+        return self.__next__()
+
+    def __next__(self):
+        if self.pushes:
+            self.current = self.pushes.pop()
+        else:
+            self.current = next(self.iterator)
+        return self.current
+
+
+@contextlib.contextmanager
+def scale_speed_settings(factor):
+    a = settings.max_executions
+    b = settings.max_until_execution_unique
+    settings.max_executions *= factor
+    settings.max_until_execution_unique *= factor
+    try:
+        yield
+    finally:
+        settings.max_executions = a
+        settings.max_until_execution_unique = b
+
+
+def indent_block(text, indention='    '):
+    """This function indents a text block with a default of four spaces."""
+    temp = ''
+    while text and text[-1] == '\n':
+        temp += text[-1]
+        text = text[:-1]
+    lines = text.split('\n')
+    return '\n'.join(map(lambda s: indention + s, lines)) + temp
+
+
+@contextlib.contextmanager
+def ignored(*exceptions):
+    """
+    Context manager that ignores all of the specified exceptions. This will
+    be in the standard library starting with Python 3.4.
+    """
+    try:
+        yield
+    except exceptions:
+        pass
+
+
+def source_to_unicode(source, encoding=None):
+    def detect_encoding():
+        """
+        For the implementation of encoding definitions in Python, look at:
+        - http://www.python.org/dev/peps/pep-0263/
+        - http://docs.python.org/2/reference/lexical_analysis.html#encoding-declarations
+        """
+        byte_mark = literal_eval(r"b'\xef\xbb\xbf'")
+        if source.startswith(byte_mark):
+            # UTF-8 byte-order mark
+            return 'utf-8'
+
+        first_two_lines = re.match(br'(?:[^\n]*\n){0,2}', source).group(0)
+        possible_encoding = re.search(br"coding[=:]\s*([-\w.]+)",
+                                      first_two_lines)
+        if possible_encoding:
+            return possible_encoding.group(1)
+        else:
+            # the default if nothing else has been set -> PEP 263
+            return encoding if encoding is not None else 'utf-8'
+
+    if isinstance(source, unicode):
+        # only cast str/bytes
+        return source
+
+    encoding = detect_encoding()
+    if not isinstance(encoding, unicode):
+        encoding = unicode(encoding, 'utf-8', 'replace')
+    # cast to unicode by default
+    return unicode(source, encoding, 'replace')
+
+
+def splitlines(string):
+    """
+    A splitlines for Python code. In contrast to Python's ``str.splitlines``,
+    looks at form feeds and other special characters as normal text. Just
+    splits ``\n`` and ``\r\n``.
+    Also different: Returns ``['']`` for an empty string input.
+    """
+    return re.split('\n|\r\n', string)
+
+
+def unite(iterable):
+    """Turns a two dimensional array into a one dimensional."""
+    return set(chain.from_iterable(iterable))
+
+
+def to_list(func):
+    def wrapper(*args, **kwargs):
+        return list(func(*args, **kwargs))
+    return wrapper

vim-plugins/bundle/jedi-vim/jedi/jedi/debug.py

@@ -0,0 +1,133 @@
+from jedi._compatibility import encoding, is_py3, u
+import inspect
+import os
+import time
+
+def _lazy_colorama_init():
+    """
+    Lazily init colorama if necessary, not to screw up stdout is debug not
+    enabled.
+
+    This version of the function does nothing.
+    """
+    pass
+
+_inited=False
+
+try:
+    if os.name == 'nt':
+        # Does not work on Windows, as pyreadline and colorama interfere
+        raise ImportError
+    else:
+        # Use colorama for nicer console output.
+        from colorama import Fore, init
+        from colorama import initialise
+        def _lazy_colorama_init():
+            """
+            Lazily init colorama if necessary, not to screw up stdout is
+            debug not enabled.
+
+            This version of the function does init colorama.
+            """
+            global _inited
+            if not _inited:
+                # pytest resets the stream at the end - causes troubles. Since
+                # after every output the stream is reset automatically we don't
+                # need this.
+                initialise.atexit_done = True
+                try:
+                    init()
+                except Exception:
+                    # Colorama fails with initializing under vim and is buggy in
+                    # version 0.3.6.
+                    pass
+            _inited = True
+
+except ImportError:
+    class Fore(object):
+        RED = ''
+        GREEN = ''
+        YELLOW = ''
+        MAGENTA = ''
+        RESET = ''
+
+NOTICE = object()
+WARNING = object()
+SPEED = object()
+
+enable_speed = False
+enable_warning = False
+enable_notice = False
+
+# callback, interface: level, str
+debug_function = None
+ignored_modules = ['jedi.parser']
+_debug_indent = 0
+_start_time = time.time()
+
+
+def reset_time():
+    global _start_time, _debug_indent
+    _start_time = time.time()
+    _debug_indent = 0
+
+
+def increase_indent(func):
+    """Decorator for makin """
+    def wrapper(*args, **kwargs):
+        global _debug_indent
+        _debug_indent += 1
+        try:
+            return func(*args, **kwargs)
+        finally:
+            _debug_indent -= 1
+    return wrapper
+
+
+def dbg(message, *args, **kwargs):
+    """ Looks at the stack, to see if a debug message should be printed. """
+    # Python 2 compatibility, because it doesn't understand default args
+    color = kwargs.pop('color', 'GREEN')
+    assert color
+
+    if debug_function and enable_notice:
+        frm = inspect.stack()[1]
+        mod = inspect.getmodule(frm[0])
+        if not (mod.__name__ in ignored_modules):
+            i = ' ' * _debug_indent
+            _lazy_colorama_init()
+            debug_function(color, i + 'dbg: ' + message % tuple(u(repr(a)) for a in args))
+
+
+def warning(message, *args, **kwargs):
+    format = kwargs.pop('format', True)
+    assert not kwargs
+
+    if debug_function and enable_warning:
+        i = ' ' * _debug_indent
+        if format:
+            message = message % tuple(u(repr(a)) for a in args)
+        debug_function('RED', i + 'warning: ' + message)
+
+
+def speed(name):
+    if debug_function and enable_speed:
+        now = time.time()
+        i = ' ' * _debug_indent
+        debug_function('YELLOW', i + 'speed: ' + '%s %s' % (name, now - _start_time))
+
+
+def print_to_stdout(color, str_out):
+    """
+    The default debug function that prints to standard out.
+
+    :param str color: A string that is an attribute of ``colorama.Fore``.
+    """
+    col = getattr(Fore, color)
+    _lazy_colorama_init()
+    if not is_py3:
+        str_out = str_out.encode(encoding, 'replace')
+    print(col + str_out + Fore.RESET)
+
+
+# debug_function = print_to_stdout

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/__init__.py

@@ -0,0 +1,534 @@
+"""
+Evaluation of Python code in |jedi| is based on three assumptions:
+
+* The code uses as least side effects as possible. Jedi understands certain
+  list/tuple/set modifications, but there's no guarantee that Jedi detects
+  everything (list.append in different modules for example).
+* No magic is being used:
+
+  - metaclasses
+  - ``setattr()`` / ``__import__()``
+  - writing to ``globals()``, ``locals()``, ``object.__dict__``
+* The programmer is not a total dick, e.g. like `this
+  <https://github.com/davidhalter/jedi/issues/24>`_ :-)
+
+The actual algorithm is based on a principle called lazy evaluation. If you
+don't know about it, google it.  That said, the typical entry point for static
+analysis is calling ``eval_statement``. There's separate logic for
+autocompletion in the API, the evaluator is all about evaluating an expression.
+
+Now you need to understand what follows after ``eval_statement``. Let's
+make an example::
+
+    import datetime
+    datetime.date.toda# <-- cursor here
+
+First of all, this module doesn't care about completion. It really just cares
+about ``datetime.date``. At the end of the procedure ``eval_statement`` will
+return the ``date`` class.
+
+To *visualize* this (simplified):
+
+- ``Evaluator.eval_statement`` doesn't do much, because there's no assignment.
+- ``Evaluator.eval_element`` cares for resolving the dotted path
+- ``Evaluator.find_types`` searches for global definitions of datetime, which
+  it finds in the definition of an import, by scanning the syntax tree.
+- Using the import logic, the datetime module is found.
+- Now ``find_types`` is called again by ``eval_element`` to find ``date``
+  inside the datetime module.
+
+Now what would happen if we wanted ``datetime.date.foo.bar``? Two more
+calls to ``find_types``. However the second call would be ignored, because the
+first one would return nothing (there's no foo attribute in ``date``).
+
+What if the import would contain another ``ExprStmt`` like this::
+
+    from foo import bar
+    Date = bar.baz
+
+Well... You get it. Just another ``eval_statement`` recursion. It's really
+easy. Python can obviously get way more complicated then this. To understand
+tuple assignments, list comprehensions and everything else, a lot more code had
+to be written.
+
+Jedi has been tested very well, so you can just start modifying code. It's best
+to write your own test first for your "new" feature. Don't be scared of
+breaking stuff. As long as the tests pass, you're most likely to be fine.
+
+I need to mention now that lazy evaluation is really good because it
+only *evaluates* what needs to be *evaluated*. All the statements and modules
+that are not used are just being ignored.
+"""
+
+import copy
+import sys
+from itertools import chain
+
+from jedi.parser import tree
+from jedi import debug
+from jedi.evaluate import representation as er
+from jedi.evaluate import imports
+from jedi.evaluate import recursion
+from jedi.evaluate import iterable
+from jedi.evaluate.cache import memoize_default
+from jedi.evaluate import stdlib
+from jedi.evaluate import finder
+from jedi.evaluate import compiled
+from jedi.evaluate import precedence
+from jedi.evaluate import param
+from jedi.evaluate import helpers
+
+
+class Evaluator(object):
+    def __init__(self, grammar, sys_path=None):
+        self.grammar = grammar
+        self.memoize_cache = {}  # for memoize decorators
+        # To memorize modules -> equals `sys.modules`.
+        self.modules = {}  # like `sys.modules`.
+        self.compiled_cache = {}  # see `evaluate.compiled.create()`
+        self.mixed_cache = {}  # see `evaluate.compiled.mixed.create()`
+        self.analysis = []
+        self.predefined_if_name_dict_dict = {}
+        self.dynamic_params_depth = 0
+        self.is_analysis = False
+
+        if sys_path is None:
+            sys_path = sys.path
+        self.sys_path = copy.copy(sys_path)
+        try:
+            self.sys_path.remove('')
+        except ValueError:
+            pass
+
+        self.reset_recursion_limitations()
+
+        # Constants
+        self.BUILTINS = compiled.get_special_object(self, 'BUILTINS')
+
+    def reset_recursion_limitations(self):
+        self.recursion_detector = recursion.RecursionDetector(self)
+        self.execution_recursion_detector = recursion.ExecutionRecursionDetector(self)
+
+    def wrap(self, element):
+        if isinstance(element, (er.Wrapper, er.InstanceElement,
+            er.ModuleWrapper, er.FunctionExecution, er.Instance, compiled.CompiledObject)) or element is None:
+            # TODO this is so ugly, please refactor.
+            return element
+
+        if element.type == 'classdef':
+            return er.Class(self, element)
+        elif element.type == 'funcdef':
+            return er.Function(self, element)
+        elif element.type == 'lambda':
+            return er.LambdaWrapper(self, element)
+        elif element.type == 'file_input':
+            return er.ModuleWrapper(self, element)
+        else:
+            return element
+
+    def find_types(self, scope, name_str, position=None, search_global=False,
+                   is_goto=False):
+        """
+        This is the search function. The most important part to debug.
+        `remove_statements` and `filter_statements` really are the core part of
+        this completion.
+
+        :param position: Position of the last statement -> tuple of line, column
+        :return: List of Names. Their parents are the types.
+        """
+        f = finder.NameFinder(self, scope, name_str, position)
+        scopes = f.scopes(search_global)
+        if is_goto:
+            return f.filter_name(scopes)
+        return f.find(scopes, attribute_lookup=not search_global)
+
+    #@memoize_default(default=[], evaluator_is_first_arg=True)
+    #@recursion.recursion_decorator
+    @debug.increase_indent
+    def eval_statement(self, stmt, seek_name=None):
+        """
+        The starting point of the completion. A statement always owns a call
+        list, which are the calls, that a statement does. In case multiple
+        names are defined in the statement, `seek_name` returns the result for
+        this name.
+
+        :param stmt: A `tree.ExprStmt`.
+        """
+        debug.dbg('eval_statement %s (%s)', stmt, seek_name)
+        rhs = stmt.get_rhs()
+        types = self.eval_element(rhs)
+
+        if seek_name:
+            types = finder.check_tuple_assignments(self, types, seek_name)
+
+        first_operation = stmt.first_operation()
+        if first_operation not in ('=', None) and not isinstance(stmt, er.InstanceElement):  # TODO don't check for this.
+            # `=` is always the last character in aug assignments -> -1
+            operator = copy.copy(first_operation)
+            operator.value = operator.value[:-1]
+            name = str(stmt.get_defined_names()[0])
+            parent = self.wrap(stmt.get_parent_scope())
+            left = self.find_types(parent, name, stmt.start_pos, search_global=True)
+
+            for_stmt = stmt.get_parent_until(tree.ForStmt)
+            if isinstance(for_stmt, tree.ForStmt) and types \
+                    and for_stmt.defines_one_name():
+                # Iterate through result and add the values, that's possible
+                # only in for loops without clutter, because they are
+                # predictable. Also only do it, if the variable is not a tuple.
+                node = for_stmt.get_input_node()
+                for_iterables = self.eval_element(node)
+                ordered = list(iterable.py__iter__(self, for_iterables, node))
+
+                for index_types in ordered:
+                    dct = {str(for_stmt.children[1]): index_types}
+                    self.predefined_if_name_dict_dict[for_stmt] = dct
+                    t = self.eval_element(rhs)
+                    left = precedence.calculate(self, left, operator, t)
+                types = left
+                if ordered:
+                    # If there are no for entries, we cannot iterate and the
+                    # types are defined by += entries. Therefore the for loop
+                    # is never called.
+                    del self.predefined_if_name_dict_dict[for_stmt]
+            else:
+                types = precedence.calculate(self, left, operator, types)
+        debug.dbg('eval_statement result %s', types)
+        return types
+
+    def eval_element(self, element):
+        if isinstance(element, iterable.AlreadyEvaluated):
+            return set(element)
+        elif isinstance(element, iterable.MergedNodes):
+            return iterable.unite(self.eval_element(e) for e in element)
+
+        if_stmt = element.get_parent_until((tree.IfStmt, tree.ForStmt, tree.IsScope))
+        predefined_if_name_dict = self.predefined_if_name_dict_dict.get(if_stmt)
+        if predefined_if_name_dict is None and isinstance(if_stmt, tree.IfStmt):
+            if_stmt_test = if_stmt.children[1]
+            name_dicts = [{}]
+            # If we already did a check, we don't want to do it again -> If
+            # predefined_if_name_dict_dict is filled, we stop.
+            # We don't want to check the if stmt itself, it's just about
+            # the content.
+            if element.start_pos > if_stmt_test.end_pos:
+                # Now we need to check if the names in the if_stmt match the
+                # names in the suite.
+                if_names = helpers.get_names_of_node(if_stmt_test)
+                element_names = helpers.get_names_of_node(element)
+                str_element_names = [str(e) for e in element_names]
+                if any(str(i) in str_element_names for i in if_names):
+                    for if_name in if_names:
+                        definitions = self.goto_definitions(if_name)
+                        # Every name that has multiple different definitions
+                        # causes the complexity to rise. The complexity should
+                        # never fall below 1.
+                        if len(definitions) > 1:
+                            if len(name_dicts) * len(definitions) > 16:
+                                debug.dbg('Too many options for if branch evaluation %s.', if_stmt)
+                                # There's only a certain amount of branches
+                                # Jedi can evaluate, otherwise it will take to
+                                # long.
+                                name_dicts = [{}]
+                                break
+
+                            original_name_dicts = list(name_dicts)
+                            name_dicts = []
+                            for definition in definitions:
+                                new_name_dicts = list(original_name_dicts)
+                                for i, name_dict in enumerate(new_name_dicts):
+                                    new_name_dicts[i] = name_dict.copy()
+                                    new_name_dicts[i][str(if_name)] = [definition]
+
+                                name_dicts += new_name_dicts
+                        else:
+                            for name_dict in name_dicts:
+                                name_dict[str(if_name)] = definitions
+            if len(name_dicts) > 1:
+                result = set()
+                for name_dict in name_dicts:
+                    self.predefined_if_name_dict_dict[if_stmt] = name_dict
+                    try:
+                        result |= self._eval_element_not_cached(element)
+                    finally:
+                        del self.predefined_if_name_dict_dict[if_stmt]
+                return result
+            else:
+                return self._eval_element_if_evaluated(element)
+                return self._eval_element_cached(element)
+        else:
+            if predefined_if_name_dict:
+                return self._eval_element_not_cached(element)
+            else:
+                return self._eval_element_if_evaluated(element)
+                return self._eval_element_cached(element)
+
+    def _eval_element_if_evaluated(self, element):
+        """
+        TODO This function is temporary: Merge with eval_element.
+        """
+        parent = element
+        while parent is not None:
+            parent = parent.parent
+            predefined_if_name_dict = self.predefined_if_name_dict_dict.get(parent)
+            if predefined_if_name_dict is not None:
+                return self._eval_element_not_cached(element)
+        return self._eval_element_cached(element)
+
+    @memoize_default(default=set(), evaluator_is_first_arg=True)
+    def _eval_element_cached(self, element):
+        return self._eval_element_not_cached(element)
+
+    @debug.increase_indent
+    def _eval_element_not_cached(self, element):
+        debug.dbg('eval_element %s@%s', element, element.start_pos)
+        types = set()
+        if isinstance(element, (tree.Name, tree.Literal)) or tree.is_node(element, 'atom'):
+            types = self._eval_atom(element)
+        elif isinstance(element, tree.Keyword):
+            # For False/True/None
+            if element.value in ('False', 'True', 'None'):
+                types.add(compiled.builtin_from_name(self, element.value))
+            # else: print e.g. could be evaluated like this in Python 2.7
+        elif element.isinstance(tree.Lambda):
+            types = set([er.LambdaWrapper(self, element)])
+        elif element.isinstance(er.LambdaWrapper):
+            types = set([element])  # TODO this is no real evaluation.
+        elif element.type == 'expr_stmt':
+            types = self.eval_statement(element)
+        elif element.type in ('power', 'atom_expr'):
+            types = self._eval_atom(element.children[0])
+            for trailer in element.children[1:]:
+                if trailer == '**':  # has a power operation.
+                    right = self.eval_element(element.children[2])
+                    types = set(precedence.calculate(self, types, trailer, right))
+                    break
+                types = self.eval_trailer(types, trailer)
+        elif element.type in ('testlist_star_expr', 'testlist',):
+            # The implicit tuple in statements.
+            types = set([iterable.ImplicitTuple(self, element)])
+        elif element.type in ('not_test', 'factor'):
+            types = self.eval_element(element.children[-1])
+            for operator in element.children[:-1]:
+                types = set(precedence.factor_calculate(self, types, operator))
+        elif element.type == 'test':
+            # `x if foo else y` case.
+            types = (self.eval_element(element.children[0]) |
+                     self.eval_element(element.children[-1]))
+        elif element.type == 'operator':
+            # Must be an ellipsis, other operators are not evaluated.
+            assert element.value == '...'
+            types = set([compiled.create(self, Ellipsis)])
+        elif element.type == 'dotted_name':
+            types = self._eval_atom(element.children[0])
+            for next_name in element.children[2::2]:
+                types = set(chain.from_iterable(self.find_types(typ, next_name)
+                                                for typ in types))
+            types = types
+        elif element.type == 'eval_input':
+            types = self._eval_element_not_cached(element.children[0])
+        else:
+            types = precedence.calculate_children(self, element.children)
+        debug.dbg('eval_element result %s', types)
+        return types
+
+    def _eval_atom(self, atom):
+        """
+        Basically to process ``atom`` nodes. The parser sometimes doesn't
+        generate the node (because it has just one child). In that case an atom
+        might be a name or a literal as well.
+        """
+        if isinstance(atom, tree.Name):
+            # This is the first global lookup.
+            stmt = atom.get_definition()
+            scope = stmt.get_parent_until(tree.IsScope, include_current=True)
+            if isinstance(scope, (tree.Function, er.FunctionExecution)):
+                # Adjust scope: If the name is not in the suite, it's a param
+                # default or annotation and will be resolved as part of the
+                # parent scope.
+                colon = scope.children.index(':')
+                if atom.start_pos < scope.children[colon + 1].start_pos:
+                    scope = scope.get_parent_scope()
+            if isinstance(stmt, tree.CompFor):
+                stmt = stmt.get_parent_until((tree.ClassOrFunc, tree.ExprStmt))
+            if stmt.type != 'expr_stmt':
+                # We only need to adjust the start_pos for statements, because
+                # there the name cannot be used.
+                stmt = atom
+            return self.find_types(scope, atom, stmt.start_pos, search_global=True)
+        elif isinstance(atom, tree.Literal):
+            return set([compiled.create(self, atom.eval())])
+        else:
+            c = atom.children
+            if c[0].type == 'string':
+                # Will be one string.
+                types = self._eval_atom(c[0])
+                for string in c[1:]:
+                    right = self._eval_atom(string)
+                    types = precedence.calculate(self, types, '+', right)
+                return types
+            # Parentheses without commas are not tuples.
+            elif c[0] == '(' and not len(c) == 2 \
+                    and not(tree.is_node(c[1], 'testlist_comp')
+                            and len(c[1].children) > 1):
+                return self.eval_element(c[1])
+
+            try:
+                comp_for = c[1].children[1]
+            except (IndexError, AttributeError):
+                pass
+            else:
+                if comp_for == ':':
+                    # Dict comprehensions have a colon at the 3rd index.
+                    try:
+                        comp_for = c[1].children[3]
+                    except IndexError:
+                        pass
+
+                if comp_for.type == 'comp_for':
+                    return set([iterable.Comprehension.from_atom(self, atom)])
+            return set([iterable.Array(self, atom)])
+
+    def eval_trailer(self, types, trailer):
+        trailer_op, node = trailer.children[:2]
+        if node == ')':  # `arglist` is optional.
+            node = ()
+
+        new_types = set()
+        if trailer_op == '[':
+            new_types |= iterable.py__getitem__(self, types, trailer)
+        else:
+            for typ in types:
+                debug.dbg('eval_trailer: %s in scope %s', trailer, typ)
+                if trailer_op == '.':
+                    new_types |= self.find_types(typ, node)
+                elif trailer_op == '(':
+                    new_types |= self.execute(typ, node, trailer)
+        return new_types
+
+    def execute_evaluated(self, obj, *args):
+        """
+        Execute a function with already executed arguments.
+        """
+        args = [iterable.AlreadyEvaluated([arg]) for arg in args]
+        return self.execute(obj, args)
+
+    @debug.increase_indent
+    def execute(self, obj, arguments=(), trailer=None):
+        if not isinstance(arguments, param.Arguments):
+            arguments = param.Arguments(self, arguments, trailer)
+
+        if self.is_analysis:
+            arguments.eval_all()
+
+        if obj.isinstance(er.Function):
+            obj = obj.get_decorated_func()
+
+        debug.dbg('execute: %s %s', obj, arguments)
+        try:
+            # Some stdlib functions like super(), namedtuple(), etc. have been
+            # hard-coded in Jedi to support them.
+            return stdlib.execute(self, obj, arguments)
+        except stdlib.NotInStdLib:
+            pass
+
+        try:
+            func = obj.py__call__
+        except AttributeError:
+            debug.warning("no execution possible %s", obj)
+            return set()
+        else:
+            types = func(arguments)
+            debug.dbg('execute result: %s in %s', types, obj)
+            return types
+
+    def goto_definitions(self, name):
+        def_ = name.get_definition()
+        is_simple_name = name.parent.type not in ('power', 'trailer')
+        if is_simple_name:
+            if name.parent.type == 'classdef' and name.parent.name == name:
+                return [self.wrap(name.parent)]
+            if name.parent.type in ('file_input', 'funcdef'):
+                return [self.wrap(name.parent)]
+            if def_.type == 'expr_stmt' and name in def_.get_defined_names():
+                return self.eval_statement(def_, name)
+            elif def_.type == 'for_stmt':
+                container_types = self.eval_element(def_.children[3])
+                for_types = iterable.py__iter__types(self, container_types, def_.children[3])
+                return finder.check_tuple_assignments(self, for_types, name)
+            elif def_.type in ('import_from', 'import_name'):
+                return imports.ImportWrapper(self, name).follow()
+
+        call = helpers.call_of_leaf(name)
+        return self.eval_element(call)
+
+    def goto(self, name):
+        def resolve_implicit_imports(names):
+            for name in names:
+                if isinstance(name.parent, helpers.FakeImport):
+                    # Those are implicit imports.
+                    s = imports.ImportWrapper(self, name)
+                    for n in s.follow(is_goto=True):
+                        yield n
+                else:
+                    yield name
+
+        stmt = name.get_definition()
+        par = name.parent
+        if par.type == 'argument' and par.children[1] == '=' and par.children[0] == name:
+            # Named param goto.
+            trailer = par.parent
+            if trailer.type == 'arglist':
+                trailer = trailer.parent
+            if trailer.type != 'classdef':
+                if trailer.type == 'decorator':
+                    types = self.eval_element(trailer.children[1])
+                else:
+                    i = trailer.parent.children.index(trailer)
+                    to_evaluate = trailer.parent.children[:i]
+                    types = self.eval_element(to_evaluate[0])
+                    for trailer in to_evaluate[1:]:
+                        types = self.eval_trailer(types, trailer)
+                param_names = []
+                for typ in types:
+                    try:
+                        params = typ.params
+                    except AttributeError:
+                        pass
+                    else:
+                        param_names += [param.name for param in params
+                                        if param.name.value == name.value]
+                return param_names
+        elif isinstance(par, tree.ExprStmt) and name in par.get_defined_names():
+            # Only take the parent, because if it's more complicated than just
+            # a name it's something you can "goto" again.
+            return [name]
+        elif isinstance(par, (tree.Param, tree.Function, tree.Class)) and par.name is name:
+            return [name]
+        elif isinstance(stmt, tree.Import):
+            modules = imports.ImportWrapper(self, name).follow(is_goto=True)
+            return list(resolve_implicit_imports(modules))
+        elif par.type == 'dotted_name':  # Is a decorator.
+            index = par.children.index(name)
+            if index > 0:
+                new_dotted = helpers.deep_ast_copy(par)
+                new_dotted.children[index - 1:] = []
+                types = self.eval_element(new_dotted)
+                return resolve_implicit_imports(iterable.unite(
+                    self.find_types(typ, name, is_goto=True) for typ in types
+                ))
+
+        scope = name.get_parent_scope()
+        if tree.is_node(par, 'trailer') and par.children[0] == '.':
+            call = helpers.call_of_leaf(name, cut_own_trailer=True)
+            types = self.eval_element(call)
+            return resolve_implicit_imports(iterable.unite(
+                self.find_types(typ, name, is_goto=True) for typ in types
+            ))
+        else:
+            if stmt.type != 'expr_stmt':
+                # We only need to adjust the start_pos for statements, because
+                # there the name cannot be used.
+                stmt = name
+            return self.find_types(scope, name, stmt.start_pos,
+                                   search_global=True, is_goto=True)

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/analysis.py

@@ -0,0 +1,216 @@
+"""
+Module for statical analysis.
+"""
+from jedi import debug
+from jedi.parser import tree
+from jedi.evaluate.compiled import CompiledObject
+
+from jedi.common import unite
+
+
+CODES = {
+    'attribute-error': (1, AttributeError, 'Potential AttributeError.'),
+    'name-error': (2, NameError, 'Potential NameError.'),
+    'import-error': (3, ImportError, 'Potential ImportError.'),
+    'type-error-too-many-arguments': (4, TypeError, None),
+    'type-error-too-few-arguments': (5, TypeError, None),
+    'type-error-keyword-argument': (6, TypeError, None),
+    'type-error-multiple-values': (7, TypeError, None),
+    'type-error-star-star': (8, TypeError, None),
+    'type-error-star': (9, TypeError, None),
+    'type-error-operation': (10, TypeError, None),
+    'type-error-not-iterable': (11, TypeError, None),
+    'type-error-isinstance': (12, TypeError, None),
+    'type-error-not-subscriptable': (13, TypeError, None),
+    'value-error-too-many-values': (14, ValueError, None),
+    'value-error-too-few-values': (15, ValueError, None),
+}
+
+
+class Error(object):
+    def __init__(self, name, module_path, start_pos, message=None):
+        self.path = module_path
+        self._start_pos = start_pos
+        self.name = name
+        if message is None:
+            message = CODES[self.name][2]
+        self.message = message
+
+    @property
+    def line(self):
+        return self._start_pos[0]
+
+    @property
+    def column(self):
+        return self._start_pos[1]
+
+    @property
+    def code(self):
+        # The class name start
+        first = self.__class__.__name__[0]
+        return first + str(CODES[self.name][0])
+
+    def __unicode__(self):
+        return '%s:%s:%s: %s %s' % (self.path, self.line, self.column,
+                                    self.code, self.message)
+
+    def __str__(self):
+        return self.__unicode__()
+
+    def __eq__(self, other):
+        return (self.path == other.path and self.name == other.name
+                and self._start_pos == other._start_pos)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return hash((self.path, self._start_pos, self.name))
+
+    def __repr__(self):
+        return '<%s %s: %s@%s,%s>' % (self.__class__.__name__,
+                                      self.name, self.path,
+                                      self._start_pos[0], self._start_pos[1])
+
+
+class Warning(Error):
+    pass
+
+
+def add(evaluator, name, jedi_obj, message=None, typ=Error, payload=None):
+    from jedi.evaluate.iterable import MergedNodes
+    while isinstance(jedi_obj, MergedNodes):
+        if len(jedi_obj) != 1:
+            # TODO is this kosher?
+            return
+        jedi_obj = list(jedi_obj)[0]
+
+    exception = CODES[name][1]
+    if _check_for_exception_catch(evaluator, jedi_obj, exception, payload):
+        return
+
+    module_path = jedi_obj.get_parent_until().path
+    instance = typ(name, module_path, jedi_obj.start_pos, message)
+    debug.warning(str(instance), format=False)
+    evaluator.analysis.append(instance)
+
+
+def _check_for_setattr(instance):
+    """
+    Check if there's any setattr method inside an instance. If so, return True.
+    """
+    module = instance.get_parent_until()
+    try:
+        stmts = module.used_names['setattr']
+    except KeyError:
+        return False
+
+    return any(instance.start_pos < stmt.start_pos < instance.end_pos
+               for stmt in stmts)
+
+
+def add_attribute_error(evaluator, scope, name):
+    message = ('AttributeError: %s has no attribute %s.' % (scope, name))
+    from jedi.evaluate.representation import Instance
+    # Check for __getattr__/__getattribute__ existance and issue a warning
+    # instead of an error, if that happens.
+    if isinstance(scope, Instance):
+        typ = Warning
+        try:
+            scope.get_subscope_by_name('__getattr__')
+        except KeyError:
+            try:
+                scope.get_subscope_by_name('__getattribute__')
+            except KeyError:
+                if not _check_for_setattr(scope):
+                    typ = Error
+    else:
+        typ = Error
+
+    payload = scope, name
+    add(evaluator, 'attribute-error', name, message, typ, payload)
+
+
+def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None):
+    """
+    Checks if a jedi object (e.g. `Statement`) sits inside a try/catch and
+    doesn't count as an error (if equal to `exception`).
+    Also checks `hasattr` for AttributeErrors and uses the `payload` to compare
+    it.
+    Returns True if the exception was catched.
+    """
+    def check_match(cls, exception):
+        try:
+            return isinstance(cls, CompiledObject) and issubclass(exception, cls.obj)
+        except TypeError:
+            return False
+
+    def check_try_for_except(obj, exception):
+        # Only nodes in try
+        iterator = iter(obj.children)
+        for branch_type in iterator:
+            colon = next(iterator)
+            suite = next(iterator)
+            if branch_type == 'try' \
+                    and not (branch_type.start_pos < jedi_obj.start_pos <= suite.end_pos):
+                return False
+
+        for node in obj.except_clauses():
+            if node is None:
+                return True  # An exception block that catches everything.
+            else:
+                except_classes = evaluator.eval_element(node)
+                for cls in except_classes:
+                    from jedi.evaluate import iterable
+                    if isinstance(cls, iterable.Array) and cls.type == 'tuple':
+                        # multiple exceptions
+                        for typ in unite(cls.py__iter__()):
+                            if check_match(typ, exception):
+                                return True
+                    else:
+                        if check_match(cls, exception):
+                            return True
+
+    def check_hasattr(node, suite):
+        try:
+            assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos
+            assert node.type in ('power', 'atom_expr')
+            base = node.children[0]
+            assert base.type == 'name' and base.value == 'hasattr'
+            trailer = node.children[1]
+            assert trailer.type == 'trailer'
+            arglist = trailer.children[1]
+            assert arglist.type == 'arglist'
+            from jedi.evaluate.param import Arguments
+            args = list(Arguments(evaluator, arglist).unpack())
+            # Arguments should be very simple
+            assert len(args) == 2
+
+            # Check name
+            key, values = args[1]
+            assert len(values) == 1
+            names = list(evaluator.eval_element(values[0]))
+            assert len(names) == 1 and isinstance(names[0], CompiledObject)
+            assert names[0].obj == str(payload[1])
+
+            # Check objects
+            key, values = args[0]
+            assert len(values) == 1
+            objects = evaluator.eval_element(values[0])
+            return payload[0] in objects
+        except AssertionError:
+            return False
+
+    obj = jedi_obj
+    while obj is not None and not obj.isinstance(tree.Function, tree.Class):
+        if obj.isinstance(tree.Flow):
+            # try/except catch check
+            if obj.isinstance(tree.TryStmt) and check_try_for_except(obj, exception):
+                return True
+            # hasattr check
+            if exception == AttributeError and obj.isinstance(tree.IfStmt, tree.WhileStmt):
+                if check_hasattr(obj.children[1], obj.children[3]):
+                    return True
+        obj = obj.parent
+
+    return False

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/cache.py

@@ -0,0 +1,58 @@
+"""
+- the popular ``memoize_default`` works like a typical memoize and returns the
+  default otherwise.
+- ``CachedMetaClass`` uses ``memoize_default`` to do the same with classes.
+"""
+
+import inspect
+
+NO_DEFAULT = object()
+
+
+def memoize_default(default=NO_DEFAULT, evaluator_is_first_arg=False, second_arg_is_evaluator=False):
+    """ This is a typical memoization decorator, BUT there is one difference:
+    To prevent recursion it sets defaults.
+
+    Preventing recursion is in this case the much bigger use than speed. I
+    don't think, that there is a big speed difference, but there are many cases
+    where recursion could happen (think about a = b; b = a).
+    """
+    def func(function):
+        def wrapper(obj, *args, **kwargs):
+            if evaluator_is_first_arg:
+                cache = obj.memoize_cache
+            elif second_arg_is_evaluator:  # needed for meta classes
+                cache = args[0].memoize_cache
+            else:
+                cache = obj._evaluator.memoize_cache
+
+            try:
+                memo = cache[function]
+            except KeyError:
+                memo = {}
+                cache[function] = memo
+
+            key = (obj, args, frozenset(kwargs.items()))
+            if key in memo:
+                return memo[key]
+            else:
+                if default is not NO_DEFAULT:
+                    memo[key] = default
+                rv = function(obj, *args, **kwargs)
+                if inspect.isgenerator(rv):
+                    rv = list(rv)
+                memo[key] = rv
+                return rv
+        return wrapper
+    return func
+
+
+class CachedMetaClass(type):
+    """
+    This is basically almost the same than the decorator above, it just caches
+    class initializations. Either you do it this way or with decorators, but
+    with decorators you lose class access (isinstance, etc).
+    """
+    @memoize_default(None, second_arg_is_evaluator=True)
+    def __call__(self, *args, **kwargs):
+        return super(CachedMetaClass, self).__call__(*args, **kwargs)

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/__init__.py

@@ -0,0 +1,544 @@
+"""
+Imitate the parser representation.
+"""
+import inspect
+import re
+import sys
+import os
+from functools import partial
+
+from jedi._compatibility import builtins as _builtins, unicode
+from jedi import debug
+from jedi.cache import underscore_memoization, memoize_method
+from jedi.parser.tree import Param, Base, Operator, zero_position_modifier
+from jedi.evaluate.helpers import FakeName
+from . import fake
+
+
+_sep = os.path.sep
+if os.path.altsep is not None:
+    _sep += os.path.altsep
+_path_re = re.compile('(?:\.[^{0}]+|[{0}]__init__\.py)$'.format(re.escape(_sep)))
+del _sep
+
+
+class CheckAttribute(object):
+    """Raises an AttributeError if the attribute X isn't available."""
+    def __init__(self, func):
+        self.func = func
+        # Remove the py in front of e.g. py__call__.
+        self.check_name = func.__name__[2:]
+
+    def __get__(self, instance, owner):
+        # This might raise an AttributeError. That's wanted.
+        getattr(instance.obj, self.check_name)
+        return partial(self.func, instance)
+
+
+class CompiledObject(Base):
+    # comply with the parser
+    start_pos = 0, 0
+    path = None  # modules have this attribute - set it to None.
+    used_names = {}  # To be consistent with modules.
+
+    def __init__(self, evaluator, obj, parent=None):
+        self._evaluator = evaluator
+        self.obj = obj
+        self.parent = parent
+
+    @CheckAttribute
+    def py__call__(self, params):
+        if inspect.isclass(self.obj):
+            from jedi.evaluate.representation import Instance
+            return set([Instance(self._evaluator, self, params)])
+        else:
+            return set(self._execute_function(params))
+
+    @CheckAttribute
+    def py__class__(self):
+        return create(self._evaluator, self.obj.__class__)
+
+    @CheckAttribute
+    def py__mro__(self):
+        return tuple(create(self._evaluator, cls) for cls in self.obj.__mro__)
+
+    @CheckAttribute
+    def py__bases__(self):
+        return tuple(create(self._evaluator, cls) for cls in self.obj.__bases__)
+
+    def py__bool__(self):
+        return bool(self.obj)
+
+    def py__file__(self):
+        return self.obj.__file__
+
+    def is_class(self):
+        return inspect.isclass(self.obj)
+
+    @property
+    def doc(self):
+        return inspect.getdoc(self.obj) or ''
+
+    @property
+    def params(self):
+        params_str, ret = self._parse_function_doc()
+        tokens = params_str.split(',')
+        if inspect.ismethoddescriptor(self.obj):
+            tokens.insert(0, 'self')
+        params = []
+        for p in tokens:
+            parts = [FakeName(part) for part in p.strip().split('=')]
+            if len(parts) > 1:
+                parts.insert(1, Operator(zero_position_modifier, '=', (0, 0)))
+            params.append(Param(parts, self))
+        return params
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, repr(self.obj))
+
+    @underscore_memoization
+    def _parse_function_doc(self):
+        if self.doc is None:
+            return '', ''
+
+        return _parse_function_doc(self.doc)
+
+    def api_type(self):
+        obj = self.obj
+        if inspect.isclass(obj):
+            return 'class'
+        elif inspect.ismodule(obj):
+            return 'module'
+        elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \
+                or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj):
+            return 'function'
+        # Everything else...
+        return 'instance'
+
+    @property
+    def type(self):
+        """Imitate the tree.Node.type values."""
+        cls = self._get_class()
+        if inspect.isclass(cls):
+            return 'classdef'
+        elif inspect.ismodule(cls):
+            return 'file_input'
+        elif inspect.isbuiltin(cls) or inspect.ismethod(cls) or \
+                inspect.ismethoddescriptor(cls):
+            return 'funcdef'
+
+    @underscore_memoization
+    def _cls(self):
+        """
+        We used to limit the lookups for instantiated objects like list(), but
+        this is not the case anymore. Python itself
+        """
+        # Ensures that a CompiledObject is returned that is not an instance (like list)
+        return self
+
+    def _get_class(self):
+        if not fake.is_class_instance(self.obj) or \
+                inspect.ismethoddescriptor(self.obj):  # slots
+            return self.obj
+
+        try:
+            return self.obj.__class__
+        except AttributeError:
+            # happens with numpy.core.umath._UFUNC_API (you get it
+            # automatically by doing `import numpy`.
+            return type
+
+    @property
+    def names_dict(self):
+        # For compatibility with `representation.Class`.
+        return self.names_dicts(False)[0]
+
+    def names_dicts(self, search_global, is_instance=False):
+        return self._names_dict_ensure_one_dict(is_instance)
+
+    @memoize_method
+    def _names_dict_ensure_one_dict(self, is_instance):
+        """
+        search_global shouldn't change the fact that there's one dict, this way
+        there's only one `object`.
+        """
+        return [LazyNamesDict(self._evaluator, self, is_instance)]
+
+    def get_subscope_by_name(self, name):
+        if name in dir(self.obj):
+            return CompiledName(self._evaluator, self, name).parent
+        else:
+            raise KeyError("CompiledObject doesn't have an attribute '%s'." % name)
+
+    @CheckAttribute
+    def py__getitem__(self, index):
+        if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return set()
+
+        return set([create(self._evaluator, self.obj[index])])
+
+    @CheckAttribute
+    def py__iter__(self):
+        if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict):
+            # Get rid of side effects, we won't call custom `__getitem__`s.
+            return
+
+        for part in self.obj:
+            yield set([create(self._evaluator, part)])
+
+    @property
+    def name(self):
+        try:
+            name = self._get_class().__name__
+        except AttributeError:
+            name = repr(self.obj)
+        return FakeName(name, self)
+
+    def _execute_function(self, params):
+        if self.type != 'funcdef':
+            return
+
+        for name in self._parse_function_doc()[1].split():
+            try:
+                bltn_obj = getattr(_builtins, name)
+            except AttributeError:
+                continue
+            else:
+                if bltn_obj is None:
+                    # We want to evaluate everything except None.
+                    # TODO do we?
+                    continue
+                bltn_obj = create(self._evaluator, bltn_obj)
+                for result in self._evaluator.execute(bltn_obj, params):
+                    yield result
+
+    @property
+    @underscore_memoization
+    def subscopes(self):
+        """
+        Returns only the faked scopes - the other ones are not important for
+        internal analysis.
+        """
+        module = self.get_parent_until()
+        faked_subscopes = []
+        for name in dir(self.obj):
+            try:
+                faked_subscopes.append(
+                    fake.get_faked(module.obj, self.obj, parent=self, name=name)
+                )
+            except fake.FakeDoesNotExist:
+                pass
+        return faked_subscopes
+
+    def is_scope(self):
+        return True
+
+    def get_self_attributes(self):
+        return []  # Instance compatibility
+
+    def get_imports(self):
+        return []  # Builtins don't have imports
+
+
+class CompiledName(FakeName):
+    def __init__(self, evaluator, compiled_obj, name):
+        super(CompiledName, self).__init__(name)
+        self._evaluator = evaluator
+        self._compiled_obj = compiled_obj
+        self.name = name
+
+    def __repr__(self):
+        try:
+            name = self._compiled_obj.name  # __name__ is not defined all the time
+        except AttributeError:
+            name = None
+        return '<%s: (%s).%s>' % (type(self).__name__, name, self.name)
+
+    def is_definition(self):
+        return True
+
+    @property
+    @underscore_memoization
+    def parent(self):
+        module = self._compiled_obj.get_parent_until()
+        return _create_from_name(self._evaluator, module, self._compiled_obj, self.name)
+
+    @parent.setter
+    def parent(self, value):
+        pass  # Just ignore this, FakeName tries to overwrite the parent attribute.
+
+
+class LazyNamesDict(object):
+    """
+    A names_dict instance for compiled objects, resembles the parser.tree.
+    """
+    name_class = CompiledName
+
+    def __init__(self, evaluator, compiled_obj, is_instance=False):
+        self._evaluator = evaluator
+        self._compiled_obj = compiled_obj
+        self._is_instance = is_instance
+
+    def __iter__(self):
+        return (v[0].value for v in self.values())
+
+    @memoize_method
+    def __getitem__(self, name):
+        try:
+            getattr(self._compiled_obj.obj, name)
+        except AttributeError:
+            raise KeyError('%s in %s not found.' % (name, self._compiled_obj))
+        except Exception:
+            # This is a bit ugly. We're basically returning this to make
+            # lookups possible without having the actual attribute. However
+            # this makes proper completion possible.
+            return [FakeName(name, create(self._evaluator, None), is_definition=True)]
+        return [self.name_class(self._evaluator, self._compiled_obj, name)]
+
+    def values(self):
+        obj = self._compiled_obj.obj
+
+        values = []
+        for name in dir(obj):
+            try:
+                values.append(self[name])
+            except KeyError:
+                # The dir function can be wrong.
+                pass
+
+        is_instance = self._is_instance or fake.is_class_instance(obj)
+        # ``dir`` doesn't include the type names.
+        if not inspect.ismodule(obj) and obj != type and not is_instance:
+            values += create(self._evaluator, type).names_dict.values()
+        return values
+
+
+def dotted_from_fs_path(fs_path, sys_path):
+    """
+    Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`.  I.e.
+    compares the path with sys.path and then returns the dotted_path. If the
+    path is not in the sys.path, just returns None.
+    """
+    if os.path.basename(fs_path).startswith('__init__.'):
+        # We are calculating the path. __init__ files are not interesting.
+        fs_path = os.path.dirname(fs_path)
+
+    # prefer
+    #   - UNIX
+    #     /path/to/pythonX.Y/lib-dynload
+    #     /path/to/pythonX.Y/site-packages
+    #   - Windows
+    #     C:\path\to\DLLs
+    #     C:\path\to\Lib\site-packages
+    # over
+    #   - UNIX
+    #     /path/to/pythonX.Y
+    #   - Windows
+    #     C:\path\to\Lib
+    path = ''
+    for s in sys_path:
+        if (fs_path.startswith(s) and len(path) < len(s)):
+            path = s
+
+    # - Window
+    # X:\path\to\lib-dynload/datetime.pyd => datetime
+    module_path = fs_path[len(path):].lstrip(os.path.sep).lstrip('/')
+    # - Window
+    # Replace like X:\path\to\something/foo/bar.py
+    return _path_re.sub('', module_path).replace(os.path.sep, '.').replace('/', '.')
+
+
+def load_module(evaluator, path=None, name=None):
+    sys_path = evaluator.sys_path
+    if path is not None:
+        dotted_path = dotted_from_fs_path(path, sys_path=sys_path)
+    else:
+        dotted_path = name
+
+    if dotted_path is None:
+        p, _, dotted_path = path.partition(os.path.sep)
+        sys_path.insert(0, p)
+
+    temp, sys.path = sys.path, sys_path
+    try:
+        __import__(dotted_path)
+    except RuntimeError:
+        if 'PySide' in dotted_path or 'PyQt' in dotted_path:
+            # RuntimeError: the PyQt4.QtCore and PyQt5.QtCore modules both wrap
+            # the QObject class.
+            # See https://github.com/davidhalter/jedi/pull/483
+            return None
+        raise
+    except ImportError:
+        # If a module is "corrupt" or not really a Python module or whatever.
+        debug.warning('Module %s not importable.', path)
+        return None
+    finally:
+        sys.path = temp
+
+    # Just access the cache after import, because of #59 as well as the very
+    # complicated import structure of Python.
+    module = sys.modules[dotted_path]
+
+    return create(evaluator, module)
+
+
+docstr_defaults = {
+    'floating point number': 'float',
+    'character': 'str',
+    'integer': 'int',
+    'dictionary': 'dict',
+    'string': 'str',
+}
+
+
+def _parse_function_doc(doc):
+    """
+    Takes a function and returns the params and return value as a tuple.
+    This is nothing more than a docstring parser.
+
+    TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
+    TODO docstrings like 'tuple of integers'
+    """
+    # parse round parentheses: def func(a, (b,c))
+    try:
+        count = 0
+        start = doc.index('(')
+        for i, s in enumerate(doc[start:]):
+            if s == '(':
+                count += 1
+            elif s == ')':
+                count -= 1
+            if count == 0:
+                end = start + i
+                break
+        param_str = doc[start + 1:end]
+    except (ValueError, UnboundLocalError):
+        # ValueError for doc.index
+        # UnboundLocalError for undefined end in last line
+        debug.dbg('no brackets found - no param')
+        end = 0
+        param_str = ''
+    else:
+        # remove square brackets, that show an optional param ( = None)
+        def change_options(m):
+            args = m.group(1).split(',')
+            for i, a in enumerate(args):
+                if a and '=' not in a:
+                    args[i] += '=None'
+            return ','.join(args)
+
+        while True:
+            param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
+                                         change_options, param_str)
+            if changes == 0:
+                break
+    param_str = param_str.replace('-', '_')  # see: isinstance.__doc__
+
+    # parse return value
+    r = re.search('-[>-]* ', doc[end:end + 7])
+    if r is None:
+        ret = ''
+    else:
+        index = end + r.end()
+        # get result type, which can contain newlines
+        pattern = re.compile(r'(,\n|[^\n-])+')
+        ret_str = pattern.match(doc, index).group(0).strip()
+        # New object -> object()
+        ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
+
+        ret = docstr_defaults.get(ret_str, ret_str)
+
+    return param_str, ret
+
+
+def _create_from_name(evaluator, module, parent, name):
+    try:
+        return fake.get_faked(module.obj, parent.obj, parent=parent, name=name)
+    except fake.FakeDoesNotExist:
+        pass
+
+    try:
+        obj = getattr(parent.obj, name)
+    except AttributeError:
+        # Happens e.g. in properties of
+        # PyQt4.QtGui.QStyleOptionComboBox.currentText
+        # -> just set it to None
+        obj = None
+    return create(evaluator, obj, parent)
+
+
+def builtin_from_name(evaluator, string):
+    bltn_obj = getattr(_builtins, string)
+    return create(evaluator, bltn_obj)
+
+
+def _a_generator(foo):
+    """Used to have an object to return for generators."""
+    yield 42
+    yield foo
+
+
+_SPECIAL_OBJECTS = {
+    'FUNCTION_CLASS': type(load_module),
+    'METHOD_CLASS': type(CompiledObject.is_class),
+    'MODULE_CLASS': type(os),
+    'GENERATOR_OBJECT': _a_generator(1.0),
+    'BUILTINS': _builtins,
+}
+
+
+def get_special_object(evaluator, identifier):
+    obj = _SPECIAL_OBJECTS[identifier]
+    return create(evaluator, obj, parent=create(evaluator, _builtins))
+
+
+def compiled_objects_cache(attribute_name):
+    def decorator(func):
+        """
+        This decorator caches just the ids, oopposed to caching the object itself.
+        Caching the id has the advantage that an object doesn't need to be
+        hashable.
+        """
+        def wrapper(evaluator, obj, parent=None, module=None):
+            cache = getattr(evaluator, attribute_name)
+            # Do a very cheap form of caching here.
+            key = id(obj), id(parent)
+            try:
+                return cache[key][0]
+            except KeyError:
+                # TODO this whole decorator looks way too ugly and this if
+                # doesn't make it better. Find a more generic solution.
+                if parent or module:
+                    result = func(evaluator, obj, parent, module)
+                else:
+                    result = func(evaluator, obj)
+                # Need to cache all of them, otherwise the id could be overwritten.
+                cache[key] = result, obj, parent, module
+                return result
+        return wrapper
+
+    return decorator
+
+
+@compiled_objects_cache('compiled_cache')
+def create(evaluator, obj, parent=None, module=None):
+    """
+    A very weird interface class to this module. The more options provided the
+    more acurate loading compiled objects is.
+    """
+    if inspect.ismodule(obj):
+        if parent is not None:
+            # Modules don't have parents, be careful with caching: recurse.
+            return create(evaluator, obj)
+    else:
+        if parent is None and obj != _builtins:
+            return create(evaluator, obj, create(evaluator, _builtins))
+
+        try:
+            return fake.get_faked(module and module.obj, obj, parent=parent)
+        except fake.FakeDoesNotExist:
+            pass
+
+    return CompiledObject(evaluator, obj, parent)

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/fake.py

@@ -0,0 +1,203 @@
+"""
+Loads functions that are mixed in to the standard library. E.g. builtins are
+written in C (binaries), but my autocompletion only understands Python code. By
+mixing in Python code, the autocompletion should work much better for builtins.
+"""
+
+import os
+import inspect
+import types
+
+from jedi._compatibility import is_py3, builtins, unicode, is_py34
+from jedi.parser import ParserWithRecovery, load_grammar
+from jedi.parser import tree as pt
+from jedi.evaluate.helpers import FakeName
+
+modules = {}
+
+
+MethodDescriptorType = type(str.replace)
+# These are not considered classes and access is granted even though they have
+# a __class__ attribute.
+NOT_CLASS_TYPES = (
+    types.BuiltinFunctionType,
+    types.CodeType,
+    types.FrameType,
+    types.FunctionType,
+    types.GeneratorType,
+    types.GetSetDescriptorType,
+    types.LambdaType,
+    types.MemberDescriptorType,
+    types.MethodType,
+    types.ModuleType,
+    types.TracebackType,
+    MethodDescriptorType
+)
+
+if is_py3:
+    NOT_CLASS_TYPES += (
+        types.MappingProxyType,
+        types.SimpleNamespace
+    )
+    if is_py34:
+        NOT_CLASS_TYPES += (types.DynamicClassAttribute,)
+
+
+class FakeDoesNotExist(Exception):
+    pass
+
+
+def _load_faked_module(module):
+    module_name = module.__name__
+    if module_name == '__builtin__' and not is_py3:
+        module_name = 'builtins'
+
+    try:
+        return modules[module_name]
+    except KeyError:
+        path = os.path.dirname(os.path.abspath(__file__))
+        try:
+            with open(os.path.join(path, 'fake', module_name) + '.pym') as f:
+                source = f.read()
+        except IOError:
+            modules[module_name] = None
+            return
+        grammar = load_grammar(version='3.4')
+        module = ParserWithRecovery(grammar, unicode(source), module_name).module
+        modules[module_name] = module
+
+        if module_name == 'builtins' and not is_py3:
+            # There are two implementations of `open` for either python 2/3.
+            # -> Rename the python2 version (`look at fake/builtins.pym`).
+            open_func = search_scope(module, 'open')
+            open_func.children[1] = FakeName('open_python3')
+            open_func = search_scope(module, 'open_python2')
+            open_func.children[1] = FakeName('open')
+        return module
+
+
+def search_scope(scope, obj_name):
+    for s in scope.subscopes:
+        if str(s.name) == obj_name:
+            return s
+
+
+def get_module(obj):
+    if inspect.ismodule(obj):
+        return obj
+    try:
+        obj = obj.__objclass__
+    except AttributeError:
+        pass
+
+    try:
+        imp_plz = obj.__module__
+    except AttributeError:
+        # Unfortunately in some cases like `int` there's no __module__
+        return builtins
+    else:
+        if imp_plz is None:
+            # Happens for example in `(_ for _ in []).send.__module__`.
+            return builtins
+        else:
+            try:
+                return __import__(imp_plz)
+            except ImportError:
+                # __module__ can be something arbitrary that doesn't exist.
+                return builtins
+
+
+def _faked(module, obj, name):
+    # Crazy underscore actions to try to escape all the internal madness.
+    if module is None:
+        module = get_module(obj)
+
+    faked_mod = _load_faked_module(module)
+    if faked_mod is None:
+        return None
+
+    # Having the module as a `parser.representation.module`, we need to scan
+    # for methods.
+    if name is None:
+        if inspect.isbuiltin(obj):
+            return search_scope(faked_mod, obj.__name__)
+        elif not inspect.isclass(obj):
+            # object is a method or descriptor
+            try:
+                objclass = obj.__objclass__
+            except AttributeError:
+                return None
+            else:
+                cls = search_scope(faked_mod, objclass.__name__)
+                if cls is None:
+                    return None
+                return search_scope(cls, obj.__name__)
+    else:
+        if obj == module:
+            return search_scope(faked_mod, name)
+        else:
+            try:
+                cls_name = obj.__name__
+            except AttributeError:
+                return None
+            cls = search_scope(faked_mod, cls_name)
+            if cls is None:
+                return None
+            return search_scope(cls, name)
+
+
+def memoize_faked(obj):
+    """
+    A typical memoize function that ignores issues with non hashable results.
+    """
+    cache = obj.cache = {}
+
+    def memoizer(*args, **kwargs):
+        key = (obj, args, frozenset(kwargs.items()))
+        try:
+            result = cache[key]
+        except TypeError:
+            return obj(*args, **kwargs)
+        except KeyError:
+            result = obj(*args, **kwargs)
+            if result is not None:
+                cache[key] = obj(*args, **kwargs)
+            return result
+        else:
+            return result
+    return memoizer
+
+
+@memoize_faked
+def _get_faked(module, obj, name=None):
+    obj = type(obj) if is_class_instance(obj) else obj
+    result = _faked(module, obj, name)
+    if result is None or isinstance(result, pt.Class):
+        # We're not interested in classes. What we want is functions.
+        raise FakeDoesNotExist
+    else:
+        # Set the docstr which was previously not set (faked modules don't
+        # contain it).
+        doc = '"""%s"""' % obj.__doc__  # TODO need escapes.
+        suite = result.children[-1]
+        string = pt.String(pt.zero_position_modifier, doc, (0, 0), '')
+        new_line = pt.Newline('\n', (0, 0), '')
+        docstr_node = pt.Node('simple_stmt', [string, new_line])
+        suite.children.insert(2, docstr_node)
+        return result
+
+
+def get_faked(module, obj, name=None, parent=None):
+    faked = _get_faked(module, obj, name)
+    faked.parent = parent
+    return faked
+
+
+def is_class_instance(obj):
+    """Like inspect.* methods."""
+    try:
+        cls = obj.__class__
+    except AttributeError:
+        return False
+    else:
+        return cls != type and not issubclass(cls, NOT_CLASS_TYPES)

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/fake/_functools.pym

@@ -0,0 +1,9 @@
+class partial():
+    def __init__(self, func, *args, **keywords):
+        self.__func = func
+        self.__args = args
+        self.__keywords = keywords
+
+    def __call__(self, *args, **kwargs):
+        # TODO should be **dict(self.__keywords, **kwargs)
+        return self.__func(*(self.__args + args), **self.__keywords)

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/fake/_sqlite3.pym

@@ -0,0 +1,26 @@
+def connect(database, timeout=None, isolation_level=None, detect_types=None, factory=None):
+    return Connection()
+
+
+class Connection():
+    def cursor(self):
+        return Cursor()
+
+
+class Cursor():
+    def cursor(self):
+        return Cursor()
+
+    def fetchone(self):
+        return Row()
+
+    def fetchmany(self, size=cursor.arraysize):
+        return [self.fetchone()]
+
+    def fetchall(self):
+        return [self.fetchone()]
+
+
+class Row():
+    def keys(self):
+        return ['']

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/fake/_sre.pym

@@ -0,0 +1,99 @@
+def compile():
+    class SRE_Match():
+        endpos = int()
+        lastgroup = int()
+        lastindex = int()
+        pos = int()
+        string = str()
+        regs = ((int(), int()),)
+
+        def __init__(self, pattern):
+            self.re = pattern
+
+        def start(self):
+            return int()
+
+        def end(self):
+            return int()
+
+        def span(self):
+            return int(), int()
+
+        def expand(self):
+            return str()
+
+        def group(self, nr):
+            return str()
+
+        def groupdict(self):
+            return {str(): str()}
+
+        def groups(self):
+            return (str(),)
+
+    class SRE_Pattern():
+        flags = int()
+        groupindex = {}
+        groups = int()
+        pattern = str()
+
+        def findall(self, string, pos=None, endpos=None):
+            """
+            findall(string[, pos[, endpos]]) --> list.
+            Return a list of all non-overlapping matches of pattern in string.
+            """
+            return [str()]
+
+        def finditer(self, string, pos=None, endpos=None):
+            """
+            finditer(string[, pos[, endpos]]) --> iterator.
+            Return an iterator over all non-overlapping matches for the
+            RE pattern in string. For each match, the iterator returns a
+            match object.
+            """
+            yield SRE_Match(self)
+
+        def match(self, string, pos=None, endpos=None):
+            """
+            match(string[, pos[, endpos]]) --> match object or None.
+            Matches zero or more characters at the beginning of the string
+            pattern
+            """
+            return SRE_Match(self)
+
+        def scanner(self, string, pos=None, endpos=None):
+            pass
+
+        def search(self, string, pos=None, endpos=None):
+            """
+            search(string[, pos[, endpos]]) --> match object or None.
+            Scan through string looking for a match, and return a corresponding
+            MatchObject instance. Return None if no position in the string matches.
+            """
+            return SRE_Match(self)
+
+        def split(self, string, maxsplit=0]):
+            """
+            split(string[, maxsplit = 0])  --> list.
+            Split string by the occurrences of pattern.
+            """
+            return [str()]
+
+        def sub(self, repl, string, count=0):
+            """
+            sub(repl, string[, count = 0]) --> newstring
+            Return the string obtained by replacing the leftmost non-overlapping
+            occurrences of pattern in string by the replacement repl.
+            """
+            return str()
+
+        def subn(self, repl, string, count=0):
+            """
+            subn(repl, string[, count = 0]) --> (newstring, number of subs)
+            Return the tuple (new_string, number_of_subs_made) found by replacing
+            the leftmost non-overlapping occurrences of pattern with the
+            replacement repl.
+            """
+            return (str(), int())
+
+    return SRE_Pattern()

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/fake/_weakref.pym

@@ -0,0 +1,8 @@
+def proxy(object, callback=None):
+    return object
+
+class weakref():
+    def __init__(self, object, callback=None):
+        self.__object = object
+    def __call__(self):
+        return self.__object

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/fake/builtins.pym

@@ -0,0 +1,262 @@
+"""
+Pure Python implementation of some builtins.
+This code is not going to be executed anywhere.
+These implementations are not always correct, but should work as good as
+possible for the auto completion.
+"""
+
+
+def next(iterator, default=None):
+    if random.choice([0, 1]):
+        if hasattr("next"):
+            return iterator.next()
+        else:
+            return iterator.__next__()
+    else:
+        if default is not None:
+            return default
+
+
+def iter(collection, sentinel=None):
+    if sentinel:
+        yield collection()
+    else:
+        for c in collection:
+            yield c
+
+
+def range(start, stop=None, step=1):
+    return [0]
+
+
+class file():
+    def __iter__(self):
+        yield ''
+    def next(self):
+        return ''
+
+
+class xrange():
+    # Attention: this function doesn't exist in Py3k (there it is range).
+    def __iter__(self):
+        yield 1
+
+    def count(self):
+        return 1
+
+    def index(self):
+        return 1
+
+
+def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True):
+    import io
+    return io.TextIOWrapper(file, mode, buffering, encoding, errors, newline, closefd)
+
+
+def open_python2(name, mode=None, buffering=None):
+    return file(name, mode, buffering)
+
+
+#--------------------------------------------------------
+# descriptors
+#--------------------------------------------------------
+class property():
+    def __init__(self, fget, fset=None, fdel=None, doc=None):
+        self.fget = fget
+        self.fset = fset
+        self.fdel = fdel
+        self.__doc__ = doc
+
+    def __get__(self, obj, cls):
+        return self.fget(obj)
+
+    def __set__(self, obj, value):
+        self.fset(obj, value)
+
+    def __delete__(self, obj):
+        self.fdel(obj)
+
+    def setter(self, func):
+        self.fset = func
+        return self
+
+    def getter(self, func):
+        self.fget = func
+        return self
+
+    def deleter(self, func):
+        self.fdel = func
+        return self
+
+
+class staticmethod():
+    def __init__(self, func):
+        self.__func = func
+
+    def __get__(self, obj, cls):
+        return self.__func
+
+
+class classmethod():
+    def __init__(self, func):
+        self.__func = func
+
+    def __get__(self, obj, cls):
+        def _method(*args, **kwargs):
+            return self.__func(cls, *args, **kwargs)
+        return _method
+
+
+#--------------------------------------------------------
+# array stuff
+#--------------------------------------------------------
+class list():
+    def __init__(self, iterable=[]):
+        self.__iterable = []
+        for i in iterable:
+            self.__iterable += [i]
+
+    def __iter__(self):
+        for i in self.__iterable:
+            yield i
+
+    def __getitem__(self, y):
+        return self.__iterable[y]
+
+    def pop(self):
+        return self.__iterable[int()]
+
+
+class tuple():
+    def __init__(self, iterable=[]):
+        self.__iterable = []
+        for i in iterable:
+            self.__iterable += [i]
+
+    def __iter__(self):
+        for i in self.__iterable:
+            yield i
+
+    def __getitem__(self, y):
+        return self.__iterable[y]
+
+    def index(self):
+        return 1
+
+    def count(self):
+        return 1
+
+
+class set():
+    def __init__(self, iterable=[]):
+        self.__iterable = iterable
+
+    def __iter__(self):
+        for i in self.__iterable:
+            yield i
+
+    def pop(self):
+        return list(self.__iterable)[-1]
+
+    def copy(self):
+        return self
+
+    def difference(self, other):
+        return self - other
+
+    def intersection(self, other):
+        return self & other
+
+    def symmetric_difference(self, other):
+        return self ^ other
+
+    def union(self, other):
+        return self | other
+
+
+class frozenset():
+    def __init__(self, iterable=[]):
+        self.__iterable = iterable
+
+    def __iter__(self):
+        for i in self.__iterable:
+            yield i
+
+    def copy(self):
+        return self
+
+
+class dict():
+    def __init__(self, **elements):
+        self.__elements = elements
+
+    def clear(self):
+        # has a strange docstr
+        pass
+
+    def get(self, k, d=None):
+        # TODO implement
+        try:
+            #return self.__elements[k]
+            pass
+        except KeyError:
+            return d
+
+    def values(self):
+        return self.__elements.values()
+
+    def setdefault(self, k, d):
+        # TODO maybe also return the content
+        return d
+
+
+class enumerate():
+    def __init__(self, sequence, start=0):
+        self.__sequence = sequence
+
+    def __iter__(self):
+        for i in self.__sequence:
+            yield 1, i
+
+    def __next__(self):
+        return next(self.__iter__())
+
+    def next(self):
+        return next(self.__iter__())
+
+
+class reversed():
+    def __init__(self, sequence):
+        self.__sequence = sequence
+
+    def __iter__(self):
+        for i in self.__sequence:
+            yield i
+
+    def __next__(self):
+        return next(self.__iter__())
+
+    def next(self):
+        return next(self.__iter__())
+
+
+def sorted(iterable, cmp=None, key=None, reverse=False):
+    return iterable
+
+
+#--------------------------------------------------------
+# basic types
+#--------------------------------------------------------
+class int():
+    def __init__(self, x, base=None):
+        pass
+
+
+class str():
+    def __init__(self, obj):
+        pass
+
+
+class type():
+    def mro():
+        return [object]

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/fake/datetime.pym

@@ -0,0 +1,4 @@
+class datetime():
+    @staticmethod
+    def now():
+        return datetime()

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/fake/io.pym

@@ -0,0 +1,6 @@
+class TextIOWrapper():
+    def __next__(self):
+        return str()
+
+    def __iter__(self):
+        yield str()

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/fake/posix.pym

@@ -0,0 +1,5 @@
+def getcwd():
+    return ''
+
+def getcwdu():
+    return ''

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/compiled/mixed.py

@@ -0,0 +1,158 @@
+"""
+Used only for REPL Completion.
+"""
+
+import inspect
+import os
+
+from jedi import common
+from jedi.parser.fast import FastParser
+from jedi.evaluate import compiled
+from jedi.cache import underscore_memoization
+
+
+class MixedObject(object):
+    """
+    A ``MixedObject`` is used in two ways:
+
+    1. It uses the default logic of ``parser.tree`` objects,
+    2. except for getattr calls. The names dicts are generated in a fashion
+       like ``CompiledObject``.
+
+    This combined logic makes it possible to provide more powerful REPL
+    completion. It allows side effects that are not noticable with the default
+    parser structure to still be completeable.
+
+    The biggest difference from CompiledObject to MixedObject is that we are
+    generally dealing with Python code and not with C code. This will generate
+    fewer special cases, because we in Python you don't have the same freedoms
+    to modify the runtime.
+    """
+    def __init__(self, evaluator, obj, node_name):
+        self._evaluator = evaluator
+        self.obj = obj
+        self.node_name = node_name
+        self.definition = node_name.get_definition()
+
+    @property
+    def names_dict(self):
+        return LazyMixedNamesDict(self._evaluator, self)
+
+    def names_dicts(self, search_global):
+        # TODO is this needed?
+        assert search_global is False
+        return [self.names_dict]
+
+    def api_type(self):
+        mappings = {
+            'expr_stmt': 'statement',
+            'classdef': 'class',
+            'funcdef': 'function',
+            'file_input': 'module',
+        }
+        return mappings[self.definition.type]
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, repr(self.obj))
+
+    def __getattr__(self, name):
+        return getattr(self.definition, name)
+
+
+class MixedName(compiled.CompiledName):
+    """
+    The ``CompiledName._compiled_object`` is our MixedObject.
+    """
+    @property
+    @underscore_memoization
+    def parent(self):
+        return create(self._evaluator, getattr(self._compiled_obj.obj, self.name))
+
+    @parent.setter
+    def parent(self, value):
+        pass  # Just ignore this, Name tries to overwrite the parent attribute.
+
+    @property
+    def start_pos(self):
+        if isinstance(self.parent, MixedObject):
+            return self.parent.node_name.start_pos
+
+        # This means a start_pos that doesn't exist (compiled objects).
+        return (0, 0)
+
+
+class LazyMixedNamesDict(compiled.LazyNamesDict):
+    name_class = MixedName
+
+
+def parse(grammar, path):
+    with open(path) as f:
+        source = f.read()
+    source = common.source_to_unicode(source)
+    return FastParser(grammar, source, path)
+
+
+def _load_module(evaluator, path, python_object):
+    module = parse(evaluator.grammar, path).module
+    python_module = inspect.getmodule(python_object)
+
+    evaluator.modules[python_module.__name__] = module
+    return module
+
+
+def find_syntax_node_name(evaluator, python_object):
+    try:
+        path = inspect.getsourcefile(python_object)
+    except TypeError:
+        # The type might not be known (e.g. class_with_dict.__weakref__)
+        return None
+    if path is None or not os.path.exists(path):
+        # The path might not exist or be e.g. <stdin>.
+        return None
+
+    module = _load_module(evaluator, path, python_object)
+
+    if inspect.ismodule(python_object):
+        # We don't need to check names for modules, because there's not really
+        # a way to write a module in a module in Python (and also __name__ can
+        # be something like ``email.utils``).
+        return module
+
+    name_str = python_object.__name__
+    if name_str == '<lambda>':
+        return None  # It's too hard to find lambdas.
+
+    names = module.used_names[name_str]
+    names = [n for n in names if n.is_definition()]
+
+    try:
+        code = python_object.__code__
+        # By using the line number of a code object we make the lookup in a
+        # file pretty easy. There's still a possibility of people defining
+        # stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people
+        # do so we just don't care.
+        line_nr = code.co_firstlineno
+    except AttributeError:
+        pass
+    else:
+        line_names = [name for name in names if name.start_pos[0] == line_nr]
+        # There's a chance that the object is not available anymore, because
+        # the code has changed in the background.
+        if line_names:
+            return line_names[-1]
+
+    # It's really hard to actually get the right definition, here as a last
+    # resort we just return the last one. This chance might lead to odd
+    # completions at some points but will lead to mostly correct type
+    # inference, because people tend to define a public name in a module only
+    # once.
+    return names[-1]
+
+
+@compiled.compiled_objects_cache('mixed_cache')
+def create(evaluator, obj):
+    name = find_syntax_node_name(evaluator, obj)
+    if name is None:
+        return compiled.create(evaluator, obj)
+    else:
+        return MixedObject(evaluator, obj, name)

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/docstrings.py

@@ -0,0 +1,204 @@
+"""
+Docstrings are another source of information for functions and classes.
+:mod:`jedi.evaluate.dynamic` tries to find all executions of functions, while
+the docstring parsing is much easier. There are two different types of
+docstrings that |jedi| understands:
+
+- `Sphinx <http://sphinx-doc.org/markup/desc.html#info-field-lists>`_
+- `Epydoc <http://epydoc.sourceforge.net/manual-fields.html>`_
+
+For example, the sphinx annotation ``:type foo: str`` clearly states that the
+type of ``foo`` is ``str``.
+
+As an addition to parameter searching, this module also provides return
+annotations.
+"""
+
+from ast import literal_eval
+import re
+from itertools import chain
+from textwrap import dedent
+
+from jedi.evaluate.cache import memoize_default
+from jedi.parser import ParserWithRecovery, load_grammar
+from jedi.parser.tree import Class
+from jedi.common import indent_block
+from jedi.evaluate.iterable import Array, FakeSequence, AlreadyEvaluated
+
+
+DOCSTRING_PARAM_PATTERNS = [
+    r'\s*:type\s+%s:\s*([^\n]+)',  # Sphinx
+    r'\s*:param\s+(\w+)\s+%s:[^\n]+',  # Sphinx param with type
+    r'\s*@type\s+%s:\s*([^\n]+)',  # Epydoc
+]
+
+DOCSTRING_RETURN_PATTERNS = [
+    re.compile(r'\s*:rtype:\s*([^\n]+)', re.M),  # Sphinx
+    re.compile(r'\s*@rtype:\s*([^\n]+)', re.M),  # Epydoc
+]
+
+REST_ROLE_PATTERN = re.compile(r':[^`]+:`([^`]+)`')
+
+
+try:
+    from numpydoc.docscrape import NumpyDocString
+except ImportError:
+    def _search_param_in_numpydocstr(docstr, param_str):
+        return []
+else:
+    def _search_param_in_numpydocstr(docstr, param_str):
+        """Search `docstr` (in numpydoc format) for type(-s) of `param_str`."""
+        params = NumpyDocString(docstr)._parsed_data['Parameters']
+        for p_name, p_type, p_descr in params:
+            if p_name == param_str:
+                m = re.match('([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type)
+                if m:
+                    p_type = m.group(1)
+
+                if p_type.startswith('{'):
+                    types = set(type(x).__name__ for x in literal_eval(p_type))
+                    return list(types)
+                else:
+                    return [p_type]
+        return []
+
+
+def _search_param_in_docstr(docstr, param_str):
+    """
+    Search `docstr` for type(-s) of `param_str`.
+
+    >>> _search_param_in_docstr(':type param: int', 'param')
+    ['int']
+    >>> _search_param_in_docstr('@type param: int', 'param')
+    ['int']
+    >>> _search_param_in_docstr(
+    ...   ':type param: :class:`threading.Thread`', 'param')
+    ['threading.Thread']
+    >>> bool(_search_param_in_docstr('no document', 'param'))
+    False
+    >>> _search_param_in_docstr(':param int param: some description', 'param')
+    ['int']
+
+    """
+    # look at #40 to see definitions of those params
+    patterns = [re.compile(p % re.escape(param_str))
+                for p in DOCSTRING_PARAM_PATTERNS]
+    for pattern in patterns:
+        match = pattern.search(docstr)
+        if match:
+            return [_strip_rst_role(match.group(1))]
+
+    return (_search_param_in_numpydocstr(docstr, param_str) or
+            [])
+
+
+def _strip_rst_role(type_str):
+    """
+    Strip off the part looks like a ReST role in `type_str`.
+
+    >>> _strip_rst_role(':class:`ClassName`')  # strip off :class:
+    'ClassName'
+    >>> _strip_rst_role(':py:obj:`module.Object`')  # works with domain
+    'module.Object'
+    >>> _strip_rst_role('ClassName')  # do nothing when not ReST role
+    'ClassName'
+
+    See also:
+    http://sphinx-doc.org/domains.html#cross-referencing-python-objects
+
+    """
+    match = REST_ROLE_PATTERN.match(type_str)
+    if match:
+        return match.group(1)
+    else:
+        return type_str
+
+
+def _evaluate_for_statement_string(evaluator, string, module):
+    code = dedent("""
+    def pseudo_docstring_stuff():
+        # Create a pseudo function for docstring statements.
+    %s
+    """)
+    if string is None:
+        return []
+
+    for element in re.findall('((?:\w+\.)*\w+)\.', string):
+        # Try to import module part in dotted name.
+        # (e.g., 'threading' in 'threading.Thread').
+        string = 'import %s\n' % element + string
+
+    # Take the default grammar here, if we load the Python 2.7 grammar here, it
+    # will be impossible to use `...` (Ellipsis) as a token. Docstring types
+    # don't need to conform with the current grammar.
+    p = ParserWithRecovery(load_grammar(), code % indent_block(string))
+    try:
+        pseudo_cls = p.module.subscopes[0]
+        # First pick suite, then simple_stmt (-2 for DEDENT) and then the node,
+        # which is also not the last item, because there's a newline.
+        stmt = pseudo_cls.children[-1].children[-2].children[-2]
+    except (AttributeError, IndexError):
+        return []
+
+    # Use the module of the param.
+    # TODO this module is not the module of the param in case of a function
+    # call. In that case it's the module of the function call.
+    # stuffed with content from a function call.
+    pseudo_cls.parent = module
+    return list(_execute_types_in_stmt(evaluator, stmt))
+
+
+def _execute_types_in_stmt(evaluator, stmt):
+    """
+    Executing all types or general elements that we find in a statement. This
+    doesn't include tuple, list and dict literals, because the stuff they
+    contain is executed. (Used as type information).
+    """
+    definitions = evaluator.eval_element(stmt)
+    return chain.from_iterable(_execute_array_values(evaluator, d) for d in definitions)
+
+
+def _execute_array_values(evaluator, array):
+    """
+    Tuples indicate that there's not just one return value, but the listed
+    ones.  `(str, int)` means that it returns a tuple with both types.
+    """
+    if isinstance(array, Array):
+        values = []
+        for types in array.py__iter__():
+            objects = set(chain.from_iterable(_execute_array_values(evaluator, typ) for typ in types))
+            values.append(AlreadyEvaluated(objects))
+        return [FakeSequence(evaluator, values, array.type)]
+    else:
+        return evaluator.execute(array)
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def follow_param(evaluator, param):
+    def eval_docstring(docstring):
+        return set(
+            [p for param_str in _search_param_in_docstr(docstring, str(param.name))
+                for p in _evaluate_for_statement_string(evaluator, param_str, module)]
+        )
+    func = param.parent_function
+    module = param.get_parent_until()
+
+    types = eval_docstring(func.raw_doc)
+    if func.name.value == '__init__':
+        cls = func.get_parent_until(Class)
+        if cls.type == 'classdef':
+            types |= eval_docstring(cls.raw_doc)
+
+    return types
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def find_return_types(evaluator, func):
+    def search_return_in_docstr(code):
+        for p in DOCSTRING_RETURN_PATTERNS:
+            match = p.search(code)
+            if match:
+                return _strip_rst_role(match.group(1))
+
+    type_str = search_return_in_docstr(func.raw_doc)
+    return _evaluate_for_statement_string(evaluator, type_str, func.get_parent_until())

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/dynamic.py

@@ -0,0 +1,149 @@
+"""
+One of the really important features of |jedi| is to have an option to
+understand code like this::
+
+    def foo(bar):
+        bar. # completion here
+    foo(1)
+
+There's no doubt wheter bar is an ``int`` or not, but if there's also a call
+like ``foo('str')``, what would happen? Well, we'll just show both. Because
+that's what a human would expect.
+
+It works as follows:
+
+- |Jedi| sees a param
+- search for function calls named ``foo``
+- execute these calls and check the input. This work with a ``ParamListener``.
+"""
+from itertools import chain
+
+from jedi._compatibility import unicode
+from jedi.parser import tree
+from jedi import settings
+from jedi import debug
+from jedi.evaluate.cache import memoize_default
+from jedi.evaluate import imports
+
+
+MAX_PARAM_SEARCHES = 20
+
+
+class ParamListener(object):
+    """
+    This listener is used to get the params for a function.
+    """
+    def __init__(self):
+        self.param_possibilities = []
+
+    def execute(self, params):
+        self.param_possibilities += params
+
+
+@debug.increase_indent
+def search_params(evaluator, param):
+    """
+    A dynamic search for param values. If you try to complete a type:
+
+    >>> def func(foo):
+    ...     foo
+    >>> func(1)
+    >>> func("")
+
+    It is not known what the type ``foo`` without analysing the whole code. You
+    have to look for all calls to ``func`` to find out what ``foo`` possibly
+    is.
+    """
+    if not settings.dynamic_params:
+        return set()
+
+    evaluator.dynamic_params_depth += 1
+    try:
+        func = param.get_parent_until(tree.Function)
+        debug.dbg('Dynamic param search for %s in %s.', param, str(func.name), color='MAGENTA')
+        # Compare the param names.
+        names = [n for n in search_function_call(evaluator, func)
+                 if n.value == param.name.value]
+        # Evaluate the ExecutedParams to types.
+        result = set(chain.from_iterable(n.parent.eval(evaluator) for n in names))
+        debug.dbg('Dynamic param result %s', result, color='MAGENTA')
+        return result
+    finally:
+        evaluator.dynamic_params_depth -= 1
+
+
+@memoize_default([], evaluator_is_first_arg=True)
+def search_function_call(evaluator, func):
+    """
+    Returns a list of param names.
+    """
+    from jedi.evaluate import representation as er
+
+    def get_possible_nodes(module, func_name):
+            try:
+                names = module.used_names[func_name]
+            except KeyError:
+                return
+
+            for name in names:
+                bracket = name.get_next_leaf()
+                trailer = bracket.parent
+                if trailer.type == 'trailer' and bracket == '(':
+                    yield name, trailer
+
+    def undecorate(typ):
+        # We have to remove decorators, because they are not the
+        # "original" functions, this way we can easily compare.
+        # At the same time we also have to remove InstanceElements.
+        if typ.isinstance(er.Function, er.Instance) \
+                and typ.decorates is not None:
+            return typ.decorates
+        elif isinstance(typ, er.InstanceElement):
+            return typ.var
+        else:
+            return typ
+
+    current_module = func.get_parent_until()
+    func_name = unicode(func.name)
+    compare = func
+    if func_name == '__init__':
+        cls = func.get_parent_scope()
+        if isinstance(cls, tree.Class):
+            func_name = unicode(cls.name)
+            compare = cls
+
+    # add the listener
+    listener = ParamListener()
+    func.listeners.add(listener)
+
+    try:
+        result = []
+        i = 0
+        for mod in imports.get_modules_containing_name(evaluator, [current_module], func_name):
+            for name, trailer in get_possible_nodes(mod, func_name):
+                i += 1
+
+                # This is a simple way to stop Jedi's dynamic param recursion
+                # from going wild: The deeper Jedi's in the recursin, the less
+                # code should be evaluated.
+                if i * evaluator.dynamic_params_depth > MAX_PARAM_SEARCHES:
+                    return listener.param_possibilities
+
+                for typ in evaluator.goto_definitions(name):
+                    undecorated = undecorate(typ)
+                    if evaluator.wrap(compare) == undecorated:
+                        # Only if we have the correct function we execute
+                        # it, otherwise just ignore it.
+                        evaluator.eval_trailer([typ], trailer)
+
+            result = listener.param_possibilities
+
+            # If there are results after processing a module, we're probably
+            # good to process.
+            if result:
+                return result
+    finally:
+        # cleanup: remove the listener; important: should not stick.
+        func.listeners.remove(listener)
+
+    return set()

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/finder.py

@@ -0,0 +1,632 @@
+"""
+Searching for names with given scope and name. This is very central in Jedi and
+Python. The name resolution is quite complicated with descripter,
+``__getattribute__``, ``__getattr__``, ``global``, etc.
+
+If you want to understand name resolution, please read the first few chapters
+in http://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/.
+
+Flow checks
++++++++++++
+
+Flow checks are not really mature. There's only a check for ``isinstance``.  It
+would check whether a flow has the form of ``if isinstance(a, type_or_tuple)``.
+Unfortunately every other thing is being ignored (e.g. a == '' would be easy to
+check for -> a is a string). There's big potential in these checks.
+"""
+from itertools import chain
+
+from jedi._compatibility import unicode
+from jedi.parser import tree
+from jedi import debug
+from jedi import common
+from jedi.common import unite
+from jedi import settings
+from jedi.evaluate import representation as er
+from jedi.evaluate import dynamic
+from jedi.evaluate import compiled
+from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
+from jedi.evaluate import iterable
+from jedi.evaluate import imports
+from jedi.evaluate import analysis
+from jedi.evaluate import flow_analysis
+from jedi.evaluate import param
+from jedi.evaluate import helpers
+from jedi.evaluate.cache import memoize_default
+
+
+def filter_after_position(names, position):
+    """
+    Removes all names after a certain position. If position is None, just
+    returns the names list.
+    """
+    if position is None:
+        return names
+
+    names_new = []
+    for n in names:
+        # Filter positions and also allow list comprehensions and lambdas.
+        if n.start_pos[0] is not None and n.start_pos < position \
+                or isinstance(n.get_definition(), (tree.CompFor, tree.Lambda)):
+            names_new.append(n)
+    return names_new
+
+
+def filter_definition_names(names, origin, position=None):
+    """
+    Filter names that are actual definitions in a scope. Names that are just
+    used will be ignored.
+    """
+    if not names:
+        return []
+
+    # Just calculate the scope from the first
+    stmt = names[0].get_definition()
+    scope = stmt.get_parent_scope()
+
+    if not (isinstance(scope, er.FunctionExecution) and
+            isinstance(scope.base, er.LambdaWrapper)):
+        names = filter_after_position(names, position)
+    names = [name for name in names if name.is_definition()]
+
+    # Private name mangling (compile.c) disallows access on names
+    # preceeded by two underscores `__` if used outside of the class. Names
+    # that also end with two underscores (e.g. __id__) are not affected.
+    for name in list(names):
+        if name.value.startswith('__') and not name.value.endswith('__'):
+            if filter_private_variable(scope, origin):
+                names.remove(name)
+    return names
+
+
+class NameFinder(object):
+    def __init__(self, evaluator, scope, name_str, position=None):
+        self._evaluator = evaluator
+        # Make sure that it's not just a syntax tree node.
+        self.scope = evaluator.wrap(scope)
+        self.name_str = name_str
+        self.position = position
+        self._found_predefined_if_name = None
+
+    @debug.increase_indent
+    def find(self, scopes, attribute_lookup):
+        """
+        :params bool attribute_lookup: Tell to logic if we're accessing the
+            attribute or the contents of e.g. a function.
+        """
+        # TODO rename scopes to names_dicts
+
+        names = self.filter_name(scopes)
+        if self._found_predefined_if_name is not None:
+            return self._found_predefined_if_name
+
+        types = self._names_to_types(names, attribute_lookup)
+
+        if not names and not types \
+                and not (isinstance(self.name_str, tree.Name) and
+                         isinstance(self.name_str.parent.parent, tree.Param)):
+            if not isinstance(self.name_str, (str, unicode)):  # TODO Remove?
+                if attribute_lookup:
+                    analysis.add_attribute_error(self._evaluator,
+                                                 self.scope, self.name_str)
+                else:
+                    message = ("NameError: name '%s' is not defined."
+                               % self.name_str)
+                    analysis.add(self._evaluator, 'name-error', self.name_str,
+                                 message)
+
+        debug.dbg('finder._names_to_types: %s -> %s', names, types)
+        return types
+
+    def scopes(self, search_global=False):
+        if search_global:
+            return global_names_dict_generator(self._evaluator, self.scope, self.position)
+        else:
+            return ((n, None) for n in self.scope.names_dicts(search_global))
+
+    def names_dict_lookup(self, names_dict, position):
+        def get_param(scope, el):
+            if isinstance(el.get_parent_until(tree.Param), tree.Param):
+                return scope.param_by_name(str(el))
+            return el
+
+        search_str = str(self.name_str)
+        try:
+            names = names_dict[search_str]
+            if not names:  # We want names, otherwise stop.
+                return []
+        except KeyError:
+            return []
+
+        names = filter_definition_names(names, self.name_str, position)
+
+        name_scope = None
+        # Only the names defined in the last position are valid definitions.
+        last_names = []
+        for name in reversed(sorted(names, key=lambda name: name.start_pos)):
+            stmt = name.get_definition()
+            name_scope = self._evaluator.wrap(stmt.get_parent_scope())
+
+            if isinstance(self.scope, er.Instance) and not isinstance(name_scope, er.Instance):
+                # Instances should not be checked for positioning, because we
+                # don't know in which order the functions are called.
+                last_names.append(name)
+                continue
+
+            if isinstance(name_scope, compiled.CompiledObject):
+                # Let's test this. TODO need comment. shouldn't this be
+                # filtered before?
+                last_names.append(name)
+                continue
+
+            if isinstance(stmt, er.ModuleWrapper):
+                # In case of REPL completion, we can infer modules names that
+                # don't really have a definition (because they are really just
+                # namespaces). In this case we can just add it.
+                last_names.append(name)
+                continue
+
+            if isinstance(name, compiled.CompiledName) \
+                    or isinstance(name, er.InstanceName) and isinstance(name._origin_name, compiled.CompiledName):
+                last_names.append(name)
+                continue
+
+            if isinstance(self.name_str, tree.Name):
+                origin_scope = self.name_str.get_parent_until(tree.Scope, reverse=True)
+                scope = self.name_str
+                check = None
+                while True:
+                    scope = scope.parent
+                    if scope.type in ("if_stmt", "for_stmt", "comp_for"):
+                        try:
+                            name_dict = self._evaluator.predefined_if_name_dict_dict[scope]
+                            types = set(name_dict[str(self.name_str)])
+                        except KeyError:
+                            continue
+                        else:
+                            if self.name_str.start_pos < scope.children[1].end_pos:
+                                # It doesn't make any sense to check if
+                                # statements in the if statement itself, just
+                                # deliver types.
+                                self._found_predefined_if_name = types
+                            else:
+                                check = flow_analysis.break_check(self._evaluator, self.scope,
+                                                                  origin_scope)
+                                if check is flow_analysis.UNREACHABLE:
+                                    self._found_predefined_if_name = set()
+                                else:
+                                    self._found_predefined_if_name = types
+                            break
+                    if isinstance(scope, tree.IsScope) or scope is None:
+                        break
+            else:
+                origin_scope = None
+
+            if isinstance(stmt.parent, compiled.CompiledObject):
+                # TODO seriously? this is stupid.
+                continue
+            check = flow_analysis.break_check(self._evaluator, name_scope,
+                                              stmt, origin_scope)
+            if check is not flow_analysis.UNREACHABLE:
+                last_names.append(name)
+
+            if check is flow_analysis.REACHABLE:
+                break
+
+        if isinstance(name_scope, er.FunctionExecution):
+            # Replace params
+            return [get_param(name_scope, n) for n in last_names]
+        return last_names
+
+    def filter_name(self, names_dicts):
+        """
+        Searches names that are defined in a scope (the different
+        `names_dicts`), until a name fits.
+        """
+        names = []
+        for names_dict, position in names_dicts:
+            names = self.names_dict_lookup(names_dict, position)
+            if names:
+                break
+
+        debug.dbg('finder.filter_name "%s" in (%s): %s@%s', self.name_str,
+                  self.scope, names, self.position)
+        return list(self._clean_names(names))
+
+    def _clean_names(self, names):
+        """
+        ``NameFinder.filter_name`` should only output names with correct
+        wrapper parents. We don't want to see AST classes out in the
+        evaluation, so remove them already here!
+        """
+        for n in names:
+            definition = n.parent
+            if isinstance(definition, (compiled.CompiledObject,
+                iterable.BuiltinMethod)):
+                # TODO this if should really be removed by changing the type of
+                #      those classes.
+                yield n
+            elif definition.type in ('funcdef', 'classdef', 'file_input'):
+                yield self._evaluator.wrap(definition).name
+            else:
+                yield n
+
+    def _check_getattr(self, inst):
+        """Checks for both __getattr__ and __getattribute__ methods"""
+        result = set()
+        # str is important, because it shouldn't be `Name`!
+        name = compiled.create(self._evaluator, str(self.name_str))
+        with common.ignored(KeyError):
+            result = inst.execute_subscope_by_name('__getattr__', name)
+        if not result:
+            # This is a little bit special. `__getattribute__` is in Python
+            # executed before `__getattr__`. But: I know no use case, where
+            # this could be practical and where jedi would return wrong types.
+            # If you ever find something, let me know!
+            # We are inversing this, because a hand-crafted `__getattribute__`
+            # could still call another hand-crafted `__getattr__`, but not the
+            # other way around.
+            with common.ignored(KeyError):
+                result = inst.execute_subscope_by_name('__getattribute__', name)
+        return result
+
+    def _names_to_types(self, names, attribute_lookup):
+        types = set()
+
+        # Add isinstance and other if/assert knowledge.
+        if isinstance(self.name_str, tree.Name):
+            # Ignore FunctionExecution parents for now.
+            flow_scope = self.name_str
+            until = flow_scope.get_parent_until(er.FunctionExecution)
+            while not isinstance(until, er.FunctionExecution):
+                flow_scope = flow_scope.get_parent_scope(include_flows=True)
+                if flow_scope is None:
+                    break
+                # TODO check if result is in scope -> no evaluation necessary
+                n = check_flow_information(self._evaluator, flow_scope,
+                                           self.name_str, self.position)
+                if n:
+                    return n
+
+        for name in names:
+            new_types = _name_to_types(self._evaluator, name, self.scope)
+            if isinstance(self.scope, (er.Class, er.Instance)) and attribute_lookup:
+                types |= set(self._resolve_descriptors(name, new_types))
+            else:
+                types |= set(new_types)
+        if not names and isinstance(self.scope, er.Instance):
+            # handling __getattr__ / __getattribute__
+            return self._check_getattr(self.scope)
+
+        return types
+
+    def _resolve_descriptors(self, name, types):
+        # The name must not be in the dictionary, but part of the class
+        # definition. __get__ is only called if the descriptor is defined in
+        # the class dictionary.
+        name_scope = name.get_definition().get_parent_scope()
+        if not isinstance(name_scope, (er.Instance, tree.Class)):
+            return types
+
+        result = set()
+        for r in types:
+            try:
+                desc_return = r.get_descriptor_returns
+            except AttributeError:
+                result.add(r)
+            else:
+                result |= desc_return(self.scope)
+        return result
+
+
+def _get_global_stmt_scopes(evaluator, global_stmt, name):
+    global_stmt_scope = global_stmt.get_parent_scope()
+    module = global_stmt_scope.get_parent_until()
+    for used_name in module.used_names[str(name)]:
+        if used_name.parent.type == 'global_stmt':
+            yield evaluator.wrap(used_name.get_parent_scope())
+
+
+@memoize_default(set(), evaluator_is_first_arg=True)
+def _name_to_types(evaluator, name, scope):
+    types = []
+    typ = name.get_definition()
+    if typ.isinstance(tree.ForStmt):
+        types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name)
+        if types:
+            return types
+    if typ.isinstance(tree.WithStmt):
+        types = pep0484.find_type_from_comment_hint_with(evaluator, typ, name)
+        if types:
+            return types
+    if typ.isinstance(tree.ForStmt, tree.CompFor):
+        container_types = evaluator.eval_element(typ.children[3])
+        for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3])
+        types = check_tuple_assignments(evaluator, for_types, name)
+    elif isinstance(typ, tree.Param):
+        types = _eval_param(evaluator, typ, scope)
+    elif typ.isinstance(tree.ExprStmt):
+        types = _remove_statements(evaluator, typ, name)
+    elif typ.isinstance(tree.WithStmt):
+        types = evaluator.eval_element(typ.node_from_name(name))
+    elif isinstance(typ, tree.Import):
+        types = imports.ImportWrapper(evaluator, name).follow()
+    elif typ.type == 'global_stmt':
+        for s in _get_global_stmt_scopes(evaluator, typ, name):
+            finder = NameFinder(evaluator, s, str(name))
+            names_dicts = finder.scopes(search_global=True)
+            # For global_stmt lookups, we only need the first possible scope,
+            # which means the function itself.
+            names_dicts = [next(names_dicts)]
+            types += finder.find(names_dicts, attribute_lookup=False)
+    elif isinstance(typ, tree.TryStmt):
+        # TODO an exception can also be a tuple. Check for those.
+        # TODO check for types that are not classes and add it to
+        # the static analysis report.
+        exceptions = evaluator.eval_element(name.get_previous_sibling().get_previous_sibling())
+        types = set(chain.from_iterable(evaluator.execute(t) for t in exceptions))
+    else:
+        if typ.isinstance(er.Function):
+            typ = typ.get_decorated_func()
+        types = set([typ])
+    return types
+
+
+def _remove_statements(evaluator, stmt, name):
+    """
+    This is the part where statements are being stripped.
+
+    Due to lazy evaluation, statements like a = func; b = a; b() have to be
+    evaluated.
+    """
+    types = set()
+    # Remove the statement docstr stuff for now, that has to be
+    # implemented with the evaluator class.
+    #if stmt.docstr:
+        #res_new.append(stmt)
+
+    check_instance = None
+    if isinstance(stmt, er.InstanceElement) and stmt.is_class_var:
+        check_instance = stmt.instance
+        stmt = stmt.var
+
+    pep0484types = \
+        pep0484.find_type_from_comment_hint_assign(evaluator, stmt, name)
+    if pep0484types:
+        return pep0484types
+    types |= evaluator.eval_statement(stmt, seek_name=name)
+
+    if check_instance is not None:
+        # class renames
+        types = set([er.get_instance_el(evaluator, check_instance, a, True)
+                     if isinstance(a, (er.Function, tree.Function))
+                     else a for a in types])
+    return types
+
+
+def _eval_param(evaluator, param, scope):
+    res_new = set()
+    func = param.get_parent_scope()
+
+    cls = func.parent.get_parent_until((tree.Class, tree.Function))
+
+    from jedi.evaluate.param import ExecutedParam, Arguments
+    if isinstance(cls, tree.Class) and param.position_nr == 0 \
+            and not isinstance(param, ExecutedParam):
+        # This is where we add self - if it has never been
+        # instantiated.
+        if isinstance(scope, er.InstanceElement):
+            res_new.add(scope.instance)
+        else:
+            inst = er.Instance(evaluator, evaluator.wrap(cls),
+                               Arguments(evaluator, ()), is_generated=True)
+            res_new.add(inst)
+        return res_new
+
+    # Instances are typically faked, if the instance is not called from
+    # outside. Here we check it for __init__ functions and return.
+    if isinstance(func, er.InstanceElement) \
+            and func.instance.is_generated and str(func.name) == '__init__':
+        param = func.var.params[param.position_nr]
+
+    # Add pep0484 and docstring knowledge.
+    pep0484_hints = pep0484.follow_param(evaluator, param)
+    doc_params = docstrings.follow_param(evaluator, param)
+    if pep0484_hints or doc_params:
+        return list(set(pep0484_hints) | set(doc_params))
+
+    if isinstance(param, ExecutedParam):
+        return res_new | param.eval(evaluator)
+    else:
+        # Param owns no information itself.
+        res_new |= dynamic.search_params(evaluator, param)
+        if not res_new:
+            if param.stars:
+                t = 'tuple' if param.stars == 1 else 'dict'
+                typ = list(evaluator.find_types(evaluator.BUILTINS, t))[0]
+                res_new = evaluator.execute(typ)
+        if param.default:
+            res_new |= evaluator.eval_element(param.default)
+        return res_new
+
+
+def check_flow_information(evaluator, flow, search_name, pos):
+    """ Try to find out the type of a variable just with the information that
+    is given by the flows: e.g. It is also responsible for assert checks.::
+
+        if isinstance(k, str):
+            k.  # <- completion here
+
+    ensures that `k` is a string.
+    """
+    if not settings.dynamic_flow_information:
+        return None
+
+    result = set()
+    if flow.is_scope():
+        # Check for asserts.
+        try:
+            names = reversed(flow.names_dict[search_name.value])
+        except (KeyError, AttributeError):
+            names = []
+
+        for name in names:
+            ass = name.get_parent_until(tree.AssertStmt)
+            if isinstance(ass, tree.AssertStmt) and pos is not None and ass.start_pos < pos:
+                result = _check_isinstance_type(evaluator, ass.assertion(), search_name)
+                if result:
+                    break
+
+    if isinstance(flow, (tree.IfStmt, tree.WhileStmt)):
+        potential_ifs = [c for c in flow.children[1::4] if c != ':']
+        for if_test in reversed(potential_ifs):
+            if search_name.start_pos > if_test.end_pos:
+                return _check_isinstance_type(evaluator, if_test, search_name)
+    return result
+
+
+def _check_isinstance_type(evaluator, element, search_name):
+    try:
+        assert element.type in ('power', 'atom_expr')
+        # this might be removed if we analyze and, etc
+        assert len(element.children) == 2
+        first, trailer = element.children
+        assert isinstance(first, tree.Name) and first.value == 'isinstance'
+        assert trailer.type == 'trailer' and trailer.children[0] == '('
+        assert len(trailer.children) == 3
+
+        # arglist stuff
+        arglist = trailer.children[1]
+        args = param.Arguments(evaluator, arglist, trailer)
+        lst = list(args.unpack())
+        # Disallow keyword arguments
+        assert len(lst) == 2 and lst[0][0] is None and lst[1][0] is None
+        name = lst[0][1][0]  # first argument, values, first value
+        # Do a simple get_code comparison. They should just have the same code,
+        # and everything will be all right.
+        classes = lst[1][1][0]
+        call = helpers.call_of_leaf(search_name)
+        assert name.get_code(normalized=True) == call.get_code(normalized=True)
+    except AssertionError:
+        return set()
+
+    result = set()
+    for cls_or_tup in evaluator.eval_element(classes):
+        if isinstance(cls_or_tup, iterable.Array) and cls_or_tup.type == 'tuple':
+            for typ in unite(cls_or_tup.py__iter__()):
+                result |= evaluator.execute(typ)
+        else:
+            result |= evaluator.execute(cls_or_tup)
+    return result
+
+
+def global_names_dict_generator(evaluator, scope, position):
+    """
+    For global name lookups. Yields tuples of (names_dict, position). If the
+    position is None, the position does not matter anymore in that scope.
+
+    This function is used to include names from outer scopes. For example, when
+    the current scope is function:
+
+    >>> from jedi._compatibility import u, no_unicode_pprint
+    >>> from jedi.parser import ParserWithRecovery, load_grammar
+    >>> parser = ParserWithRecovery(load_grammar(), u('''
+    ... x = ['a', 'b', 'c']
+    ... def func():
+    ...     y = None
+    ... '''))
+    >>> scope = parser.module.subscopes[0]
+    >>> scope
+    <Function: func@3-5>
+
+    `global_names_dict_generator` is a generator.  First it yields names from
+    most inner scope.
+
+    >>> from jedi.evaluate import Evaluator
+    >>> evaluator = Evaluator(load_grammar())
+    >>> scope = evaluator.wrap(scope)
+    >>> pairs = list(global_names_dict_generator(evaluator, scope, (4, 0)))
+    >>> no_unicode_pprint(pairs[0])
+    ({'func': [], 'y': [<Name: y@4,4>]}, (4, 0))
+
+    Then it yields the names from one level "lower". In this example, this
+    is the most outer scope. As you can see, the position in the tuple is now
+    None, because typically the whole module is loaded before the function is
+    called.
+
+    >>> no_unicode_pprint(pairs[1])
+    ({'func': [<Name: func@3,4>], 'x': [<Name: x@2,0>]}, None)
+
+    After that we have a few underscore names that are part of the module.
+
+    >>> sorted(pairs[2][0].keys())
+    ['__doc__', '__file__', '__name__', '__package__']
+    >>> pairs[3]  # global names -> there are none in our example.
+    ({}, None)
+    >>> pairs[4]  # package modules -> Also none.
+    ({}, None)
+
+    Finally, it yields names from builtin, if `include_builtin` is
+    true (default).
+
+    >>> pairs[5][0].values()                              #doctest: +ELLIPSIS
+    [[<CompiledName: ...>], ...]
+    """
+    in_func = False
+    while scope is not None:
+        if not (scope.type == 'classdef' and in_func):
+            # Names in methods cannot be resolved within the class.
+
+            for names_dict in scope.names_dicts(True):
+                yield names_dict, position
+                if hasattr(scope, 'resets_positions'):
+                    # TODO This is so ugly, seriously. However there's
+                    #      currently no good way of influencing
+                    #      global_names_dict_generator when it comes to certain
+                    #      objects.
+                    position = None
+            if scope.type == 'funcdef':
+                # The position should be reset if the current scope is a function.
+                in_func = True
+                position = None
+        scope = evaluator.wrap(scope.get_parent_scope())
+
+    # Add builtins to the global scope.
+    for names_dict in evaluator.BUILTINS.names_dicts(True):
+        yield names_dict, None
+
+
+def check_tuple_assignments(evaluator, types, name):
+    """
+    Checks if tuples are assigned.
+    """
+    for index, node in name.assignment_indexes():
+        iterated = iterable.py__iter__(evaluator, types, node)
+        for _ in range(index + 1):
+            try:
+                types = next(iterated)
+            except StopIteration:
+                # We could do this with the default param in next. But this
+                # would allow this loop to run for a very long time if the
+                # index number is high. Therefore break if the loop is
+                # finished.
+                types = set()
+                break
+    return types
+
+
+def filter_private_variable(scope, origin_node):
+    """Check if a variable is defined inside the same class or outside."""
+    instance = scope.get_parent_scope()
+    coming_from = origin_node
+    while coming_from is not None \
+            and not isinstance(coming_from, (tree.Class, compiled.CompiledObject)):
+        coming_from = coming_from.get_parent_scope()
+
+    # CompiledObjects don't have double underscore attributes, but Jedi abuses
+    # those for fakes (builtins.pym -> list).
+    if isinstance(instance, compiled.CompiledObject):
+        return instance != coming_from
+    else:
+        return isinstance(instance, er.Instance) and instance.base.base != coming_from

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/flow_analysis.py

@@ -0,0 +1,91 @@
+from jedi.parser import tree
+
+
+class Status(object):
+    lookup_table = {}
+
+    def __init__(self, value, name):
+        self._value = value
+        self._name = name
+        Status.lookup_table[value] = self
+
+    def invert(self):
+        if self is REACHABLE:
+            return UNREACHABLE
+        elif self is UNREACHABLE:
+            return REACHABLE
+        else:
+            return UNSURE
+
+    def __and__(self, other):
+        if UNSURE in (self, other):
+            return UNSURE
+        else:
+            return REACHABLE if self._value and other._value else UNREACHABLE
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, self._name)
+
+
+REACHABLE = Status(True, 'reachable')
+UNREACHABLE = Status(False, 'unreachable')
+UNSURE = Status(None, 'unsure')
+
+
+def break_check(evaluator, base_scope, stmt, origin_scope=None):
+    element_scope = evaluator.wrap(stmt.get_parent_scope(include_flows=True))
+    # Direct parents get resolved, we filter scopes that are separate branches.
+    # This makes sense for autocompletion and static analysis. For actual
+    # Python it doesn't matter, because we're talking about potentially
+    # unreachable code.
+    # e.g. `if 0:` would cause all name lookup within the flow make
+    # unaccessible. This is not a "problem" in Python, because the code is
+    # never called. In Jedi though, we still want to infer types.
+    while origin_scope is not None:
+        if element_scope == origin_scope:
+            return REACHABLE
+        origin_scope = origin_scope.parent
+    x = _break_check(evaluator, stmt, base_scope, element_scope)
+    return x
+
+
+def _break_check(evaluator, stmt, base_scope, element_scope):
+    element_scope = evaluator.wrap(element_scope)
+    base_scope = evaluator.wrap(base_scope)
+
+    reachable = REACHABLE
+    if isinstance(element_scope, tree.IfStmt):
+        if element_scope.node_after_else(stmt):
+            for check_node in element_scope.check_nodes():
+                reachable = _check_if(evaluator, check_node)
+                if reachable in (REACHABLE, UNSURE):
+                    break
+            reachable = reachable.invert()
+        else:
+            node = element_scope.node_in_which_check_node(stmt)
+            if node is not None:
+                reachable = _check_if(evaluator, node)
+    elif isinstance(element_scope, (tree.TryStmt, tree.WhileStmt)):
+        return UNSURE
+
+    # Only reachable branches need to be examined further.
+    if reachable in (UNREACHABLE, UNSURE):
+        return reachable
+
+    if element_scope.type == 'file_input':
+        # The definition is in another module and therefore just return what we
+        # have generated.
+        return reachable
+    if base_scope != element_scope and base_scope != element_scope.parent:
+        return reachable & _break_check(evaluator, stmt, base_scope, element_scope.parent)
+    else:
+        return reachable
+
+
+def _check_if(evaluator, node):
+    types = evaluator.eval_element(node)
+    values = set(x.py__bool__() for x in types)
+    if len(values) == 1:
+        return Status.lookup_table[values.pop()]
+    else:
+        return UNSURE

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/helpers.py

@@ -0,0 +1,200 @@
+import copy
+from itertools import chain
+
+from jedi.parser import tree
+
+
+def deep_ast_copy(obj, parent=None, new_elements=None):
+    """
+    Much, much faster than copy.deepcopy, but just for Parser elements (Doesn't
+    copy parents).
+    """
+
+    if new_elements is None:
+        new_elements = {}
+
+    def copy_node(obj):
+        # If it's already in the cache, just return it.
+        try:
+            return new_elements[obj]
+        except KeyError:
+            # Actually copy and set attributes.
+            new_obj = copy.copy(obj)
+            new_elements[obj] = new_obj
+
+        # Copy children
+        new_children = []
+        for child in obj.children:
+            typ = child.type
+            if typ in ('newline', 'operator', 'keyword', 'number', 'string',
+                       'indent', 'dedent', 'endmarker', 'error_leaf'):
+                # At the moment we're not actually copying those primitive
+                # elements, because there's really no need to. The parents are
+                # obviously wrong, but that's not an issue.
+                new_child = child
+            elif typ == 'name':
+                new_elements[child] = new_child = copy.copy(child)
+                new_child.parent = new_obj
+            else:  # Is a BaseNode.
+                new_child = copy_node(child)
+                new_child.parent = new_obj
+            new_children.append(new_child)
+        new_obj.children = new_children
+
+        # Copy the names_dict (if there is one).
+        try:
+            names_dict = obj.names_dict
+        except AttributeError:
+            pass
+        else:
+            try:
+                new_obj.names_dict = new_names_dict = {}
+            except AttributeError:  # Impossible to set CompFor.names_dict
+                pass
+            else:
+                for string, names in names_dict.items():
+                    new_names_dict[string] = [new_elements[n] for n in names]
+        return new_obj
+
+    if isinstance(obj, tree.BaseNode):
+        new_obj = copy_node(obj)
+    else:
+        # Special case of a Name object.
+        new_elements[obj] = new_obj = copy.copy(obj)
+
+    if parent is not None:
+        new_obj.parent = parent
+    return new_obj
+
+
+def call_of_leaf(leaf, cut_own_trailer=False):
+    """
+    Creates a "call" node that consist of all ``trailer`` and ``power``
+    objects.  E.g. if you call it with ``append``::
+
+        list([]).append(3) or None
+
+    You would get a node with the content ``list([]).append`` back.
+
+    This generates a copy of the original ast node.
+
+    If you're using the leaf, e.g. the bracket `)` it will return ``list([])``.
+
+    # TODO remove cut_own_trailer option, since its always used with it. Just
+    #      ignore it, It's not what we want anyway. Or document it better?
+    """
+    trailer = leaf.parent
+    # The leaf may not be the last or first child, because there exist three
+    # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples
+    # we should not match anything more than x.
+    if trailer.type != 'trailer' or leaf not in (trailer.children[0], trailer.children[-1]):
+        if trailer.type == 'atom':
+            return trailer
+        return leaf
+
+    power = trailer.parent
+    index = power.children.index(trailer)
+    power = deep_ast_copy(power)
+    if cut_own_trailer:
+        cut = index
+    else:
+        cut = index + 1
+    power.children[cut:] = []
+
+    if power.type == 'error_node':
+        start = index
+        while True:
+            start -= 1
+            if power.children[start].type != 'trailer':
+                break
+        transformed = tree.Node('power', power.children[start:])
+        transformed.parent = power.parent
+        return transformed
+
+    return power
+
+
+def get_names_of_node(node):
+    try:
+        children = node.children
+    except AttributeError:
+        if node.type == 'name':
+            return [node]
+        else:
+            return []
+    else:
+        return list(chain.from_iterable(get_names_of_node(c) for c in children))
+
+
+def get_module_names(module, all_scopes):
+    """
+    Returns a dictionary with name parts as keys and their call paths as
+    values.
+    """
+    if all_scopes:
+        dct = module.used_names
+    else:
+        dct = module.names_dict
+    return chain.from_iterable(dct.values())
+
+
+class FakeImport(tree.ImportName):
+    def __init__(self, name, parent, level=0):
+        super(FakeImport, self).__init__([])
+        self.parent = parent
+        self._level = level
+        self.name = name
+
+    def get_defined_names(self):
+        return [self.name]
+
+    def aliases(self):
+        return {}
+
+    @property
+    def level(self):
+        return self._level
+
+    @property
+    def start_pos(self):
+        return 0, 0
+
+    def paths(self):
+        return [[self.name]]
+
+    def is_definition(self):
+        return True
+
+
+class FakeName(tree.Name):
+    def __init__(self, name_str, parent=None, start_pos=(0, 0), is_definition=None):
+        """
+        In case is_definition is defined (not None), that bool value will be
+        returned.
+        """
+        super(FakeName, self).__init__(tree.zero_position_modifier, name_str, start_pos)
+        self.parent = parent
+        self._is_definition = is_definition
+
+    def get_definition(self):
+        return self.parent
+
+    def is_definition(self):
+        if self._is_definition is None:
+            return super(FakeName, self).is_definition()
+        else:
+            return self._is_definition
+
+
+class LazyName(FakeName):
+    def __init__(self, name, parent_callback, is_definition=None):
+        super(LazyName, self).__init__(name, is_definition=is_definition)
+        self._parent_callback = parent_callback
+
+    @property
+    def parent(self):
+        return self._parent_callback()
+
+    @parent.setter
+    def parent(self, value):
+        pass  # Do nothing, super classes can try to set the parent.

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/imports.py

@@ -0,0 +1,517 @@
+"""
+:mod:`jedi.evaluate.imports` is here to resolve import statements and return
+the modules/classes/functions/whatever, which they stand for. However there's
+not any actual importing done. This module is about finding modules in the
+filesystem. This can be quite tricky sometimes, because Python imports are not
+always that simple.
+
+This module uses imp for python up to 3.2 and importlib for python 3.3 on; the
+correct implementation is delegated to _compatibility.
+
+This module also supports import autocompletion, which means to complete
+statements like ``from datetim`` (curser at the end would return ``datetime``).
+"""
+import imp
+import os
+import pkgutil
+import sys
+from itertools import chain
+
+from jedi._compatibility import find_module, unicode
+from jedi import common
+from jedi import debug
+from jedi.parser import fast
+from jedi.parser import tree
+from jedi.parser.utils import save_parser, load_parser, parser_cache
+from jedi.evaluate import sys_path
+from jedi.evaluate import helpers
+from jedi import settings
+from jedi.common import source_to_unicode
+from jedi.evaluate import compiled
+from jedi.evaluate import analysis
+from jedi.evaluate.cache import memoize_default, NO_DEFAULT
+
+
+def completion_names(evaluator, imp, pos):
+    name = imp.name_for_position(pos)
+    module = evaluator.wrap(imp.get_parent_until())
+    if name is None:
+        level = 0
+        for node in imp.children:
+            if node.end_pos <= pos:
+                if node in ('.', '...'):
+                    level += len(node.value)
+        import_path = []
+    else:
+        # Completion on an existing name.
+
+        # The import path needs to be reduced by one, because we're completing.
+        import_path = imp.path_for_name(name)[:-1]
+        level = imp.level
+
+    importer = Importer(evaluator, tuple(import_path), module, level)
+    if isinstance(imp, tree.ImportFrom):
+        c = imp.children
+        only_modules = c[c.index('import')].start_pos >= pos
+    else:
+        only_modules = True
+    return importer.completion_names(evaluator, only_modules)
+
+
+class ImportWrapper(tree.Base):
+    def __init__(self, evaluator, name):
+        self._evaluator = evaluator
+        self._name = name
+
+        self._import = name.get_parent_until(tree.Import)
+        self.import_path = self._import.path_for_name(name)
+
+    @memoize_default()
+    def follow(self, is_goto=False):
+        module = self._evaluator.wrap(self._import.get_parent_until())
+        import_path = self._import.path_for_name(self._name)
+        from_import_name = None
+        try:
+            from_names = self._import.get_from_names()
+        except AttributeError:
+            # Is an import_name
+            pass
+        else:
+            if len(from_names) + 1 == len(import_path):
+                # We have to fetch the from_names part first and then check
+                # if from_names exists in the modules.
+                from_import_name = import_path[-1]
+                import_path = from_names
+
+        importer = Importer(self._evaluator, tuple(import_path),
+                            module, self._import.level)
+
+        types = importer.follow()
+
+        #if self._import.is_nested() and not self.nested_resolve:
+        #    scopes = [NestedImportModule(module, self._import)]
+
+        if from_import_name is not None:
+            types = set(chain.from_iterable(
+                self._evaluator.find_types(t, unicode(from_import_name),
+                                           is_goto=is_goto)
+                for t in types))
+
+            if not types:
+                path = import_path + [from_import_name]
+                importer = Importer(self._evaluator, tuple(path),
+                                    module, self._import.level)
+                types = importer.follow()
+                # goto only accepts `Name`
+                if is_goto:
+                    types = set(s.name for s in types)
+        else:
+            # goto only accepts `Name`
+            if is_goto:
+                types = set(s.name for s in types)
+
+        debug.dbg('after import: %s', types)
+        return types
+
+
+class NestedImportModule(tree.Module):
+    """
+    TODO while there's no use case for nested import module right now, we might
+        be able to use them for static analysis checks later on.
+    """
+    def __init__(self, module, nested_import):
+        self._module = module
+        self._nested_import = nested_import
+
+    def _get_nested_import_name(self):
+        """
+        Generates an Import statement, that can be used to fake nested imports.
+        """
+        i = self._nested_import
+        # This is not an existing Import statement. Therefore, set position to
+        # 0 (0 is not a valid line number).
+        zero = (0, 0)
+        names = [unicode(name) for name in i.namespace_names[1:]]
+        name = helpers.FakeName(names, self._nested_import)
+        new = tree.Import(i._sub_module, zero, zero, name)
+        new.parent = self._module
+        debug.dbg('Generated a nested import: %s', new)
+        return helpers.FakeName(str(i.namespace_names[1]), new)
+
+    def __getattr__(self, name):
+        return getattr(self._module, name)
+
+    def __repr__(self):
+        return "<%s: %s of %s>" % (self.__class__.__name__, self._module,
+                                   self._nested_import)
+
+
+def _add_error(evaluator, name, message=None):
+    if hasattr(name, 'parent'):
+        # Should be a name, not a string!
+        analysis.add(evaluator, 'import-error', name, message)
+
+
+def get_init_path(directory_path):
+    """
+    The __init__ file can be searched in a directory. If found return it, else
+    None.
+    """
+    for suffix, _, _ in imp.get_suffixes():
+        path = os.path.join(directory_path, '__init__' + suffix)
+        if os.path.exists(path):
+            return path
+    return None
+
+
+class Importer(object):
+    def __init__(self, evaluator, import_path, module, level=0):
+        """
+        An implementation similar to ``__import__``. Use `follow`
+        to actually follow the imports.
+
+        *level* specifies whether to use absolute or relative imports. 0 (the
+        default) means only perform absolute imports. Positive values for level
+        indicate the number of parent directories to search relative to the
+        directory of the module calling ``__import__()`` (see PEP 328 for the
+        details).
+
+        :param import_path: List of namespaces (strings or Names).
+        """
+        debug.speed('import %s' % (import_path,))
+        self._evaluator = evaluator
+        self.level = level
+        self.module = module
+        try:
+            self.file_path = module.py__file__()
+        except AttributeError:
+            # Can be None for certain compiled modules like 'builtins'.
+            self.file_path = None
+
+        if level:
+            base = module.py__package__().split('.')
+            if base == ['']:
+                base = []
+            if level > len(base):
+                path = module.py__file__()
+                if path is not None:
+                    import_path = list(import_path)
+                    for i in range(level):
+                        path = os.path.dirname(path)
+                    dir_name = os.path.basename(path)
+                    # This is not the proper way to do relative imports. However, since
+                    # Jedi cannot be sure about the entry point, we just calculate an
+                    # absolute path here.
+                    if dir_name:
+                        import_path.insert(0, dir_name)
+                    else:
+                        _add_error(self._evaluator, import_path[-1])
+                        import_path = []
+                        # TODO add import error.
+                        debug.warning('Attempted relative import beyond top-level package.')
+            else:
+                # Here we basically rewrite the level to 0.
+                import_path = tuple(base) + tuple(import_path)
+        self.import_path = import_path
+
+    @property
+    def str_import_path(self):
+        """Returns the import path as pure strings instead of `Name`."""
+        return tuple(str(name) for name in self.import_path)
+
+    @memoize_default()
+    def sys_path_with_modifications(self):
+        in_path = []
+        sys_path_mod = list(sys_path.sys_path_with_modifications(self._evaluator, self.module))
+        if self.file_path is not None:
+            # If you edit e.g. gunicorn, there will be imports like this:
+            # `from gunicorn import something`. But gunicorn is not in the
+            # sys.path. Therefore look if gunicorn is a parent directory, #56.
+            if self.import_path:  # TODO is this check really needed?
+                for path in sys_path.traverse_parents(self.file_path):
+                    if os.path.basename(path) == self.str_import_path[0]:
+                        in_path.append(os.path.dirname(path))
+
+            # Since we know nothing about the call location of the sys.path,
+            # it's a possibility that the current directory is the origin of
+            # the Python execution.
+            sys_path_mod.insert(0, os.path.dirname(self.file_path))
+
+        return in_path + sys_path_mod
+
+    @memoize_default(NO_DEFAULT)
+    def follow(self):
+        if not self.import_path:
+            return set()
+        return self._do_import(self.import_path, self.sys_path_with_modifications())
+
+    def _do_import(self, import_path, sys_path):
+        """
+        This method is very similar to importlib's `_gcd_import`.
+        """
+        import_parts = [str(i) for i in import_path]
+
+        # Handle "magic" Flask extension imports:
+        # ``flask.ext.foo`` is really ``flask_foo`` or ``flaskext.foo``.
+        if len(import_path) > 2 and import_parts[:2] == ['flask', 'ext']:
+            # New style.
+            ipath = ('flask_' + str(import_parts[2]),) + import_path[3:]
+            modules = self._do_import(ipath, sys_path)
+            if modules:
+                return modules
+            else:
+                # Old style
+                return self._do_import(('flaskext',) + import_path[2:], sys_path)
+
+        module_name = '.'.join(import_parts)
+        try:
+            return set([self._evaluator.modules[module_name]])
+        except KeyError:
+            pass
+
+        if len(import_path) > 1:
+            # This is a recursive way of importing that works great with
+            # the module cache.
+            bases = self._do_import(import_path[:-1], sys_path)
+            if not bases:
+                return set()
+            # We can take the first element, because only the os special
+            # case yields multiple modules, which is not important for
+            # further imports.
+            parent_module = list(bases)[0]
+
+            # This is a huge exception, we follow a nested import
+            # ``os.path``, because it's a very important one in Python
+            # that is being achieved by messing with ``sys.modules`` in
+            # ``os``.
+            if [str(i) for i in import_path] == ['os', 'path']:
+                return self._evaluator.find_types(parent_module, 'path')
+
+            try:
+                paths = parent_module.py__path__()
+            except AttributeError:
+                # The module is not a package.
+                _add_error(self._evaluator, import_path[-1])
+                return set()
+            else:
+                debug.dbg('search_module %s in paths %s', module_name, paths)
+                for path in paths:
+                    # At the moment we are only using one path. So this is
+                    # not important to be correct.
+                    try:
+                        module_file, module_path, is_pkg = \
+                            find_module(import_parts[-1], [path])
+                        break
+                    except ImportError:
+                        module_path = None
+                if module_path is None:
+                    _add_error(self._evaluator, import_path[-1])
+                    return set()
+        else:
+            parent_module = None
+            try:
+                debug.dbg('search_module %s in %s', import_parts[-1], self.file_path)
+                # Override the sys.path. It works only good that way.
+                # Injecting the path directly into `find_module` did not work.
+                sys.path, temp = sys_path, sys.path
+                try:
+                    module_file, module_path, is_pkg = \
+                        find_module(import_parts[-1])
+                finally:
+                    sys.path = temp
+            except ImportError:
+                # The module is not a package.
+                _add_error(self._evaluator, import_path[-1])
+                return set()
+
+        source = None
+        if is_pkg:
+            # In this case, we don't have a file yet. Search for the
+            # __init__ file.
+            if module_path.endswith(('.zip', '.egg')):
+                source = module_file.loader.get_source(module_name)
+            else:
+                module_path = get_init_path(module_path)
+        elif module_file:
+            source = module_file.read()
+            module_file.close()
+
+        if module_file is None and not module_path.endswith(('.py', '.zip', '.egg')):
+            module = compiled.load_module(self._evaluator, module_path)
+        else:
+            module = _load_module(self._evaluator, module_path, source, sys_path, parent_module)
+
+        if module is None:
+            # The file might raise an ImportError e.g. and therefore not be
+            # importable.
+            return set()
+
+        self._evaluator.modules[module_name] = module
+        return set([module])
+
+    def _generate_name(self, name):
+        # Create a pseudo import to be able to follow them.
+        name = helpers.FakeName(name)
+        imp = helpers.FakeImport(name, parent=self.module)
+        name.parent = imp
+        return name
+
+    def _get_module_names(self, search_path=None):
+        """
+        Get the names of all modules in the search_path. This means file names
+        and not names defined in the files.
+        """
+
+        names = []
+        # add builtin module names
+        if search_path is None:
+            names += [self._generate_name(name) for name in sys.builtin_module_names]
+
+        if search_path is None:
+            search_path = self.sys_path_with_modifications()
+        for module_loader, name, is_pkg in pkgutil.iter_modules(search_path):
+            names.append(self._generate_name(name))
+        return names
+
+    def completion_names(self, evaluator, only_modules=False):
+        """
+        :param only_modules: Indicates wheter it's possible to import a
+            definition that is not defined in a module.
+        """
+        from jedi.evaluate import finder
+        names = []
+        if self.import_path:
+            # flask
+            if self.str_import_path == ('flask', 'ext'):
+                # List Flask extensions like ``flask_foo``
+                for mod in self._get_module_names():
+                    modname = str(mod)
+                    if modname.startswith('flask_'):
+                        extname = modname[len('flask_'):]
+                        names.append(self._generate_name(extname))
+                # Now the old style: ``flaskext.foo``
+                for dir in self.sys_path_with_modifications():
+                    flaskext = os.path.join(dir, 'flaskext')
+                    if os.path.isdir(flaskext):
+                        names += self._get_module_names([flaskext])
+
+            for scope in self.follow():
+                # Non-modules are not completable.
+                if not scope.type == 'file_input':  # not a module
+                    continue
+
+                # namespace packages
+                if isinstance(scope, tree.Module) and scope.path.endswith('__init__.py'):
+                    paths = scope.py__path__()
+                    names += self._get_module_names(paths)
+
+                if only_modules:
+                    # In the case of an import like `from x.` we don't need to
+                    # add all the variables.
+                    if ('os',) == self.str_import_path and not self.level:
+                        # os.path is a hardcoded exception, because it's a
+                        # ``sys.modules`` modification.
+                        names.append(self._generate_name('path'))
+
+                    continue
+
+                for names_dict in scope.names_dicts(search_global=False):
+                    _names = list(chain.from_iterable(names_dict.values()))
+                    if not _names:
+                        continue
+                    _names = finder.filter_definition_names(_names, scope)
+                    names += _names
+        else:
+            # Empty import path=completion after import
+            if not self.level:
+                names += self._get_module_names()
+
+            if self.file_path is not None:
+                path = os.path.abspath(self.file_path)
+                for i in range(self.level - 1):
+                    path = os.path.dirname(path)
+                names += self._get_module_names([path])
+
+        return names
+
+
+def _load_module(evaluator, path=None, source=None, sys_path=None, parent_module=None):
+    def load(source):
+        dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
+        if path is not None and path.endswith(('.py', '.zip', '.egg')) \
+                and dotted_path not in settings.auto_import_modules:
+            if source is None:
+                with open(path, 'rb') as f:
+                    source = f.read()
+        else:
+            return compiled.load_module(evaluator, path)
+        p = path
+        p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
+        save_parser(path, p)
+        from jedi.evaluate.representation import ModuleWrapper
+        return ModuleWrapper(evaluator, p.module, parent_module)
+
+    if sys_path is None:
+        sys_path = evaluator.sys_path
+
+    cached = load_parser(path)
+    module = load(source) if cached is None else cached.module
+    module = evaluator.wrap(module)
+    return module
+
+
+def add_module(evaluator, module_name, module):
+    if '.' not in module_name:
+        # We cannot add paths with dots, because that would collide with
+        # the sepatator dots for nested packages. Therefore we return
+        # `__main__` in ModuleWrapper.py__name__(), which is similar to
+        # Python behavior.
+        evaluator.modules[module_name] = module
+
+
+def get_modules_containing_name(evaluator, mods, name):
+    """
+    Search a name in the directories of modules.
+    """
+    def check_python_file(path):
+        try:
+            return parser_cache[path].parser.module
+        except KeyError:
+            try:
+                return check_fs(path)
+            except IOError:
+                return None
+
+    def check_fs(path):
+        with open(path, 'rb') as f:
+            source = source_to_unicode(f.read())
+            if name in source:
+                module_name = os.path.basename(path)[:-3]  # Remove `.py`.
+                module = _load_module(evaluator, path, source)
+                add_module(evaluator, module_name, module)
+                return module
+
+    # skip non python modules
+    mods = set(m for m in mods if not isinstance(m, compiled.CompiledObject))
+    mod_paths = set()
+    for m in mods:
+        mod_paths.add(m.path)
+        yield m
+
+    if settings.dynamic_params_for_other_modules:
+        paths = set(settings.additional_dynamic_modules)
+        for p in mod_paths:
+            if p is not None:
+                # We need abspath, because the seetings paths might not already
+                # have been converted to absolute paths.
+                d = os.path.dirname(os.path.abspath(p))
+                for entry in os.listdir(d):
+                    if entry not in mod_paths:
+                        if entry.endswith('.py'):
+                            paths.add(d + os.path.sep + entry)
+
+        for p in sorted(paths):
+            # make testing easier, sort it - same results on every interpreter
+            c = check_python_file(p)
+            if c is not None and c not in mods and not isinstance(c, compiled.CompiledObject):
+                yield c

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/iterable.py

@@ -0,0 +1,863 @@
+"""
+Contains all classes and functions to deal with lists, dicts, generators and
+iterators in general.
+
+Array modifications
+*******************
+
+If the content of an array (``set``/``list``) is requested somewhere, the
+current module will be checked for appearances of ``arr.append``,
+``arr.insert``, etc.  If the ``arr`` name points to an actual array, the
+content will be added
+
+This can be really cpu intensive, as you can imagine. Because |jedi| has to
+follow **every** ``append`` and check wheter it's the right array. However this
+works pretty good, because in *slow* cases, the recursion detector and other
+settings will stop this process.
+
+It is important to note that:
+
+1. Array modfications work only in the current module.
+2. Jedi only checks Array additions; ``list.pop``, etc are ignored.
+"""
+from jedi.common import unite, safe_property
+from jedi import debug
+from jedi import settings
+from jedi._compatibility import use_metaclass, unicode, zip_longest
+from jedi.parser import tree
+from jedi.evaluate import compiled
+from jedi.evaluate import helpers
+from jedi.evaluate.cache import CachedMetaClass, memoize_default
+from jedi.evaluate import analysis
+from jedi.evaluate import pep0484
+from jedi import common
+
+
+class IterableWrapper(tree.Base):
+    def is_class(self):
+        return False
+
+    @memoize_default()
+    def _get_names_dict(self, names_dict):
+        builtin_methods = {}
+        for cls in reversed(type(self).mro()):
+            try:
+                builtin_methods.update(cls.builtin_methods)
+            except AttributeError:
+                pass
+
+        if not builtin_methods:
+            return names_dict
+
+        dct = {}
+        for names in names_dict.values():
+            for name in names:
+                name_str = name.value
+                try:
+                    method = builtin_methods[name_str, self.type]
+                except KeyError:
+                    dct[name_str] = [name]
+                else:
+                    parent = BuiltinMethod(self, method, name.parent)
+                    dct[name_str] = [helpers.FakeName(name_str, parent, is_definition=True)]
+        return dct
+
+
+class BuiltinMethod(IterableWrapper):
+    """``Generator.__next__`` ``dict.values`` methods and so on."""
+    def __init__(self, builtin, method, builtin_func):
+        self._builtin = builtin
+        self._method = method
+        self._builtin_func = builtin_func
+
+    def py__call__(self, params):
+        return self._method(self._builtin)
+
+    def __getattr__(self, name):
+        return getattr(self._builtin_func, name)
+
+
+def has_builtin_methods(cls):
+    cls.builtin_methods = {}
+    for func in cls.__dict__.values():
+        try:
+            cls.builtin_methods.update(func.registered_builtin_methods)
+        except AttributeError:
+            pass
+    return cls
+
+
+def register_builtin_method(method_name, type=None):
+    def wrapper(func):
+        dct = func.__dict__.setdefault('registered_builtin_methods', {})
+        dct[method_name, type] = func
+        return func
+    return wrapper
+
+
+@has_builtin_methods
+class GeneratorMixin(object):
+    type = None
+
+    @register_builtin_method('send')
+    @register_builtin_method('next')
+    @register_builtin_method('__next__')
+    def py__next__(self):
+        # TODO add TypeError if params are given.
+        return unite(self.py__iter__())
+
+    @memoize_default()
+    def names_dicts(self, search_global=False):  # is always False
+        gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT')
+        yield self._get_names_dict(gen_obj.names_dict)
+
+    def py__bool__(self):
+        return True
+
+    def py__class__(self):
+        gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT')
+        return gen_obj.py__class__()
+
+
+class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin)):
+    """Handling of `yield` functions."""
+
+    def __init__(self, evaluator, func, var_args):
+        super(Generator, self).__init__()
+        self._evaluator = evaluator
+        self.func = func
+        self.var_args = var_args
+
+    def py__iter__(self):
+        from jedi.evaluate.representation import FunctionExecution
+        f = FunctionExecution(self._evaluator, self.func, self.var_args)
+        return f.get_yield_types()
+
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'end_pos', 'parent', 'get_imports',
+                        'doc', 'docstr', 'get_parent_until',
+                        'get_code', 'subscopes']:
+            raise AttributeError("Accessing %s of %s is not allowed."
+                                 % (self, name))
+        return getattr(self.func, name)
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self.func)
+
+
+class Comprehension(IterableWrapper):
+    @staticmethod
+    def from_atom(evaluator, atom):
+        bracket = atom.children[0]
+        if bracket == '{':
+            if atom.children[1].children[1] == ':':
+                cls = DictComprehension
+            else:
+                cls = SetComprehension
+        elif bracket == '(':
+            cls = GeneratorComprehension
+        elif bracket == '[':
+            cls = ListComprehension
+        return cls(evaluator, atom)
+
+    def __init__(self, evaluator, atom):
+        self._evaluator = evaluator
+        self._atom = atom
+
+    def _get_comprehension(self):
+        # The atom contains a testlist_comp
+        return self._atom.children[1]
+
+    def _get_comp_for(self):
+        # The atom contains a testlist_comp
+        return self._get_comprehension().children[1]
+
+    @memoize_default()
+    def _eval_node(self, index=0):
+        """
+        The first part `x + 1` of the list comprehension:
+
+            [x + 1 for x in foo]
+        """
+        comp_for = self._get_comp_for()
+        # For nested comprehensions we need to search the last one.
+        from jedi.evaluate.representation import InstanceElement
+        node = self._get_comprehension().children[index]
+        if isinstance(node, InstanceElement):
+            # This seems to be a strange case that I haven't found a way to
+            # write tests against. However since it's my new goal to get rid of
+            # InstanceElement anyway, I don't care.
+            node = node.var
+        last_comp = list(comp_for.get_comp_fors())[-1]
+        return helpers.deep_ast_copy(node, parent=last_comp)
+
+    def _nested(self, comp_fors):
+        evaluator = self._evaluator
+        comp_for = comp_fors[0]
+        input_node = comp_for.children[3]
+        input_types = evaluator.eval_element(input_node)
+
+        iterated = py__iter__(evaluator, input_types, input_node)
+        exprlist = comp_for.children[1]
+        for i, types in enumerate(iterated):
+            evaluator.predefined_if_name_dict_dict[comp_for] = \
+                unpack_tuple_to_dict(evaluator, types, exprlist)
+            try:
+                for result in self._nested(comp_fors[1:]):
+                    yield result
+            except IndexError:
+                iterated = evaluator.eval_element(self._eval_node())
+                if self.type == 'dict':
+                    yield iterated, evaluator.eval_element(self._eval_node(2))
+                else:
+                    yield iterated
+            finally:
+                del evaluator.predefined_if_name_dict_dict[comp_for]
+
+    @memoize_default(default=[])
+    @common.to_list
+    def _iterate(self):
+        comp_fors = tuple(self._get_comp_for().get_comp_fors())
+        for result in self._nested(comp_fors):
+            yield result
+
+    def py__iter__(self):
+        return self._iterate()
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self._atom)
+
+
+@has_builtin_methods
+class ArrayMixin(object):
+    @memoize_default()
+    def names_dicts(self, search_global=False):  # Always False.
+        # `array.type` is a string with the type, e.g. 'list'.
+        scope = compiled.builtin_from_name(self._evaluator, self.type)
+        # builtins only have one class -> [0]
+        scopes = self._evaluator.execute_evaluated(scope, self)
+        names_dicts = list(scopes)[0].names_dicts(search_global)
+        #yield names_dicts[0]
+        yield self._get_names_dict(names_dicts[1])
+
+    def py__bool__(self):
+        return None  # We don't know the length, because of appends.
+
+    def py__class__(self):
+        return compiled.builtin_from_name(self._evaluator, self.type)
+
+    @safe_property
+    def parent(self):
+        return self._evaluator.BUILTINS
+
+    @property
+    def name(self):
+        return FakeSequence(self._evaluator, [], self.type).name
+
+    @memoize_default()
+    def dict_values(self):
+        return unite(self._evaluator.eval_element(v) for k, v in self._items())
+
+    @register_builtin_method('values', type='dict')
+    def _imitate_values(self):
+        items = self.dict_values()
+        return create_evaluated_sequence_set(self._evaluator, items, sequence_type='list')
+        #return set([FakeSequence(self._evaluator, [AlreadyEvaluated(items)], 'tuple')])
+
+    @register_builtin_method('items', type='dict')
+    def _imitate_items(self):
+        items = [set([FakeSequence(self._evaluator, (k, v), 'tuple')])
+                 for k, v in self._items()]
+
+        return create_evaluated_sequence_set(self._evaluator, *items, sequence_type='list')
+
+
+class ListComprehension(Comprehension, ArrayMixin):
+    type = 'list'
+
+    def py__getitem__(self, index):
+        all_types = list(self.py__iter__())
+        result = all_types[index]
+        if isinstance(index, slice):
+            return create_evaluated_sequence_set(
+                self._evaluator,
+                unite(result),
+                sequence_type='list'
+            )
+        return result
+
+
+class SetComprehension(Comprehension, ArrayMixin):
+    type = 'set'
+
+
+@has_builtin_methods
+class DictComprehension(Comprehension, ArrayMixin):
+    type = 'dict'
+
+    def _get_comp_for(self):
+        return self._get_comprehension().children[3]
+
+    def py__iter__(self):
+        for keys, values in self._iterate():
+            yield keys
+
+    def py__getitem__(self, index):
+        for keys, values in self._iterate():
+            for k in keys:
+                if isinstance(k, compiled.CompiledObject):
+                    if k.obj == index:
+                        return values
+        return self.dict_values()
+
+    def dict_values(self):
+        return unite(values for keys, values in self._iterate())
+
+    @register_builtin_method('items', type='dict')
+    def _imitate_items(self):
+        items = set(FakeSequence(self._evaluator,
+                    (AlreadyEvaluated(keys), AlreadyEvaluated(values)), 'tuple')
+                    for keys, values in self._iterate())
+
+        return create_evaluated_sequence_set(self._evaluator, items, sequence_type='list')
+
+
+class GeneratorComprehension(Comprehension, GeneratorMixin):
+    pass
+
+
+class Array(IterableWrapper, ArrayMixin):
+    mapping = {'(': 'tuple',
+               '[': 'list',
+               '{': 'dict'}
+
+    def __init__(self, evaluator, atom):
+        self._evaluator = evaluator
+        self.atom = atom
+        self.type = Array.mapping[atom.children[0]]
+        """The builtin name of the array (list, set, tuple or dict)."""
+
+        c = self.atom.children
+        array_node = c[1]
+        if self.type == 'dict' and array_node != '}' \
+                and (not hasattr(array_node, 'children')
+                     or ':' not in array_node.children):
+            self.type = 'set'
+
+    @property
+    def name(self):
+        return helpers.FakeName(self.type, parent=self)
+
+    def py__getitem__(self, index):
+        """Here the index is an int/str. Raises IndexError/KeyError."""
+        if self.type == 'dict':
+            for key, value in self._items():
+                for k in self._evaluator.eval_element(key):
+                    if isinstance(k, compiled.CompiledObject) \
+                            and index == k.obj:
+                        return self._evaluator.eval_element(value)
+            raise KeyError('No key found in dictionary %s.' % self)
+
+        # Can raise an IndexError
+        if isinstance(index, slice):
+            return set([self])
+        else:
+            return self._evaluator.eval_element(self._items()[index])
+
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'get_only_subelement', 'parent',
+                        'get_parent_until', 'items']:
+            raise AttributeError('Strange access on %s: %s.' % (self, name))
+        return getattr(self.atom, name)
+
+    # @memoize_default()
+    def py__iter__(self):
+        """
+        While values returns the possible values for any array field, this
+        function returns the value for a certain index.
+        """
+        if self.type == 'dict':
+            # Get keys.
+            types = set()
+            for k, _ in self._items():
+                types |= self._evaluator.eval_element(k)
+            # We don't know which dict index comes first, therefore always
+            # yield all the types.
+            for _ in types:
+                yield types
+        else:
+            for value in self._items():
+                yield self._evaluator.eval_element(value)
+
+            additions = check_array_additions(self._evaluator, self)
+            if additions:
+                yield additions
+
+    def _values(self):
+        """Returns a list of a list of node."""
+        if self.type == 'dict':
+            return unite(v for k, v in self._items())
+        else:
+            return self._items()
+
+    def _items(self):
+        c = self.atom.children
+        array_node = c[1]
+        if array_node in (']', '}', ')'):
+            return []  # Direct closing bracket, doesn't contain items.
+
+        if tree.is_node(array_node, 'testlist_comp'):
+            return array_node.children[::2]
+        elif tree.is_node(array_node, 'dictorsetmaker'):
+            kv = []
+            iterator = iter(array_node.children)
+            for key in iterator:
+                op = next(iterator, None)
+                if op is None or op == ',':
+                    kv.append(key)  # A set.
+                else:
+                    assert op == ':'  # A dict.
+                    kv.append((key, next(iterator)))
+                    next(iterator, None)  # Possible comma.
+            return kv
+        else:
+            return [array_node]
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self.atom)
+
+
+class _FakeArray(Array):
+    def __init__(self, evaluator, container, type):
+        self.type = type
+        self._evaluator = evaluator
+        self.atom = container
+
+
+class ImplicitTuple(_FakeArray):
+    def __init__(self, evaluator, testlist):
+        super(ImplicitTuple, self).__init__(evaluator, testlist, 'tuple')
+        self._testlist = testlist
+
+    def _items(self):
+        return self._testlist.children[::2]
+
+
+class FakeSequence(_FakeArray):
+    def __init__(self, evaluator, sequence_values, type):
+        """
+        type should be one of "tuple", "list"
+        """
+        super(FakeSequence, self).__init__(evaluator, sequence_values, type)
+        self._sequence_values = sequence_values
+
+    def _items(self):
+        return self._sequence_values
+
+
+def create_evaluated_sequence_set(evaluator, *types_order, **kwargs):
+    """
+    ``sequence_type`` is a named argument, that doesn't work in Python2. For backwards
+    compatibility reasons, we're now using kwargs.
+    """
+    sequence_type = kwargs.pop('sequence_type')
+    assert not kwargs
+
+    sets = tuple(AlreadyEvaluated(types) for types in types_order)
+    return set([FakeSequence(evaluator, sets, sequence_type)])
+
+
+class AlreadyEvaluated(frozenset):
+    """A simple container to add already evaluated objects to an array."""
+    def get_code(self, normalized=False):
+        # For debugging purposes.
+        return str(self)
+
+
+class MergedNodes(frozenset):
+    pass
+
+
+class FakeDict(_FakeArray):
+    def __init__(self, evaluator, dct):
+        super(FakeDict, self).__init__(evaluator, dct, 'dict')
+        self._dct = dct
+
+    def py__iter__(self):
+        yield set(compiled.create(self._evaluator, key) for key in self._dct)
+
+    def py__getitem__(self, index):
+        return unite(self._evaluator.eval_element(v) for v in self._dct[index])
+
+    def _items(self):
+        for key, values in self._dct.items():
+            # TODO this is not proper. The values could be multiple values?!
+            yield key, values[0]
+
+
+class MergedArray(_FakeArray):
+    def __init__(self, evaluator, arrays):
+        super(MergedArray, self).__init__(evaluator, arrays, arrays[-1].type)
+        self._arrays = arrays
+
+    def py__iter__(self):
+        for array in self._arrays:
+            for types in array.py__iter__():
+                yield types
+
+    def py__getitem__(self, index):
+        return unite(self.py__iter__())
+
+    def _items(self):
+        for array in self._arrays:
+            for a in array._items():
+                yield a
+
+    def __len__(self):
+        return sum(len(a) for a in self._arrays)
+
+
+def unpack_tuple_to_dict(evaluator, types, exprlist):
+    """
+    Unpacking tuple assignments in for statements and expr_stmts.
+    """
+    if exprlist.type == 'name':
+        return {exprlist.value: types}
+    elif exprlist.type == 'atom' and exprlist.children[0] in '([':
+        return unpack_tuple_to_dict(evaluator, types, exprlist.children[1])
+    elif exprlist.type in ('testlist', 'testlist_comp', 'exprlist',
+                           'testlist_star_expr'):
+        dct = {}
+        parts = iter(exprlist.children[::2])
+        n = 0
+        for iter_types in py__iter__(evaluator, types, exprlist):
+            n += 1
+            try:
+                part = next(parts)
+            except StopIteration:
+                analysis.add(evaluator, 'value-error-too-many-values', part,
+                             message="ValueError: too many values to unpack (expected %s)" % n)
+            else:
+                dct.update(unpack_tuple_to_dict(evaluator, iter_types, part))
+        has_parts = next(parts, None)
+        if types and has_parts is not None:
+            analysis.add(evaluator, 'value-error-too-few-values', has_parts,
+                         message="ValueError: need more than %s values to unpack" % n)
+        return dct
+    elif exprlist.type == 'power' or exprlist.type == 'atom_expr':
+        # Something like ``arr[x], var = ...``.
+        # This is something that is not yet supported, would also be difficult
+        # to write into a dict.
+        return {}
+    elif exprlist.type == 'star_expr':  # `a, *b, c = x` type unpackings
+        # Currently we're not supporting them.
+        return {}
+    raise NotImplementedError
+
+
+def py__iter__(evaluator, types, node=None):
+    debug.dbg('py__iter__')
+    type_iters = []
+    for typ in types:
+        try:
+            iter_method = typ.py__iter__
+        except AttributeError:
+            if node is not None:
+                analysis.add(evaluator, 'type-error-not-iterable', node,
+                             message="TypeError: '%s' object is not iterable" % typ)
+        else:
+            type_iters.append(iter_method())
+            #for result in iter_method():
+                #yield result
+
+    for t in zip_longest(*type_iters, fillvalue=set()):
+        yield unite(t)
+
+
+def py__iter__types(evaluator, types, node=None):
+    """
+    Calls `py__iter__`, but ignores the ordering in the end and just returns
+    all types that it contains.
+    """
+    return unite(py__iter__(evaluator, types, node))
+
+
+def py__getitem__(evaluator, types, trailer):
+    from jedi.evaluate.representation import Class
+    result = set()
+
+    trailer_op, node, trailer_cl = trailer.children
+    assert trailer_op == "["
+    assert trailer_cl == "]"
+
+    # special case: PEP0484 typing module, see
+    # https://github.com/davidhalter/jedi/issues/663
+    for typ in list(types):
+        if isinstance(typ, Class):
+            typing_module_types = \
+                pep0484.get_types_for_typing_module(evaluator, typ, node)
+            if typing_module_types is not None:
+                types.remove(typ)
+                result |= typing_module_types
+
+    if not types:
+        # all consumed by special cases
+        return result
+
+    for index in create_index_types(evaluator, node):
+        if isinstance(index, (compiled.CompiledObject, Slice)):
+            index = index.obj
+
+        if type(index) not in (float, int, str, unicode, slice):
+            # If the index is not clearly defined, we have to get all the
+            # possiblities.
+            for typ in list(types):
+                if isinstance(typ, Array) and typ.type == 'dict':
+                    types.remove(typ)
+                    result |= typ.dict_values()
+            return result | py__iter__types(evaluator, types)
+
+        for typ in types:
+            # The actual getitem call.
+            try:
+                getitem = typ.py__getitem__
+            except AttributeError:
+                analysis.add(evaluator, 'type-error-not-subscriptable', trailer_op,
+                             message="TypeError: '%s' object is not subscriptable" % typ)
+            else:
+                try:
+                    result |= getitem(index)
+                except IndexError:
+                    result |= py__iter__types(evaluator, set([typ]))
+                except KeyError:
+                    # Must be a dict. Lists don't raise KeyErrors.
+                    result |= typ.dict_values()
+    return result
+
+
+def check_array_additions(evaluator, array):
+    """ Just a mapper function for the internal _check_array_additions """
+    if array.type not in ('list', 'set'):
+        # TODO also check for dict updates
+        return set()
+
+    is_list = array.type == 'list'
+    try:
+        current_module = array.atom.get_parent_until()
+    except AttributeError:
+        # If there's no get_parent_until, it's a FakeSequence or another Fake
+        # type. Those fake types are used inside Jedi's engine. No values may
+        # be added to those after their creation.
+        return set()
+    return _check_array_additions(evaluator, array, current_module, is_list)
+
+
+@memoize_default(default=set(), evaluator_is_first_arg=True)
+@debug.increase_indent
+def _check_array_additions(evaluator, compare_array, module, is_list):
+    """
+    Checks if a `Array` has "add" (append, insert, extend) statements:
+
+    >>> a = [""]
+    >>> a.append(1)
+    """
+    debug.dbg('Dynamic array search for %s' % compare_array, color='MAGENTA')
+    if not settings.dynamic_array_additions or isinstance(module, compiled.CompiledObject):
+        debug.dbg('Dynamic array search aborted.', color='MAGENTA')
+        return set()
+
+    def check_additions(arglist, add_name):
+        params = list(param.Arguments(evaluator, arglist).unpack())
+        result = set()
+        if add_name in ['insert']:
+            params = params[1:]
+        if add_name in ['append', 'add', 'insert']:
+            for key, nodes in params:
+                result |= unite(evaluator.eval_element(node) for node in nodes)
+        elif add_name in ['extend', 'update']:
+            for key, nodes in params:
+                for node in nodes:
+                    types = evaluator.eval_element(node)
+                    result |= py__iter__types(evaluator, types, node)
+        return result
+
+    from jedi.evaluate import representation as er, param
+
+    def get_execution_parent(element):
+        """ Used to get an Instance/FunctionExecution parent """
+        if isinstance(element, Array):
+            node = element.atom
+        else:
+            # Is an Instance with an
+            # Arguments([AlreadyEvaluated([_ArrayInstance])]) inside
+            # Yeah... I know... It's complicated ;-)
+            node = list(element.var_args.argument_node[0])[0].var_args.trailer
+        if isinstance(node, er.InstanceElement) or node is None:
+            return node
+        return node.get_parent_until(er.FunctionExecution)
+
+    temp_param_add, settings.dynamic_params_for_other_modules = \
+        settings.dynamic_params_for_other_modules, False
+
+    search_names = ['append', 'extend', 'insert'] if is_list else ['add', 'update']
+    comp_arr_parent = get_execution_parent(compare_array)
+
+    added_types = set()
+    for add_name in search_names:
+        try:
+            possible_names = module.used_names[add_name]
+        except KeyError:
+            continue
+        else:
+            for name in possible_names:
+                # Check if the original scope is an execution. If it is, one
+                # can search for the same statement, that is in the module
+                # dict. Executions are somewhat special in jedi, since they
+                # literally copy the contents of a function.
+                if isinstance(comp_arr_parent, er.FunctionExecution):
+                    if comp_arr_parent.start_pos < name.start_pos < comp_arr_parent.end_pos:
+                        name = comp_arr_parent.name_for_position(name.start_pos)
+                    else:
+                        # Don't check definitions that are not defined in the
+                        # same function. This is not "proper" anyway. It also
+                        # improves Jedi's speed for array lookups, since we
+                        # don't have to check the whole source tree anymore.
+                        continue
+                trailer = name.parent
+                power = trailer.parent
+                trailer_pos = power.children.index(trailer)
+                try:
+                    execution_trailer = power.children[trailer_pos + 1]
+                except IndexError:
+                    continue
+                else:
+                    if execution_trailer.type != 'trailer' \
+                            or execution_trailer.children[0] != '(' \
+                            or execution_trailer.children[1] == ')':
+                        continue
+                power = helpers.call_of_leaf(name, cut_own_trailer=True)
+                # InstanceElements are special, because they don't get copied,
+                # but have this wrapper around them.
+                if isinstance(comp_arr_parent, er.InstanceElement):
+                    power = er.get_instance_el(evaluator, comp_arr_parent.instance, power)
+
+                if evaluator.recursion_detector.push_stmt(power):
+                    # Check for recursion. Possible by using 'extend' in
+                    # combination with function calls.
+                    continue
+                try:
+                    if compare_array in evaluator.eval_element(power):
+                        # The arrays match. Now add the results
+                        added_types |= check_additions(execution_trailer.children[1], add_name)
+                finally:
+                    evaluator.recursion_detector.pop_stmt()
+    # reset settings
+    settings.dynamic_params_for_other_modules = temp_param_add
+    debug.dbg('Dynamic array result %s' % added_types, color='MAGENTA')
+    return added_types
+
+
+def check_array_instances(evaluator, instance):
+    """Used for set() and list() instances."""
+    if not settings.dynamic_array_additions:
+        return instance.var_args
+
+    ai = _ArrayInstance(evaluator, instance)
+    from jedi.evaluate import param
+    return param.Arguments(evaluator, [AlreadyEvaluated([ai])])
+
+
+class _ArrayInstance(IterableWrapper):
+    """
+    Used for the usage of set() and list().
+    This is definitely a hack, but a good one :-)
+    It makes it possible to use set/list conversions.
+
+    In contrast to Array, ListComprehension and all other iterable types, this
+    is something that is only used inside `evaluate/compiled/fake/builtins.py`
+    and therefore doesn't need `names_dicts`, `py__bool__` and so on, because
+    we don't use these operations in `builtins.py`.
+    """
+    def __init__(self, evaluator, instance):
+        self._evaluator = evaluator
+        self.instance = instance
+        self.var_args = instance.var_args
+
+    def py__iter__(self):
+        try:
+            _, first_nodes = next(self.var_args.unpack())
+        except StopIteration:
+            types = set()
+        else:
+            types = unite(self._evaluator.eval_element(node) for node in first_nodes)
+            for types in py__iter__(self._evaluator, types, first_nodes[0]):
+                yield types
+
+        module = self.var_args.get_parent_until()
+        if module is None:
+            return
+        is_list = str(self.instance.name) == 'list'
+        additions = _check_array_additions(self._evaluator, self.instance, module, is_list)
+        if additions:
+            yield additions
+
+
+class Slice(object):
+    def __init__(self, evaluator, start, stop, step):
+        self._evaluator = evaluator
+        # all of them are either a Precedence or None.
+        self._start = start
+        self._stop = stop
+        self._step = step
+
+    @property
+    def obj(self):
+        """
+        Imitate CompiledObject.obj behavior and return a ``builtin.slice()``
+        object.
+        """
+        def get(element):
+            if element is None:
+                return None
+
+            result = self._evaluator.eval_element(element)
+            if len(result) != 1:
+                # For simplicity, we want slices to be clear defined with just
+                # one type.  Otherwise we will return an empty slice object.
+                raise IndexError
+            try:
+                return list(result)[0].obj
+            except AttributeError:
+                return None
+
+        try:
+            return slice(get(self._start), get(self._stop), get(self._step))
+        except IndexError:
+            return slice(None, None, None)
+
+
+def create_index_types(evaluator, index):
+    """
+    Handles slices in subscript nodes.
+    """
+    if index == ':':
+        # Like array[:]
+        return set([Slice(evaluator, None, None, None)])
+    elif tree.is_node(index, 'subscript'):  # subscript is a slice operation.
+        # Like array[:3]
+        result = []
+        for el in index.children:
+            if el == ':':
+                if not result:
+                    result.append(None)
+            elif tree.is_node(el, 'sliceop'):
+                if len(el.children) == 2:
+                    result.append(el.children[1])
+            else:
+                result.append(el)
+        result += [None] * (3 - len(result))
+
+        return set([Slice(evaluator, *result)])
+
+    # No slices
+    return evaluator.eval_element(index)

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/jedi_typing.py

@@ -0,0 +1,100 @@
+"""
+This module is not intended to be used in jedi, rather it will be fed to the
+jedi-parser to replace classes in the typing module
+"""
+
+try:
+    from collections import abc
+except ImportError:
+    # python 2
+    import collections as abc
+
+
+def factory(typing_name, indextypes):
+    class Iterable(abc.Iterable):
+        def __iter__(self):
+            while True:
+                yield indextypes[0]()
+
+    class Iterator(Iterable, abc.Iterator):
+        def next(self):
+            """ needed for python 2 """
+            return self.__next__()
+
+        def __next__(self):
+            return indextypes[0]()
+
+    class Sequence(abc.Sequence):
+        def __getitem__(self, index):
+            return indextypes[0]()
+
+    class MutableSequence(Sequence, abc.MutableSequence):
+        pass
+
+    class List(MutableSequence, list):
+        pass
+
+    class Tuple(Sequence, tuple):
+        def __getitem__(self, index):
+            if indextypes[1] == Ellipsis:
+                # https://www.python.org/dev/peps/pep-0484/#the-typing-module
+                # Tuple[int, ...] means a tuple of ints of indetermined length
+                return indextypes[0]()
+            else:
+                return indextypes[index]()
+
+    class AbstractSet(Iterable, abc.Set):
+        pass
+
+    class MutableSet(AbstractSet, abc.MutableSet):
+        pass
+
+    class KeysView(Iterable, abc.KeysView):
+        pass
+
+    class ValuesView(abc.ValuesView):
+        def __iter__(self):
+            while True:
+                yield indextypes[1]()
+
+    class ItemsView(abc.ItemsView):
+        def __iter__(self):
+            while True:
+                yield (indextypes[0](), indextypes[1]())
+
+    class Mapping(Iterable, abc.Mapping):
+        def __getitem__(self, item):
+            return indextypes[1]()
+
+        def keys(self):
+            return KeysView()
+
+        def values(self):
+            return ValuesView()
+
+        def items(self):
+            return ItemsView()
+
+    class MutableMapping(Mapping, abc.MutableMapping):
+        pass
+
+    class Dict(MutableMapping, dict):
+        pass
+
+    dct = {
+        "Sequence": Sequence,
+        "MutableSequence": MutableSequence,
+        "List": List,
+        "Iterable": Iterable,
+        "Iterator": Iterator,
+        "AbstractSet": AbstractSet,
+        "MutableSet": MutableSet,
+        "Mapping": Mapping,
+        "MutableMapping": MutableMapping,
+        "Tuple": Tuple,
+        "KeysView": KeysView,
+        "ItemsView": ItemsView,
+        "ValuesView": ValuesView,
+        "Dict": Dict,
+    }
+    return dct[typing_name]

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/param.py

@@ -0,0 +1,438 @@
+from collections import defaultdict
+from itertools import chain
+
+from jedi._compatibility import unicode, zip_longest
+from jedi import debug
+from jedi import common
+from jedi.parser import tree
+from jedi.evaluate import iterable
+from jedi.evaluate import analysis
+from jedi.evaluate import precedence
+from jedi.evaluate.helpers import FakeName
+from jedi.cache import underscore_memoization
+
+
+def try_iter_content(types, depth=0):
+    """Helper method for static analysis."""
+    if depth > 10:
+        # It's possible that a loop has references on itself (especially with
+        # CompiledObject). Therefore don't loop infinitely.
+        return
+
+    for typ in types:
+        try:
+            f = typ.py__iter__
+        except AttributeError:
+            pass
+        else:
+            for iter_types in f():
+                try_iter_content(iter_types, depth + 1)
+
+
+class Arguments(tree.Base):
+    def __init__(self, evaluator, argument_node, trailer=None):
+        """
+        The argument_node is either a parser node or a list of evaluated
+        objects. Those evaluated objects may be lists of evaluated objects
+        themselves (one list for the first argument, one for the second, etc).
+
+        :param argument_node: May be an argument_node or a list of nodes.
+        """
+        self.argument_node = argument_node
+        self._evaluator = evaluator
+        self.trailer = trailer  # Can be None, e.g. in a class definition.
+
+    def _split(self):
+        if isinstance(self.argument_node, (tuple, list)):
+            for el in self.argument_node:
+                yield 0, el
+        else:
+            if not (tree.is_node(self.argument_node, 'arglist') or (
+                    # in python 3.5 **arg is an argument, not arglist
+                    (tree.is_node(self.argument_node, 'argument') and
+                     self.argument_node.children[0] in ('*', '**')))):
+                yield 0, self.argument_node
+                return
+
+            iterator = iter(self.argument_node.children)
+            for child in iterator:
+                if child == ',':
+                    continue
+                elif child in ('*', '**'):
+                    yield len(child.value), next(iterator)
+                elif tree.is_node(child, 'argument') and \
+                        child.children[0] in ('*', '**'):
+                    assert len(child.children) == 2
+                    yield len(child.children[0].value), child.children[1]
+                else:
+                    yield 0, child
+
+    def get_parent_until(self, *args, **kwargs):
+        if self.trailer is None:
+            try:
+                element = self.argument_node[0]
+                from jedi.evaluate.iterable import AlreadyEvaluated
+                if isinstance(element, AlreadyEvaluated):
+                    element = list(self._evaluator.eval_element(element))[0]
+            except IndexError:
+                return None
+            else:
+                return element.get_parent_until(*args, **kwargs)
+        else:
+            return self.trailer.get_parent_until(*args, **kwargs)
+
+    def as_tuple(self):
+        for stars, argument in self._split():
+            if tree.is_node(argument, 'argument'):
+                argument, default = argument.children[::2]
+            else:
+                default = None
+            yield argument, default, stars
+
+    def unpack(self, func=None):
+        named_args = []
+        for stars, el in self._split():
+            if stars == 1:
+                arrays = self._evaluator.eval_element(el)
+                iterators = [_iterate_star_args(self._evaluator, a, el, func)
+                             for a in arrays]
+                iterators = list(iterators)
+                for values in list(zip_longest(*iterators)):
+                    yield None, [v for v in values if v is not None]
+            elif stars == 2:
+                arrays = self._evaluator.eval_element(el)
+                dicts = [_star_star_dict(self._evaluator, a, el, func)
+                         for a in arrays]
+                for dct in dicts:
+                    for key, values in dct.items():
+                        yield key, values
+            else:
+                if tree.is_node(el, 'argument'):
+                    c = el.children
+                    if len(c) == 3:  # Keyword argument.
+                        named_args.append((c[0].value, (c[2],)))
+                    else:  # Generator comprehension.
+                        # Include the brackets with the parent.
+                        comp = iterable.GeneratorComprehension(
+                            self._evaluator, self.argument_node.parent)
+                        yield None, (iterable.AlreadyEvaluated([comp]),)
+                elif isinstance(el, (list, tuple)):
+                    yield None, el
+                else:
+                    yield None, (el,)
+
+        # Reordering var_args is necessary, because star args sometimes appear
+        # after named argument, but in the actual order it's prepended.
+        for key_arg in named_args:
+            yield key_arg
+
+    def _reorder_var_args(var_args):
+        named_index = None
+        new_args = []
+        for i, stmt in enumerate(var_args):
+            if isinstance(stmt, tree.ExprStmt):
+                if named_index is None and stmt.assignment_details:
+                    named_index = i
+
+                if named_index is not None:
+                    expression_list = stmt.expression_list()
+                    if expression_list and expression_list[0] == '*':
+                        new_args.insert(named_index, stmt)
+                        named_index += 1
+                        continue
+
+            new_args.append(stmt)
+        return new_args
+
+    def eval_argument_clinic(self, arguments):
+        """Uses a list with argument clinic information (see PEP 436)."""
+        iterator = self.unpack()
+        for i, (name, optional, allow_kwargs) in enumerate(arguments):
+            key, va_values = next(iterator, (None, []))
+            if key is not None:
+                raise NotImplementedError
+            if not va_values and not optional:
+                debug.warning('TypeError: %s expected at least %s arguments, got %s',
+                              name, len(arguments), i)
+                raise ValueError
+            values = set(chain.from_iterable(self._evaluator.eval_element(el)
+                                             for el in va_values))
+            if not values and not optional:
+                # For the stdlib we always want values. If we don't get them,
+                # that's ok, maybe something is too hard to resolve, however,
+                # we will not proceed with the evaluation of that function.
+                debug.warning('argument_clinic "%s" not resolvable.', name)
+                raise ValueError
+            yield values
+
+    def scope(self):
+        # Returns the scope in which the arguments are used.
+        return (self.trailer or self.argument_node).get_parent_until(tree.IsScope)
+
+    def eval_args(self):
+        # TODO this method doesn't work with named args and a lot of other
+        # things. Use unpack.
+        return [self._evaluator.eval_element(el) for stars, el in self._split()]
+
+    def __repr__(self):
+        return '<%s: %s>' % (type(self).__name__, self.argument_node)
+
+    def get_calling_var_args(self):
+        if tree.is_node(self.argument_node, 'arglist', 'argument') \
+                or self.argument_node == () and self.trailer is not None:
+            return _get_calling_var_args(self._evaluator, self)
+        else:
+            return None
+
+    def eval_all(self, func=None):
+        """
+        Evaluates all arguments as a support for static analysis
+        (normally Jedi).
+        """
+        for key, element_values in self.unpack():
+            for element in element_values:
+                types = self._evaluator.eval_element(element)
+                try_iter_content(types)
+
+
+class ExecutedParam(tree.Param):
+    """Fake a param and give it values."""
+    def __init__(self, original_param, var_args, values):
+        self._original_param = original_param
+        self.var_args = var_args
+        self._values = values
+
+    def eval(self, evaluator):
+        types = set()
+        for v in self._values:
+            types |= evaluator.eval_element(v)
+        return types
+
+    @property
+    def position_nr(self):
+        # Need to use the original logic here, because it uses the parent.
+        return self._original_param.position_nr
+
+    @property
+    @underscore_memoization
+    def name(self):
+        return FakeName(str(self._original_param.name), self, self.start_pos)
+
+    def __getattr__(self, name):
+        return getattr(self._original_param, name)
+
+
+def _get_calling_var_args(evaluator, var_args):
+    old_var_args = None
+    while var_args != old_var_args:
+        old_var_args = var_args
+        for name, default, stars in reversed(list(var_args.as_tuple())):
+            if not stars or not isinstance(name, tree.Name):
+                continue
+
+            names = evaluator.goto(name)
+            if len(names) != 1:
+                break
+            param = names[0].get_definition()
+            if not isinstance(param, ExecutedParam):
+                if isinstance(param, tree.Param):
+                    # There is no calling var_args in this case - there's just
+                    # a param without any input.
+                    return None
+                break
+            # We never want var_args to be a tuple. This should be enough for
+            # now, we can change it later, if we need to.
+            if isinstance(param.var_args, Arguments):
+                var_args = param.var_args
+    return var_args.argument_node or var_args.trailer
+
+
+def get_params(evaluator, func, var_args):
+    param_names = []
+    param_dict = {}
+    for param in func.params:
+        param_dict[str(param.name)] = param
+    unpacked_va = list(var_args.unpack(func))
+    from jedi.evaluate.representation import InstanceElement
+    if isinstance(func, InstanceElement):
+        # Include self at this place.
+        unpacked_va.insert(0, (None, [iterable.AlreadyEvaluated([func.instance])]))
+    var_arg_iterator = common.PushBackIterator(iter(unpacked_va))
+
+    non_matching_keys = defaultdict(lambda: [])
+    keys_used = {}
+    keys_only = False
+    had_multiple_value_error = False
+    for param in func.params:
+        # The value and key can both be null. There, the defaults apply.
+        # args / kwargs will just be empty arrays / dicts, respectively.
+        # Wrong value count is just ignored. If you try to test cases that are
+        # not allowed in Python, Jedi will maybe not show any completions.
+        default = [] if param.default is None else [param.default]
+        key, va_values = next(var_arg_iterator, (None, default))
+        while key is not None:
+            keys_only = True
+            k = unicode(key)
+            try:
+                key_param = param_dict[unicode(key)]
+            except KeyError:
+                non_matching_keys[key] = va_values
+            else:
+                param_names.append(ExecutedParam(key_param, var_args, va_values).name)
+
+            if k in keys_used:
+                had_multiple_value_error = True
+                m = ("TypeError: %s() got multiple values for keyword argument '%s'."
+                     % (func.name, k))
+                calling_va = _get_calling_var_args(evaluator, var_args)
+                if calling_va is not None:
+                    analysis.add(evaluator, 'type-error-multiple-values',
+                                 calling_va, message=m)
+            else:
+                try:
+                    keys_used[k] = param_names[-1]
+                except IndexError:
+                    # TODO this is wrong stupid and whatever.
+                    pass
+            key, va_values = next(var_arg_iterator, (None, ()))
+
+        values = []
+        if param.stars == 1:
+            # *args param
+            lst_values = [iterable.MergedNodes(va_values)] if va_values else []
+            for key, va_values in var_arg_iterator:
+                # Iterate until a key argument is found.
+                if key:
+                    var_arg_iterator.push_back((key, va_values))
+                    break
+                if va_values:
+                    lst_values.append(iterable.MergedNodes(va_values))
+            seq = iterable.FakeSequence(evaluator, lst_values, 'tuple')
+            values = [iterable.AlreadyEvaluated([seq])]
+        elif param.stars == 2:
+            # **kwargs param
+            dct = iterable.FakeDict(evaluator, dict(non_matching_keys))
+            values = [iterable.AlreadyEvaluated([dct])]
+            non_matching_keys = {}
+        else:
+            # normal param
+            if va_values:
+                values = va_values
+            else:
+                # No value: Return an empty container
+                values = []
+                if not keys_only:
+                    calling_va = var_args.get_calling_var_args()
+                    if calling_va is not None:
+                        m = _error_argument_count(func, len(unpacked_va))
+                        analysis.add(evaluator, 'type-error-too-few-arguments',
+                                     calling_va, message=m)
+
+        # Now add to result if it's not one of the previously covered cases.
+        if (not keys_only or param.stars == 2):
+            param_names.append(ExecutedParam(param, var_args, values).name)
+            keys_used[unicode(param.name)] = param_names[-1]
+
+    if keys_only:
+        # All arguments should be handed over to the next function. It's not
+        # about the values inside, it's about the names. Jedi needs to now that
+        # there's nothing to find for certain names.
+        for k in set(param_dict) - set(keys_used):
+            param = param_dict[k]
+            values = [] if param.default is None else [param.default]
+            param_names.append(ExecutedParam(param, var_args, values).name)
+
+            if not (non_matching_keys or had_multiple_value_error
+                    or param.stars or param.default):
+                # add a warning only if there's not another one.
+                calling_va = _get_calling_var_args(evaluator, var_args)
+                if calling_va is not None:
+                    m = _error_argument_count(func, len(unpacked_va))
+                    analysis.add(evaluator, 'type-error-too-few-arguments',
+                                 calling_va, message=m)
+
+    for key, va_values in non_matching_keys.items():
+        m = "TypeError: %s() got an unexpected keyword argument '%s'." \
+            % (func.name, key)
+        for value in va_values:
+            analysis.add(evaluator, 'type-error-keyword-argument', value.parent, message=m)
+
+    remaining_params = list(var_arg_iterator)
+    if remaining_params:
+        m = _error_argument_count(func, len(unpacked_va))
+        # Just report an error for the first param that is not needed (like
+        # cPython).
+        first_key, first_values = remaining_params[0]
+        for v in first_values:
+            if first_key is not None:
+                # Is a keyword argument, return the whole thing instead of just
+                # the value node.
+                v = v.parent
+                try:
+                    non_kw_param = keys_used[first_key]
+                except KeyError:
+                    pass
+                else:
+                    origin_args = non_kw_param.parent.var_args.argument_node
+                    # TODO  calculate the var_args tree and check if it's in
+                    # the tree (if not continue).
+                    # print('\t\tnonkw', non_kw_param.parent.var_args.argument_node, )
+                    if origin_args not in [f.parent.parent for f in first_values]:
+                        continue
+            analysis.add(evaluator, 'type-error-too-many-arguments',
+                         v, message=m)
+    return param_names
+
+
+def _iterate_star_args(evaluator, array, input_node, func=None):
+    from jedi.evaluate.representation import Instance
+    if isinstance(array, iterable.Array):
+        # TODO ._items is not the call we want here. Replace in the future.
+        for node in array._items():
+            yield node
+    elif isinstance(array, iterable.Generator):
+        for types in array.py__iter__():
+            yield iterable.AlreadyEvaluated(types)
+    elif isinstance(array, Instance) and array.name.get_code() == 'tuple':
+        debug.warning('Ignored a tuple *args input %s' % array)
+    else:
+        if func is not None:
+            m = "TypeError: %s() argument after * must be a sequence, not %s" \
+                % (func.name.value, array)
+            analysis.add(evaluator, 'type-error-star', input_node, message=m)
+
+
+def _star_star_dict(evaluator, array, input_node, func):
+    dct = defaultdict(lambda: [])
+    from jedi.evaluate.representation import Instance
+    if isinstance(array, Instance) and array.name.get_code() == 'dict':
+        # For now ignore this case. In the future add proper iterators and just
+        # make one call without crazy isinstance checks.
+        return {}
+
+    if isinstance(array, iterable.FakeDict):
+        return array._dct
+    elif isinstance(array, iterable.Array) and array.type == 'dict':
+        # TODO bad call to non-public API
+        for key_node, value in array._items():
+            for key in evaluator.eval_element(key_node):
+                if precedence.is_string(key):
+                    dct[key.obj].append(value)
+
+    else:
+        if func is not None:
+            m = "TypeError: %s argument after ** must be a mapping, not %s" \
+                % (func.name.value, array)
+            analysis.add(evaluator, 'type-error-star-star', input_node, message=m)
+    return dict(dct)
+
+
+def _error_argument_count(func, actual_count):
+    default_arguments = sum(1 for p in func.params if p.default or p.stars)
+
+    if default_arguments == 0:
+        before = 'exactly '
+    else:
+        before = 'from %s to ' % (len(func.params) - default_arguments)
+    return ('TypeError: %s() takes %s%s arguments (%s given).'
+            % (func.name, before, len(func.params), actual_count))

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/pep0484.py

@@ -0,0 +1,195 @@
+"""
+PEP 0484 ( https://www.python.org/dev/peps/pep-0484/ ) describes type hints
+through function annotations. There is a strong suggestion in this document
+that only the type of type hinting defined in PEP0484 should be allowed
+as annotations in future python versions.
+
+The (initial / probably incomplete) implementation todo list for pep-0484:
+v Function parameter annotations with builtin/custom type classes
+v Function returntype annotations with builtin/custom type classes
+v Function parameter annotations with strings (forward reference)
+v Function return type annotations with strings (forward reference)
+v Local variable type hints
+v Assigned types: `Url = str\ndef get(url:Url) -> str:`
+v Type hints in `with` statements
+x Stub files support
+x support `@no_type_check` and `@no_type_check_decorator`
+x support for typing.cast() operator
+x support for type hint comments for functions, `# type: (int, str) -> int`.
+    See comment from Guido https://github.com/davidhalter/jedi/issues/662
+"""
+
+import itertools
+
+import os
+from jedi.parser import \
+    Parser, load_grammar, ParseError, ParserWithRecovery, tree
+from jedi.evaluate.cache import memoize_default
+from jedi.common import unite
+from jedi.evaluate import compiled
+from jedi import debug
+from jedi import _compatibility
+import re
+
+
+def _evaluate_for_annotation(evaluator, annotation, index=None):
+    """
+    Evaluates a string-node, looking for an annotation
+    If index is not None, the annotation is expected to be a tuple
+    and we're interested in that index
+    """
+    if annotation is not None:
+        definitions = evaluator.eval_element(
+            _fix_forward_reference(evaluator, annotation))
+        if index is not None:
+            definitions = list(itertools.chain.from_iterable(
+                definition.py__getitem__(index) for definition in definitions
+                if definition.type == 'tuple' and
+                len(list(definition.py__iter__())) >= index))
+        return list(itertools.chain.from_iterable(
+            evaluator.execute(d) for d in definitions))
+    else:
+        return []
+
+
+def _fix_forward_reference(evaluator, node):
+    evaled_nodes = evaluator.eval_element(node)
+    if len(evaled_nodes) != 1:
+        debug.warning("Eval'ed typing index %s should lead to 1 object, "
+                      " not %s" % (node, evaled_nodes))
+        return node
+    evaled_node = list(evaled_nodes)[0]
+    if isinstance(evaled_node, compiled.CompiledObject) and \
+            isinstance(evaled_node.obj, str):
+        try:
+            p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
+                       start_symbol='eval_input')
+            newnode = p.get_parsed_node()
+        except ParseError:
+            debug.warning('Annotation not parsed: %s' % evaled_node.obj)
+            return node
+        else:
+            module = node.get_parent_until()
+            p.position_modifier.line = module.end_pos[0]
+            newnode.parent = module
+            return newnode
+    else:
+        return node
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def follow_param(evaluator, param):
+    annotation = param.annotation()
+    return _evaluate_for_annotation(evaluator, annotation)
+
+
+@memoize_default(None, evaluator_is_first_arg=True)
+def find_return_types(evaluator, func):
+    annotation = func.py__annotations__().get("return", None)
+    return _evaluate_for_annotation(evaluator, annotation)
+
+
+_typing_module = None
+
+
+def _get_typing_replacement_module():
+    """
+    The idea is to return our jedi replacement for the PEP-0484 typing module
+    as discussed at https://github.com/davidhalter/jedi/issues/663
+    """
+    global _typing_module
+    if _typing_module is None:
+        typing_path = \
+            os.path.abspath(os.path.join(__file__, "../jedi_typing.py"))
+        with open(typing_path) as f:
+            code = _compatibility.unicode(f.read())
+        p = ParserWithRecovery(load_grammar(), code)
+        _typing_module = p.module
+    return _typing_module
+
+
+def get_types_for_typing_module(evaluator, typ, node):
+    from jedi.evaluate.iterable import FakeSequence
+    if not typ.base.get_parent_until().name.value == "typing":
+        return None
+    # we assume that any class using [] in a module called
+    # "typing" with a name for which we have a replacement
+    # should be replaced by that class. This is not 100%
+    # airtight but I don't have a better idea to check that it's
+    # actually the PEP-0484 typing module and not some other
+    if tree.is_node(node, "subscriptlist"):
+        nodes = node.children[::2]  # skip the commas
+    else:
+        nodes = [node]
+    del node
+
+    nodes = [_fix_forward_reference(evaluator, node) for node in nodes]
+
+    # hacked in Union and Optional, since it's hard to do nicely in parsed code
+    if typ.name.value == "Union":
+        return unite(evaluator.eval_element(node) for node in nodes)
+    if typ.name.value == "Optional":
+        return evaluator.eval_element(nodes[0])
+
+    typing = _get_typing_replacement_module()
+    factories = evaluator.find_types(typing, "factory")
+    assert len(factories) == 1
+    factory = list(factories)[0]
+    assert factory
+    function_body_nodes = factory.children[4].children
+    valid_classnames = set(child.name.value
+                           for child in function_body_nodes
+                           if isinstance(child, tree.Class))
+    if typ.name.value not in valid_classnames:
+        return None
+    compiled_classname = compiled.create(evaluator, typ.name.value)
+
+    args = FakeSequence(evaluator, nodes, "tuple")
+
+    result = evaluator.execute_evaluated(factory, compiled_classname, args)
+    return result
+
+
+def find_type_from_comment_hint_for(evaluator, node, name):
+    return \
+        _find_type_from_comment_hint(evaluator, node, node.children[1], name)
+
+
+def find_type_from_comment_hint_with(evaluator, node, name):
+    assert len(node.children[1].children) == 3, \
+        "Can only be here when children[1] is 'foo() as f'"
+    return _find_type_from_comment_hint(
+        evaluator, node, node.children[1].children[2], name)
+
+
+def find_type_from_comment_hint_assign(evaluator, node, name):
+    return \
+        _find_type_from_comment_hint(evaluator, node, node.children[0], name)
+
+
+def _find_type_from_comment_hint(evaluator, node, varlist, name):
+    index = None
+    if varlist.type in ("testlist_star_expr", "exprlist"):
+        # something like "a, b = 1, 2"
+        index = 0
+        for child in varlist.children:
+            if child == name:
+                break
+            if child.type == "operator":
+                continue
+            index += 1
+        else:
+            return []
+
+    comment = node.get_following_comment_same_line()
+    if comment is None:
+        return []
+    match = re.match(r"^#\s*type:\s*([^#]*)", comment)
+    if not match:
+        return []
+    annotation = tree.String(
+        tree.zero_position_modifier,
+        repr(str(match.group(1).strip())),
+        node.start_pos)
+    annotation.parent = node.parent
+    return _evaluate_for_annotation(evaluator, annotation, index)

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/precedence.py

@@ -0,0 +1,178 @@
+"""
+Handles operator precedence.
+"""
+import operator
+
+from jedi._compatibility import unicode
+from jedi.parser import tree
+from jedi import debug
+from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name
+from jedi.evaluate import analysis
+
+# Maps Python syntax to the operator module.
+COMPARISON_OPERATORS = {
+    '==': operator.eq,
+    '!=': operator.ne,
+    'is': operator.is_,
+    'is not': operator.is_not,
+    '<': operator.lt,
+    '<=': operator.le,
+    '>': operator.gt,
+    '>=': operator.ge,
+}
+
+
+def literals_to_types(evaluator, result):
+    # Changes literals ('a', 1, 1.0, etc) to its type instances (str(),
+    # int(), float(), etc).
+    new_result = set()
+    for typ in result:
+        if is_literal(typ):
+            # Literals are only valid as long as the operations are
+            # correct. Otherwise add a value-free instance.
+            cls = builtin_from_name(evaluator, typ.name.value)
+            new_result |= evaluator.execute(cls)
+        else:
+            new_result.add(typ)
+    return new_result
+
+
+def calculate_children(evaluator, children):
+    """
+    Calculate a list of children with operators.
+    """
+    iterator = iter(children)
+    types = evaluator.eval_element(next(iterator))
+    for operator in iterator:
+        right = next(iterator)
+        if tree.is_node(operator, 'comp_op'):  # not in / is not
+            operator = ' '.join(str(c.value) for c in operator.children)
+
+        # handle lazy evaluation of and/or here.
+        if operator in ('and', 'or'):
+            left_bools = set([left.py__bool__() for left in types])
+            if left_bools == set([True]):
+                if operator == 'and':
+                    types = evaluator.eval_element(right)
+            elif left_bools == set([False]):
+                if operator != 'and':
+                    types = evaluator.eval_element(right)
+            # Otherwise continue, because of uncertainty.
+        else:
+            types = calculate(evaluator, types, operator,
+                              evaluator.eval_element(right))
+    debug.dbg('calculate_children types %s', types)
+    return types
+
+
+def calculate(evaluator, left_result, operator, right_result):
+    result = set()
+    if not left_result or not right_result:
+        # illegal slices e.g. cause left/right_result to be None
+        result = (left_result or set()) | (right_result or set())
+        result = literals_to_types(evaluator, result)
+    else:
+        # I don't think there's a reasonable chance that a string
+        # operation is still correct, once we pass something like six
+        # objects.
+        if len(left_result) * len(right_result) > 6:
+            result = literals_to_types(evaluator, left_result | right_result)
+        else:
+            for left in left_result:
+                for right in right_result:
+                    result |= _element_calculate(evaluator, left, operator, right)
+    return result
+
+
+def factor_calculate(evaluator, types, operator):
+    """
+    Calculates `+`, `-`, `~` and `not` prefixes.
+    """
+    for typ in types:
+        if operator == '-':
+            if _is_number(typ):
+                yield create(evaluator, -typ.obj)
+        elif operator == 'not':
+            value = typ.py__bool__()
+            if value is None:  # Uncertainty.
+                return
+            yield create(evaluator, not value)
+        else:
+            yield typ
+
+
+def _is_number(obj):
+    return isinstance(obj, CompiledObject) \
+        and isinstance(obj.obj, (int, float))
+
+
+def is_string(obj):
+    return isinstance(obj, CompiledObject) \
+        and isinstance(obj.obj, (str, unicode))
+
+
+def is_literal(obj):
+    return _is_number(obj) or is_string(obj)
+
+
+def _is_tuple(obj):
+    from jedi.evaluate import iterable
+    return isinstance(obj, iterable.Array) and obj.type == 'tuple'
+
+
+def _is_list(obj):
+    from jedi.evaluate import iterable
+    return isinstance(obj, iterable.Array) and obj.type == 'list'
+
+
+def _element_calculate(evaluator, left, operator, right):
+    from jedi.evaluate import iterable, representation as er
+    l_is_num = _is_number(left)
+    r_is_num = _is_number(right)
+    if operator == '*':
+        # for iterables, ignore * operations
+        if isinstance(left, iterable.Array) or is_string(left):
+            return set([left])
+        elif isinstance(right, iterable.Array) or is_string(right):
+            return set([right])
+    elif operator == '+':
+        if l_is_num and r_is_num or is_string(left) and is_string(right):
+            return set([create(evaluator, left.obj + right.obj)])
+        elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right):
+            return set([iterable.MergedArray(evaluator, (left, right))])
+    elif operator == '-':
+        if l_is_num and r_is_num:
+            return set([create(evaluator, left.obj - right.obj)])
+    elif operator == '%':
+        # With strings and numbers the left type typically remains. Except for
+        # `int() % float()`.
+        return set([left])
+    elif operator in COMPARISON_OPERATORS:
+        operation = COMPARISON_OPERATORS[operator]
+        if isinstance(left, CompiledObject) and isinstance(right, CompiledObject):
+            # Possible, because the return is not an option. Just compare.
+            left = left.obj
+            right = right.obj
+
+        try:
+            result = operation(left, right)
+        except TypeError:
+            # Could be True or False.
+            return set([create(evaluator, True), create(evaluator, False)])
+        else:
+            return set([create(evaluator, result)])
+    elif operator == 'in':
+        return set()
+
+    def check(obj):
+        """Checks if a Jedi object is either a float or an int."""
+        return isinstance(obj, er.Instance) and obj.name.get_code() in ('int', 'float')
+
+    # Static analysis, one is a number, the other one is not.
+    if operator in ('+', '-') and l_is_num != r_is_num \
+            and not (check(left) or check(right)):
+        message = "TypeError: unsupported operand type(s) for +: %s and %s"
+        analysis.add(evaluator, 'type-error-operation', operator,
+                     message % (left, right))
+
+    return set([left, right])

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/recursion.py

@@ -0,0 +1,157 @@
+"""
+Recursions are the recipe of |jedi| to conquer Python code. However, someone
+must stop recursions going mad. Some settings are here to make |jedi| stop at
+the right time. You can read more about them :ref:`here <settings-recursion>`.
+
+Next to :mod:`jedi.evaluate.cache` this module also makes |jedi| not
+thread-safe. Why?  ``execution_recursion_decorator`` uses class variables to
+count the function calls.
+"""
+from jedi import debug
+from jedi import settings
+from jedi.evaluate import iterable
+
+
+def recursion_decorator(func):
+    def run(evaluator, stmt, *args, **kwargs):
+        rec_detect = evaluator.recursion_detector
+        if rec_detect.push_stmt(stmt):
+            return set()
+        else:
+            result = func(evaluator, stmt, *args, **kwargs)
+            rec_detect.pop_stmt()
+        return result
+    return run
+
+
+class RecursionDetector(object):
+    """
+    A decorator to detect recursions in statements. In a recursion a statement
+    at the same place, in the same module may not be executed two times.
+    """
+    def __init__(self, evaluator):
+        self.top = None
+        self.current = None
+        self._evaluator = evaluator
+
+    def push_stmt(self, stmt):
+        self.current = _RecursionNode(self._evaluator, stmt, self.current)
+        check = self._check_recursion()
+        if check:
+            debug.warning('catched stmt recursion: %s against %s @%s', stmt,
+                          check.stmt, stmt.start_pos)
+            self.pop_stmt()
+            return True
+        return False
+
+    def pop_stmt(self):
+        if self.current is not None:
+            # I don't know how current can be None, but sometimes it happens
+            # with Python3.
+            self.current = self.current.parent
+
+    def _check_recursion(self):
+        test = self.current
+        while True:
+            test = test.parent
+            if self.current == test:
+                return test
+            if not test:
+                return False
+
+    def node_statements(self):
+        result = []
+        n = self.current
+        while n:
+            result.insert(0, n.stmt)
+            n = n.parent
+        return result
+
+
+class _RecursionNode(object):
+    """ A node of the RecursionDecorator. """
+    def __init__(self, evaluator, stmt, parent):
+        self._evaluator = evaluator
+        self.script = stmt.get_parent_until()
+        self.position = stmt.start_pos
+        self.parent = parent
+        self.stmt = stmt
+
+        # Don't check param instances, they are not causing recursions
+        # The same's true for the builtins, because the builtins are really
+        # simple.
+        self.is_ignored = self.script == self._evaluator.BUILTINS
+
+    def __eq__(self, other):
+        if not other:
+            return None
+
+        return self.script == other.script \
+            and self.position == other.position \
+            and not self.is_ignored and not other.is_ignored
+
+
+def execution_recursion_decorator(func):
+    def run(execution, **kwargs):
+        detector = execution._evaluator.execution_recursion_detector
+        if detector.push_execution(execution):
+            result = set()
+        else:
+            result = func(execution, **kwargs)
+        detector.pop_execution()
+        return result
+
+    return run
+
+
+class ExecutionRecursionDetector(object):
+    """
+    Catches recursions of executions.
+    """
+    def __init__(self, evaluator):
+        self.recursion_level = 0
+        self.parent_execution_funcs = []
+        self.execution_funcs = set()
+        self.execution_count = 0
+        self._evaluator = evaluator
+
+    def __call__(self, execution):
+        debug.dbg('Execution recursions: %s', execution, self.recursion_level,
+                  self.execution_count, len(self.execution_funcs))
+        if self.check_recursion(execution):
+            result = set()
+        else:
+            result = self.func(execution)
+        self.pop_execution()
+        return result
+
+    def pop_execution(self):
+        self.parent_execution_funcs.pop()
+        self.recursion_level -= 1
+
+    def push_execution(self, execution):
+        in_par_execution_funcs = execution.base in self.parent_execution_funcs
+        in_execution_funcs = execution.base in self.execution_funcs
+        self.recursion_level += 1
+        self.execution_count += 1
+        self.execution_funcs.add(execution.base)
+        self.parent_execution_funcs.append(execution.base)
+
+        if self.execution_count > settings.max_executions:
+            return True
+
+        if isinstance(execution.base, (iterable.Array, iterable.Generator)):
+            return False
+        module = execution.get_parent_until()
+        if module == self._evaluator.BUILTINS:
+            return False
+
+        if in_par_execution_funcs:
+            if self.recursion_level > settings.max_function_recursion_level:
+                return True
+        if in_execution_funcs and \
+                len(self.execution_funcs) > settings.max_until_execution_unique:
+            return True
+        if self.execution_count > settings.max_executions_without_builtins:
+            return True
+        return False

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/representation.py

@@ -0,0 +1,974 @@
+"""
+Like described in the :mod:`jedi.parser.tree` module,
+there's a need for an ast like module to represent the states of parsed
+modules.
+
+But now there are also structures in Python that need a little bit more than
+that. An ``Instance`` for example is only a ``Class`` before it is
+instantiated. This class represents these cases.
+
+So, why is there also a ``Class`` class here? Well, there are decorators and
+they change classes in Python 3.
+
+Representation modules also define "magic methods". Those methods look like
+``py__foo__`` and are typically mappable to the Python equivalents ``__call__``
+and others. Here's a list:
+
+====================================== ========================================
+**Method**                             **Description**
+-------------------------------------- ----------------------------------------
+py__call__(params: Array)              On callable objects, returns types.
+py__bool__()                           Returns True/False/None; None means that
+                                       there's no certainty.
+py__bases__()                          Returns a list of base classes.
+py__mro__()                            Returns a list of classes (the mro).
+py__iter__()                           Returns a generator of a set of types.
+py__class__()                          Returns the class of an instance.
+py__getitem__(index: int/str)          Returns a a set of types of the index.
+                                       Can raise an IndexError/KeyError.
+py__file__()                           Only on modules.
+py__package__()                        Only on modules. For the import system.
+py__path__()                           Only on modules. For the import system.
+====================================== ========================================
+
+__
+"""
+import os
+import pkgutil
+import imp
+import re
+from itertools import chain
+
+from jedi._compatibility import use_metaclass, unicode, Python3Method, is_py3
+from jedi.parser import tree
+from jedi import debug
+from jedi import common
+from jedi.cache import underscore_memoization, cache_star_import
+from jedi.evaluate.cache import memoize_default, CachedMetaClass, NO_DEFAULT
+from jedi.evaluate import compiled
+from jedi.evaluate.compiled import mixed
+from jedi.evaluate import recursion
+from jedi.evaluate import iterable
+from jedi.evaluate import docstrings
+from jedi.evaluate import pep0484
+from jedi.evaluate import helpers
+from jedi.evaluate import param
+from jedi.evaluate import flow_analysis
+from jedi.evaluate import imports
+
+
+class Executed(tree.Base):
+    """
+    An instance is also an executable - because __init__ is called
+    :param var_args: The param input array, consist of a parser node or a list.
+    """
+    def __init__(self, evaluator, base, var_args=()):
+        self._evaluator = evaluator
+        self.base = base
+        self.var_args = var_args
+
+    def is_scope(self):
+        return True
+
+    def get_parent_until(self, *args, **kwargs):
+        return tree.Base.get_parent_until(self, *args, **kwargs)
+
+    @common.safe_property
+    def parent(self):
+        return self.base.parent
+
+
+class Instance(use_metaclass(CachedMetaClass, Executed)):
+    """
+    This class is used to evaluate instances.
+    """
+    def __init__(self, evaluator, base, var_args, is_generated=False):
+        super(Instance, self).__init__(evaluator, base, var_args)
+        self.decorates = None
+        # Generated instances are classes that are just generated by self
+        # (No var_args) used.
+        self.is_generated = is_generated
+
+        if base.name.get_code() in ['list', 'set'] \
+                and evaluator.BUILTINS == base.get_parent_until():
+            # compare the module path with the builtin name.
+            self.var_args = iterable.check_array_instances(evaluator, self)
+        elif not is_generated:
+            # Need to execute the __init__ function, because the dynamic param
+            # searching needs it.
+            try:
+                method = self.get_subscope_by_name('__init__')
+            except KeyError:
+                pass
+            else:
+                evaluator.execute(method, self.var_args)
+
+    def is_class(self):
+        return False
+
+    @property
+    def py__call__(self):
+        def actual(params):
+            return self._evaluator.execute(method, params)
+
+        try:
+            method = self.get_subscope_by_name('__call__')
+        except KeyError:
+            # Means the Instance is not callable.
+            raise AttributeError
+
+        return actual
+
+    def py__class__(self):
+        return self.base
+
+    def py__bool__(self):
+        # Signalize that we don't know about the bool type.
+        return None
+
+    @memoize_default()
+    def _get_method_execution(self, func):
+        func = get_instance_el(self._evaluator, self, func, True)
+        return FunctionExecution(self._evaluator, func, self.var_args)
+
+    def _get_func_self_name(self, func):
+        """
+        Returns the name of the first param in a class method (which is
+        normally self.
+        """
+        try:
+            return str(func.params[0].name)
+        except IndexError:
+            return None
+
+    def _self_names_dict(self, add_mro=True):
+        names = {}
+        # This loop adds the names of the self object, copies them and removes
+        # the self.
+        for sub in self.base.subscopes:
+            if isinstance(sub, tree.Class):
+                continue
+            # Get the self name, if there's one.
+            self_name = self._get_func_self_name(sub)
+            if self_name is None:
+                continue
+
+            if sub.name.value == '__init__' and not self.is_generated:
+                # ``__init__`` is special because the params need are injected
+                # this way. Therefore an execution is necessary.
+                if not sub.get_decorators():
+                    # __init__ decorators should generally just be ignored,
+                    # because to follow them and their self variables is too
+                    # complicated.
+                    sub = self._get_method_execution(sub)
+            for name_list in sub.names_dict.values():
+                for name in name_list:
+                    if name.value == self_name and name.get_previous_sibling() is None:
+                        trailer = name.get_next_sibling()
+                        if tree.is_node(trailer, 'trailer') \
+                                and len(trailer.children) == 2 \
+                                and trailer.children[0] == '.':
+                            name = trailer.children[1]  # After dot.
+                            if name.is_definition():
+                                arr = names.setdefault(name.value, [])
+                                arr.append(get_instance_el(self._evaluator, self, name))
+        return names
+
+    def get_subscope_by_name(self, name):
+        sub = self.base.get_subscope_by_name(name)
+        return get_instance_el(self._evaluator, self, sub, True)
+
+    def execute_subscope_by_name(self, name, *args):
+        method = self.get_subscope_by_name(name)
+        return self._evaluator.execute_evaluated(method, *args)
+
+    def get_descriptor_returns(self, obj):
+        """ Throws a KeyError if there's no method. """
+        # Arguments in __get__ descriptors are obj, class.
+        # `method` is the new parent of the array, don't know if that's good.
+        none_obj = compiled.create(self._evaluator, None)
+        args = [obj, obj.base] if isinstance(obj, Instance) else [none_obj, obj]
+        try:
+            return self.execute_subscope_by_name('__get__', *args)
+        except KeyError:
+            return set([self])
+
+    @memoize_default()
+    def names_dicts(self, search_global):
+        yield self._self_names_dict()
+
+        for s in self.base.py__mro__()[1:]:
+            if not isinstance(s, compiled.CompiledObject):
+                # Compiled objects don't have `self.` names.
+                for inst in self._evaluator.execute(s):
+                    yield inst._self_names_dict(add_mro=False)
+
+        for names_dict in self.base.names_dicts(search_global=False, is_instance=True):
+            yield LazyInstanceDict(self._evaluator, self, names_dict)
+
+    def py__getitem__(self, index):
+        try:
+            method = self.get_subscope_by_name('__getitem__')
+        except KeyError:
+            debug.warning('No __getitem__, cannot access the array.')
+            return set()
+        else:
+            index_obj = compiled.create(self._evaluator, index)
+            return self._evaluator.execute_evaluated(method, index_obj)
+
+    def py__iter__(self):
+        try:
+            method = self.get_subscope_by_name('__iter__')
+        except KeyError:
+            debug.warning('No __iter__ on %s.' % self)
+            return
+        else:
+            iters = self._evaluator.execute(method)
+            for generator in iters:
+                if isinstance(generator, Instance):
+                    # `__next__` logic.
+                    name = '__next__' if is_py3 else 'next'
+                    try:
+                        yield generator.execute_subscope_by_name(name)
+                    except KeyError:
+                        debug.warning('Instance has no __next__ function in %s.', generator)
+                else:
+                    for typ in generator.py__iter__():
+                        yield typ
+
+    @property
+    @underscore_memoization
+    def name(self):
+        name = self.base.name
+        return helpers.FakeName(unicode(name), self, name.start_pos)
+
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'end_pos', 'get_imports', 'type',
+                        'doc', 'raw_doc']:
+            raise AttributeError("Instance %s: Don't touch this (%s)!"
+                                 % (self, name))
+        return getattr(self.base, name)
+
+    def __repr__(self):
+        dec = ''
+        if self.decorates is not None:
+            dec = " decorates " + repr(self.decorates)
+        return "<%s of %s(%s)%s>" % (type(self).__name__, self.base,
+                                     self.var_args, dec)
+
+
+class LazyInstanceDict(object):
+    def __init__(self, evaluator, instance, dct):
+        self._evaluator = evaluator
+        self._instance = instance
+        self._dct = dct
+
+    def __getitem__(self, name):
+        return [get_instance_el(self._evaluator, self._instance, var, True)
+                for var in self._dct[name]]
+
+    def values(self):
+        return [self[key] for key in self._dct]
+
+
+class InstanceName(tree.Name):
+    def __init__(self, origin_name, parent):
+        super(InstanceName, self).__init__(tree.zero_position_modifier,
+                                           origin_name.value,
+                                           origin_name.start_pos)
+        self._origin_name = origin_name
+        self.parent = parent
+
+    def is_definition(self):
+        return self._origin_name.is_definition()
+
+
+def get_instance_el(evaluator, instance, var, is_class_var=False):
+    """
+    Returns an InstanceElement if it makes sense, otherwise leaves the object
+    untouched.
+
+    Basically having an InstanceElement is context information. That is needed
+    in quite a lot of cases, which includes Nodes like ``power``, that need to
+    know where a self name comes from for example.
+    """
+    if isinstance(var, tree.Name):
+        parent = get_instance_el(evaluator, instance, var.parent, is_class_var)
+        return InstanceName(var, parent)
+    elif var.type != 'funcdef' \
+            and isinstance(var, (Instance, compiled.CompiledObject, tree.Leaf,
+                           tree.Module, FunctionExecution)):
+        return var
+
+    var = evaluator.wrap(var)
+    return InstanceElement(evaluator, instance, var, is_class_var)
+
+
+class InstanceElement(use_metaclass(CachedMetaClass, tree.Base)):
+    """
+    InstanceElement is a wrapper for any object, that is used as an instance
+    variable (e.g. self.variable or class methods).
+    """
+    def __init__(self, evaluator, instance, var, is_class_var):
+        self._evaluator = evaluator
+        self.instance = instance
+        self.var = var
+        self.is_class_var = is_class_var
+
+    @common.safe_property
+    @memoize_default()
+    def parent(self):
+        par = self.var.parent
+        if isinstance(par, Class) and par == self.instance.base \
+                or isinstance(par, tree.Class) \
+                and par == self.instance.base.base:
+            par = self.instance
+        else:
+            par = get_instance_el(self._evaluator, self.instance, par,
+                                  self.is_class_var)
+        return par
+
+    def get_parent_until(self, *args, **kwargs):
+        return tree.BaseNode.get_parent_until(self, *args, **kwargs)
+
+    def get_definition(self):
+        return self.get_parent_until((tree.ExprStmt, tree.IsScope, tree.Import))
+
+    def get_decorated_func(self):
+        """ Needed because the InstanceElement should not be stripped """
+        func = self.var.get_decorated_func()
+        func = get_instance_el(self._evaluator, self.instance, func)
+        return func
+
+    def get_rhs(self):
+        return get_instance_el(self._evaluator, self.instance,
+                               self.var.get_rhs(), self.is_class_var)
+
+    def is_definition(self):
+        return self.var.is_definition()
+
+    @property
+    def children(self):
+        # Copy and modify the array.
+        return [get_instance_el(self._evaluator, self.instance, command, self.is_class_var)
+                for command in self.var.children]
+
+    @property
+    @memoize_default()
+    def name(self):
+        name = self.var.name
+        return helpers.FakeName(unicode(name), self, name.start_pos)
+
+    def __iter__(self):
+        for el in self.var.__iter__():
+            yield get_instance_el(self._evaluator, self.instance, el,
+                                  self.is_class_var)
+
+    def __getitem__(self, index):
+        return get_instance_el(self._evaluator, self.instance, self.var[index],
+                               self.is_class_var)
+
+    def __getattr__(self, name):
+        return getattr(self.var, name)
+
+    def isinstance(self, *cls):
+        return isinstance(self.var, cls)
+
+    def is_scope(self):
+        """
+        Since we inherit from Base, it would overwrite the action we want here.
+        """
+        return self.var.is_scope()
+
+    def py__call__(self, params):
+        if isinstance(self.var, compiled.CompiledObject):
+            # This check is a bit strange, but CompiledObject itself is a bit
+            # more complicated than we would it actually like to be.
+            return self.var.py__call__(params)
+        else:
+            return Function.py__call__(self, params)
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self.var)
+
+
+class Wrapper(tree.Base):
+    def is_scope(self):
+        return True
+
+    def is_class(self):
+        return False
+
+    def py__bool__(self):
+        """
+        Since Wrapper is a super class for classes, functions and modules,
+        the return value will always be true.
+        """
+        return True
+
+    @property
+    @underscore_memoization
+    def name(self):
+        name = self.base.name
+        return helpers.FakeName(unicode(name), self, name.start_pos)
+
+
+class Class(use_metaclass(CachedMetaClass, Wrapper)):
+    """
+    This class is not only important to extend `tree.Class`, it is also a
+    important for descriptors (if the descriptor methods are evaluated or not).
+    """
+    def __init__(self, evaluator, base):
+        self._evaluator = evaluator
+        self.base = base
+
+    @memoize_default(default=())
+    def py__mro__(self):
+        def add(cls):
+            if cls not in mro:
+                mro.append(cls)
+
+        mro = [self]
+        # TODO Do a proper mro resolution. Currently we are just listing
+        # classes. However, it's a complicated algorithm.
+        for cls in self.py__bases__():
+            # TODO detect for TypeError: duplicate base class str,
+            # e.g.  `class X(str, str): pass`
+            try:
+                mro_method = cls.py__mro__
+            except AttributeError:
+                # TODO add a TypeError like:
+                """
+                >>> class Y(lambda: test): pass
+                Traceback (most recent call last):
+                  File "<stdin>", line 1, in <module>
+                TypeError: function() argument 1 must be code, not str
+                >>> class Y(1): pass
+                Traceback (most recent call last):
+                  File "<stdin>", line 1, in <module>
+                TypeError: int() takes at most 2 arguments (3 given)
+                """
+                pass
+            else:
+                add(cls)
+                for cls_new in mro_method():
+                    add(cls_new)
+        return tuple(mro)
+
+    @memoize_default(default=())
+    def py__bases__(self):
+        arglist = self.base.get_super_arglist()
+        if arglist:
+            args = param.Arguments(self._evaluator, arglist)
+            return list(chain.from_iterable(args.eval_args()))
+        else:
+            return [compiled.create(self._evaluator, object)]
+
+    def py__call__(self, params):
+        return set([Instance(self._evaluator, self, params)])
+
+    def py__class__(self):
+        return compiled.create(self._evaluator, type)
+
+    @property
+    def params(self):
+        try:
+            return self.get_subscope_by_name('__init__').params
+        except KeyError:
+            return []  # object.__init__
+
+    def names_dicts(self, search_global, is_instance=False):
+        if search_global:
+            yield self.names_dict
+        else:
+            for scope in self.py__mro__():
+                if isinstance(scope, compiled.CompiledObject):
+                    yield scope.names_dicts(False, is_instance)[0]
+                else:
+                    yield scope.names_dict
+
+    def is_class(self):
+        return True
+
+    def get_subscope_by_name(self, name):
+        for s in self.py__mro__():
+            for sub in reversed(s.subscopes):
+                if sub.name.value == name:
+                    return sub
+        raise KeyError("Couldn't find subscope.")
+
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'end_pos', 'parent', 'raw_doc',
+                        'doc', 'get_imports', 'get_parent_until', 'get_code',
+                        'subscopes', 'names_dict', 'type']:
+            raise AttributeError("Don't touch this: %s of %s !" % (name, self))
+        return getattr(self.base, name)
+
+    def __repr__(self):
+        return "<e%s of %s>" % (type(self).__name__, self.base)
+
+
+class Function(use_metaclass(CachedMetaClass, Wrapper)):
+    """
+    Needed because of decorators. Decorators are evaluated here.
+    """
+    def __init__(self, evaluator, func, is_decorated=False):
+        """ This should not be called directly """
+        self._evaluator = evaluator
+        self.base = self.base_func = func
+        self.is_decorated = is_decorated
+        # A property that is set by the decorator resolution.
+        self.decorates = None
+
+    @memoize_default()
+    def get_decorated_func(self):
+        """
+        Returns the function, that should to be executed in the end.
+        This is also the places where the decorators are processed.
+        """
+        f = self.base_func
+        decorators = self.base_func.get_decorators()
+
+        if not decorators or self.is_decorated:
+            return self
+
+        # Only enter it, if has not already been processed.
+        if not self.is_decorated:
+            for dec in reversed(decorators):
+                debug.dbg('decorator: %s %s', dec, f)
+                dec_results = self._evaluator.eval_element(dec.children[1])
+                trailer = dec.children[2:-1]
+                if trailer:
+                    # Create a trailer and evaluate it.
+                    trailer = tree.Node('trailer', trailer)
+                    trailer.parent = dec
+                    dec_results = self._evaluator.eval_trailer(dec_results, trailer)
+
+                if not len(dec_results):
+                    debug.warning('decorator not found: %s on %s', dec, self.base_func)
+                    return self
+                decorator = dec_results.pop()
+                if dec_results:
+                    debug.warning('multiple decorators found %s %s',
+                                  self.base_func, dec_results)
+
+                # Create param array.
+                if isinstance(f, Function):
+                    old_func = f  # TODO this is just hacky. change.
+                elif f.type == 'funcdef':
+                    old_func = Function(self._evaluator, f, is_decorated=True)
+                else:
+                    old_func = f
+
+                wrappers = self._evaluator.execute_evaluated(decorator, old_func)
+                if not len(wrappers):
+                    debug.warning('no wrappers found %s', self.base_func)
+                    return self
+                if len(wrappers) > 1:
+                    # TODO resolve issue with multiple wrappers -> multiple types
+                    debug.warning('multiple wrappers found %s %s',
+                                  self.base_func, wrappers)
+                f = list(wrappers)[0]
+                if isinstance(f, (Instance, Function)):
+                    f.decorates = self
+
+                debug.dbg('decorator end %s', f)
+        return f
+
+    def names_dicts(self, search_global):
+        if search_global:
+            yield self.names_dict
+        else:
+            scope = self.py__class__()
+            for names_dict in scope.names_dicts(False):
+                yield names_dict
+
+    @Python3Method
+    def py__call__(self, params):
+        if self.base.is_generator():
+            return set([iterable.Generator(self._evaluator, self, params)])
+        else:
+            return FunctionExecution(self._evaluator, self, params).get_return_types()
+
+    @memoize_default()
+    def py__annotations__(self):
+        parser_func = self.base
+        return_annotation = parser_func.annotation()
+        if return_annotation:
+            dct = {'return': return_annotation}
+        else:
+            dct = {}
+        for function_param in parser_func.params:
+            param_annotation = function_param.annotation()
+            if param_annotation is not None:
+                dct[function_param.name.value] = param_annotation
+        return dct
+
+    def py__class__(self):
+        # This differentiation is only necessary for Python2. Python3 does not
+        # use a different method class.
+        if isinstance(self.base.get_parent_scope(), tree.Class):
+            name = 'METHOD_CLASS'
+        else:
+            name = 'FUNCTION_CLASS'
+        return compiled.get_special_object(self._evaluator, name)
+
+    def __getattr__(self, name):
+        return getattr(self.base_func, name)
+
+    def __repr__(self):
+        dec = ''
+        if self.decorates is not None:
+            dec = " decorates " + repr(self.decorates)
+        return "<e%s of %s%s>" % (type(self).__name__, self.base_func, dec)
+
+
+class LambdaWrapper(Function):
+    def get_decorated_func(self):
+        return self
+
+
+class FunctionExecution(Executed):
+    """
+    This class is used to evaluate functions and their returns.
+
+    This is the most complicated class, because it contains the logic to
+    transfer parameters. It is even more complicated, because there may be
+    multiple calls to functions and recursion has to be avoided. But this is
+    responsibility of the decorators.
+    """
+    type = 'funcdef'
+
+    def __init__(self, evaluator, base, *args, **kwargs):
+        super(FunctionExecution, self).__init__(evaluator, base, *args, **kwargs)
+        self._copy_dict = {}
+        funcdef = base.base_func
+        if isinstance(funcdef, mixed.MixedObject):
+            # The extra information in mixed is not needed anymore. We can just
+            # unpack it and give it the tree object.
+            funcdef = funcdef.definition
+
+        # Just overwrite the old version. We don't need it anymore.
+        funcdef = helpers.deep_ast_copy(funcdef, new_elements=self._copy_dict)
+        for child in funcdef.children:
+            if child.type not in ('operator', 'keyword'):
+                # Not all nodes are properly copied by deep_ast_copy.
+                child.parent = self
+        self.children = funcdef.children
+        self.names_dict = funcdef.names_dict
+
+    @memoize_default(default=set())
+    @recursion.execution_recursion_decorator
+    def get_return_types(self, check_yields=False):
+        func = self.base
+
+        if func.isinstance(LambdaWrapper):
+            return self._evaluator.eval_element(self.children[-1])
+
+        if func.listeners:
+            # Feed the listeners, with the params.
+            for listener in func.listeners:
+                listener.execute(self._get_params())
+            # If we do have listeners, that means that there's not a regular
+            # execution ongoing. In this case Jedi is interested in the
+            # inserted params, not in the actual execution of the function.
+            return set()
+
+        if check_yields:
+            types = set()
+            returns = self.yields
+        else:
+            returns = self.returns
+            types = set(docstrings.find_return_types(self._evaluator, func))
+            types |= set(pep0484.find_return_types(self._evaluator, func))
+
+        for r in returns:
+            check = flow_analysis.break_check(self._evaluator, self, r)
+            if check is flow_analysis.UNREACHABLE:
+                debug.dbg('Return unreachable: %s', r)
+            else:
+                if check_yields:
+                    types |= iterable.unite(self._eval_yield(r))
+                else:
+                    types |= self._evaluator.eval_element(r.children[1])
+            if check is flow_analysis.REACHABLE:
+                debug.dbg('Return reachable: %s', r)
+                break
+        return types
+
+    def _eval_yield(self, yield_expr):
+        element = yield_expr.children[1]
+        if element.type == 'yield_arg':
+            # It must be a yield from.
+            yield_from_types = self._evaluator.eval_element(element.children[1])
+            for result in iterable.py__iter__(self._evaluator, yield_from_types, element):
+                yield result
+        else:
+            yield self._evaluator.eval_element(element)
+
+    @recursion.execution_recursion_decorator
+    def get_yield_types(self):
+        yields = self.yields
+        stopAt = tree.ForStmt, tree.WhileStmt, FunctionExecution, tree.IfStmt
+        for_parents = [(x, x.get_parent_until((stopAt))) for x in yields]
+
+        # Calculate if the yields are placed within the same for loop.
+        yields_order = []
+        last_for_stmt = None
+        for yield_, for_stmt in for_parents:
+            # For really simple for loops we can predict the order. Otherwise
+            # we just ignore it.
+            parent = for_stmt.parent
+            if parent.type == 'suite':
+                parent = parent.parent
+            if for_stmt.type == 'for_stmt' and parent == self \
+                    and for_stmt.defines_one_name():  # Simplicity for now.
+                if for_stmt == last_for_stmt:
+                    yields_order[-1][1].append(yield_)
+                else:
+                    yields_order.append((for_stmt, [yield_]))
+            elif for_stmt == self:
+                yields_order.append((None, [yield_]))
+            else:
+                yield self.get_return_types(check_yields=True)
+                return
+            last_for_stmt = for_stmt
+
+        evaluator = self._evaluator
+        for for_stmt, yields in yields_order:
+            if for_stmt is None:
+                # No for_stmt, just normal yields.
+                for yield_ in yields:
+                    for result in self._eval_yield(yield_):
+                        yield result
+            else:
+                input_node = for_stmt.get_input_node()
+                for_types = evaluator.eval_element(input_node)
+                ordered = iterable.py__iter__(evaluator, for_types, input_node)
+                for index_types in ordered:
+                    dct = {str(for_stmt.children[1]): index_types}
+                    evaluator.predefined_if_name_dict_dict[for_stmt] = dct
+                    for yield_in_same_for_stmt in yields:
+                        for result in self._eval_yield(yield_in_same_for_stmt):
+                            yield result
+                    del evaluator.predefined_if_name_dict_dict[for_stmt]
+
+    def names_dicts(self, search_global):
+        yield self.names_dict
+
+    @memoize_default(default=NO_DEFAULT)
+    def _get_params(self):
+        """
+        This returns the params for an TODO and is injected as a
+        'hack' into the tree.Function class.
+        This needs to be here, because Instance can have __init__ functions,
+        which act the same way as normal functions.
+        """
+        return param.get_params(self._evaluator, self.base, self.var_args)
+
+    def param_by_name(self, name):
+        return [n for n in self._get_params() if str(n) == name][0]
+
+    def name_for_position(self, position):
+        return tree.Function.name_for_position(self, position)
+
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'end_pos', 'imports', 'name', 'type']:
+            raise AttributeError('Tried to access %s: %s. Why?' % (name, self))
+        return getattr(self.base, name)
+
+    @common.safe_property
+    @memoize_default()
+    def returns(self):
+        return tree.Scope._search_in_scope(self, tree.ReturnStmt)
+
+    @common.safe_property
+    @memoize_default()
+    def yields(self):
+        return tree.Scope._search_in_scope(self, tree.YieldExpr)
+
+    @common.safe_property
+    @memoize_default()
+    def statements(self):
+        return tree.Scope._search_in_scope(self, tree.ExprStmt)
+
+    @common.safe_property
+    @memoize_default()
+    def subscopes(self):
+        return tree.Scope._search_in_scope(self, tree.Scope)
+
+    def __repr__(self):
+        return "<%s of %s>" % (type(self).__name__, self.base)
+
+
+class GlobalName(helpers.FakeName):
+    def __init__(self, name):
+        """
+        We need to mark global names somehow. Otherwise they are just normal
+        names that are not definitions.
+        """
+        super(GlobalName, self).__init__(name.value, name.parent,
+                                         name.start_pos, is_definition=True)
+
+
+class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)):
+    def __init__(self, evaluator, module, parent_module=None):
+        self._evaluator = evaluator
+        self.base = self._module = module
+        self._parent_module = parent_module
+
+    def names_dicts(self, search_global):
+        yield self.base.names_dict
+        yield self._module_attributes_dict()
+
+        for star_module in self.star_imports():
+            yield star_module.names_dict
+
+        yield dict((str(n), [GlobalName(n)]) for n in self.base.global_names)
+        yield self._sub_modules_dict()
+
+    # I'm not sure if the star import cache is really that effective anymore
+    # with all the other really fast import caches. Recheck. Also we would need
+    # to push the star imports into Evaluator.modules, if we reenable this.
+    #@cache_star_import
+    @memoize_default([])
+    def star_imports(self):
+        modules = []
+        for i in self.base.imports:
+            if i.is_star_import():
+                name = i.star_import_name()
+                new = imports.ImportWrapper(self._evaluator, name).follow()
+                for module in new:
+                    if isinstance(module, tree.Module):
+                        modules += module.star_imports()
+                modules += new
+        return modules
+
+    @memoize_default()
+    def _module_attributes_dict(self):
+        def parent_callback():
+            # Create a string type object (without a defined string in it):
+            return list(self._evaluator.execute(compiled.create(self._evaluator, str)))[0]
+
+        names = ['__file__', '__package__', '__doc__', '__name__']
+        # All the additional module attributes are strings.
+        return dict((n, [helpers.LazyName(n, parent_callback, is_definition=True)])
+                    for n in names)
+
+    @property
+    @memoize_default()
+    def name(self):
+        return helpers.FakeName(unicode(self.base.name), self, (1, 0))
+
+    def _get_init_directory(self):
+        """
+        :return: The path to the directory of a package. None in case it's not
+                 a package.
+        """
+        for suffix, _, _ in imp.get_suffixes():
+            ending = '__init__' + suffix
+            py__file__ = self.py__file__()
+            if py__file__ is not None and py__file__.endswith(ending):
+                # Remove the ending, including the separator.
+                return self.py__file__()[:-len(ending) - 1]
+        return None
+
+    def py__name__(self):
+        for name, module in self._evaluator.modules.items():
+            if module == self:
+                return name
+
+        return '__main__'
+
+    def py__file__(self):
+        """
+        In contrast to Python's __file__ can be None.
+        """
+        if self._module.path is None:
+            return None
+
+        return os.path.abspath(self._module.path)
+
+    def py__package__(self):
+        if self._get_init_directory() is None:
+            return re.sub(r'\.?[^\.]+$', '', self.py__name__())
+        else:
+            return self.py__name__()
+
+    def _py__path__(self):
+        if self._parent_module is None:
+            search_path = self._evaluator.sys_path
+        else:
+            search_path = self._parent_module.py__path__()
+        init_path = self.py__file__()
+        if os.path.basename(init_path) == '__init__.py':
+            with open(init_path, 'rb') as f:
+                content = common.source_to_unicode(f.read())
+                # these are strings that need to be used for namespace packages,
+                # the first one is ``pkgutil``, the second ``pkg_resources``.
+                options = ('declare_namespace(__name__)', 'extend_path(__path__')
+                if options[0] in content or options[1] in content:
+                    # It is a namespace, now try to find the rest of the
+                    # modules on sys_path or whatever the search_path is.
+                    paths = set()
+                    for s in search_path:
+                        other = os.path.join(s, unicode(self.name))
+                        if os.path.isdir(other):
+                            paths.add(other)
+                    return list(paths)
+        # Default to this.
+        return [self._get_init_directory()]
+
+    @property
+    def py__path__(self):
+        """
+        Not seen here, since it's a property. The callback actually uses a
+        variable, so use it like::
+
+            foo.py__path__(sys_path)
+
+        In case of a package, this returns Python's __path__ attribute, which
+        is a list of paths (strings).
+        Raises an AttributeError if the module is not a package.
+        """
+        path = self._get_init_directory()
+
+        if path is None:
+            raise AttributeError('Only packages have __path__ attributes.')
+        else:
+            return self._py__path__
+
+    @memoize_default()
+    def _sub_modules_dict(self):
+        """
+        Lists modules in the directory of this module (if this module is a
+        package).
+        """
+        path = self._module.path
+        names = {}
+        if path is not None and path.endswith(os.path.sep + '__init__.py'):
+            mods = pkgutil.iter_modules([os.path.dirname(path)])
+            for module_loader, name, is_pkg in mods:
+                fake_n = helpers.FakeName(name)
+                # It's obviously a relative import to the current module.
+                imp = helpers.FakeImport(fake_n, self, level=1)
+                fake_n.parent = imp
+                names[name] = [fake_n]
+
+        # TODO add something like this in the future, its cleaner than the
+        #   import hacks.
+        # ``os.path`` is a hardcoded exception, because it's a
+        # ``sys.modules`` modification.
+        #if str(self.name) == 'os':
+        #    names.append(helpers.FakeName('path', parent=self))
+
+        return names
+
+    def py__class__(self):
+        return compiled.get_special_object(self._evaluator, 'MODULE_CLASS')
+
+    def __getattr__(self, name):
+        return getattr(self._module, name)
+
+    def __repr__(self):
+        return "<%s: %s>" % (type(self).__name__, self._module)

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/site.py

@@ -0,0 +1,110 @@
+"""An adapted copy of relevant site-packages functionality from Python stdlib.
+
+This file contains some functions related to handling site-packages in Python
+with jedi-specific modifications:
+
+- the functions operate on sys_path argument rather than global sys.path
+
+- in .pth files "import ..." lines that allow execution of arbitrary code are
+  skipped to prevent code injection into jedi interpreter
+
+"""
+
+# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved
+
+from __future__ import print_function
+
+import sys
+import os
+
+
+def makepath(*paths):
+    dir = os.path.join(*paths)
+    try:
+        dir = os.path.abspath(dir)
+    except OSError:
+        pass
+    return dir, os.path.normcase(dir)
+
+
+def _init_pathinfo(sys_path):
+    """Return a set containing all existing directory entries from sys_path"""
+    d = set()
+    for dir in sys_path:
+        try:
+            if os.path.isdir(dir):
+                dir, dircase = makepath(dir)
+                d.add(dircase)
+        except TypeError:
+            continue
+    return d
+
+
+def addpackage(sys_path, sitedir, name, known_paths):
+    """Process a .pth file within the site-packages directory:
+       For each line in the file, either combine it with sitedir to a path
+       and add that to known_paths, or execute it if it starts with 'import '.
+    """
+    if known_paths is None:
+        known_paths = _init_pathinfo(sys_path)
+        reset = 1
+    else:
+        reset = 0
+    fullname = os.path.join(sitedir, name)
+    try:
+        f = open(fullname, "r")
+    except OSError:
+        return
+    with f:
+        for n, line in enumerate(f):
+            if line.startswith("#"):
+                continue
+            try:
+                if line.startswith(("import ", "import\t")):
+                    # Change by immerrr: don't evaluate import lines to prevent
+                    # code injection into jedi through pth files.
+                    #
+                    # exec(line)
+                    continue
+                line = line.rstrip()
+                dir, dircase = makepath(sitedir, line)
+                if not dircase in known_paths and os.path.exists(dir):
+                    sys_path.append(dir)
+                    known_paths.add(dircase)
+            except Exception:
+                print("Error processing line {:d} of {}:\n".format(n+1, fullname),
+                      file=sys.stderr)
+                import traceback
+                for record in traceback.format_exception(*sys.exc_info()):
+                    for line in record.splitlines():
+                        print('  '+line, file=sys.stderr)
+                print("\nRemainder of file ignored", file=sys.stderr)
+                break
+    if reset:
+        known_paths = None
+    return known_paths
+
+
+def addsitedir(sys_path, sitedir, known_paths=None):
+    """Add 'sitedir' argument to sys_path if missing and handle .pth files in
+    'sitedir'"""
+    if known_paths is None:
+        known_paths = _init_pathinfo(sys_path)
+        reset = 1
+    else:
+        reset = 0
+    sitedir, sitedircase = makepath(sitedir)
+    if not sitedircase in known_paths:
+        sys_path.append(sitedir)        # Add path component
+        known_paths.add(sitedircase)
+    try:
+        names = os.listdir(sitedir)
+    except OSError:
+        return
+    names = [name for name in names if name.endswith(".pth")]
+    for name in sorted(names):
+        addpackage(sys_path, sitedir, name, known_paths)
+    if reset:
+        known_paths = None
+    return known_paths

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/stdlib.py

@@ -0,0 +1,280 @@
+"""
+Implementations of standard library functions, because it's not possible to
+understand them with Jedi.
+
+To add a new implementation, create a function and add it to the
+``_implemented`` dict at the bottom of this module.
+
+Note that this module exists only to implement very specific functionality in
+the standard library. The usual way to understand the standard library is the
+compiled module that returns the types for C-builtins.
+"""
+import collections
+import re
+
+from jedi._compatibility import unicode
+from jedi.common import unite
+from jedi.evaluate import compiled
+from jedi.evaluate import representation as er
+from jedi.evaluate import iterable
+from jedi.parser import ParserWithRecovery
+from jedi.parser import tree
+from jedi import debug
+from jedi.evaluate import precedence
+from jedi.evaluate import param
+from jedi.evaluate import analysis
+
+
+class NotInStdLib(LookupError):
+    pass
+
+
+def execute(evaluator, obj, arguments):
+    try:
+        obj_name = str(obj.name)
+    except AttributeError:
+        pass
+    else:
+        if obj.parent == evaluator.BUILTINS:
+            module_name = 'builtins'
+        elif isinstance(obj.parent, tree.Module):
+            module_name = str(obj.parent.name)
+        else:
+            module_name = ''
+
+        # for now we just support builtin functions.
+        try:
+            func = _implemented[module_name][obj_name]
+        except KeyError:
+            pass
+        else:
+            return func(evaluator, obj, arguments)
+    raise NotInStdLib()
+
+
+def _follow_param(evaluator, arguments, index):
+    try:
+        key, values = list(arguments.unpack())[index]
+    except IndexError:
+        return set()
+    else:
+        return unite(evaluator.eval_element(v) for v in values)
+
+
+def argument_clinic(string, want_obj=False, want_scope=False, want_arguments=False):
+    """
+    Works like Argument Clinic (PEP 436), to validate function params.
+    """
+    clinic_args = []
+    allow_kwargs = False
+    optional = False
+    while string:
+        # Optional arguments have to begin with a bracket. And should always be
+        # at the end of the arguments. This is therefore not a proper argument
+        # clinic implementation. `range()` for exmple allows an optional start
+        # value at the beginning.
+        match = re.match('(?:(?:(\[),? ?|, ?|)(\w+)|, ?/)\]*', string)
+        string = string[len(match.group(0)):]
+        if not match.group(2):  # A slash -> allow named arguments
+            allow_kwargs = True
+            continue
+        optional = optional or bool(match.group(1))
+        word = match.group(2)
+        clinic_args.append((word, optional, allow_kwargs))
+
+    def f(func):
+        def wrapper(evaluator, obj, arguments):
+            debug.dbg('builtin start %s' % obj, color='MAGENTA')
+            try:
+                lst = list(arguments.eval_argument_clinic(clinic_args))
+            except ValueError:
+                return set()
+            else:
+                kwargs = {}
+                if want_scope:
+                    kwargs['scope'] = arguments.scope()
+                if want_obj:
+                    kwargs['obj'] = obj
+                if want_arguments:
+                    kwargs['arguments'] = arguments
+                return func(evaluator, *lst, **kwargs)
+            finally:
+                debug.dbg('builtin end', color='MAGENTA')
+
+        return wrapper
+    return f
+
+
+@argument_clinic('object, name[, default], /')
+def builtins_getattr(evaluator, objects, names, defaults=None):
+    # follow the first param
+    for obj in objects:
+        if not isinstance(obj, (er.Instance, er.Class, tree.Module, compiled.CompiledObject)):
+            debug.warning('getattr called without instance')
+            continue
+
+        for name in names:
+            if precedence.is_string(name):
+                return evaluator.find_types(obj, name.obj)
+            else:
+                debug.warning('getattr called without str')
+                continue
+    return set()
+
+
+@argument_clinic('object[, bases, dict], /')
+def builtins_type(evaluator, objects, bases, dicts):
+    if bases or dicts:
+        # It's a type creation... maybe someday...
+        return set()
+    else:
+        return set([o.py__class__() for o in objects])
+
+
+class SuperInstance(er.Instance):
+    """To be used like the object ``super`` returns."""
+    def __init__(self, evaluator, cls):
+        su = cls.py_mro()[1]
+        super().__init__(evaluator, su and su[0] or self)
+
+
+@argument_clinic('[type[, obj]], /', want_scope=True)
+def builtins_super(evaluator, types, objects, scope):
+    # TODO make this able to detect multiple inheritance super
+    accept = (tree.Function, er.FunctionExecution)
+    if scope.isinstance(*accept):
+        wanted = (tree.Class, er.Instance)
+        cls = scope.get_parent_until(accept + wanted,
+                                     include_current=False)
+        if isinstance(cls, wanted):
+            if isinstance(cls, tree.Class):
+                cls = er.Class(evaluator, cls)
+            elif isinstance(cls, er.Instance):
+                cls = cls.base
+            su = cls.py__bases__()
+            if su:
+                return evaluator.execute(su[0])
+    return set()
+
+
+@argument_clinic('sequence, /', want_obj=True, want_arguments=True)
+def builtins_reversed(evaluator, sequences, obj, arguments):
+    # While we could do without this variable (just by using sequences), we
+    # want static analysis to work well. Therefore we need to generated the
+    # values again.
+    first_arg = next(arguments.as_tuple())[0]
+    ordered = list(iterable.py__iter__(evaluator, sequences, first_arg))
+
+    rev = [iterable.AlreadyEvaluated(o) for o in reversed(ordered)]
+    # Repack iterator values and then run it the normal way. This is
+    # necessary, because `reversed` is a function and autocompletion
+    # would fail in certain cases like `reversed(x).__iter__` if we
+    # just returned the result directly.
+    rev = iterable.AlreadyEvaluated(
+        [iterable.FakeSequence(evaluator, rev, 'list')]
+    )
+    return set([er.Instance(evaluator, obj, param.Arguments(evaluator, [rev]))])
+
+
+@argument_clinic('obj, type, /', want_arguments=True)
+def builtins_isinstance(evaluator, objects, types, arguments):
+    bool_results = set([])
+    for o in objects:
+        try:
+            mro_func = o.py__class__().py__mro__
+        except AttributeError:
+            # This is temporary. Everything should have a class attribute in
+            # Python?! Maybe we'll leave it here, because some numpy objects or
+            # whatever might not.
+            return set([compiled.create(True), compiled.create(False)])
+
+        mro = mro_func()
+
+        for cls_or_tup in types:
+            if cls_or_tup.is_class():
+                bool_results.add(cls_or_tup in mro)
+            elif str(cls_or_tup.name) == 'tuple' \
+                    and cls_or_tup.get_parent_scope() == evaluator.BUILTINS:
+                # Check for tuples.
+                classes = unite(cls_or_tup.py__iter__())
+                bool_results.add(any(cls in mro for cls in classes))
+            else:
+                _, nodes = list(arguments.unpack())[1]
+                for node in nodes:
+                    message = 'TypeError: isinstance() arg 2 must be a ' \
+                              'class, type, or tuple of classes and types, ' \
+                              'not %s.' % cls_or_tup
+                    analysis.add(evaluator, 'type-error-isinstance', node, message)
+
+    return set(compiled.create(evaluator, x) for x in bool_results)
+
+
+def collections_namedtuple(evaluator, obj, arguments):
+    """
+    Implementation of the namedtuple function.
+
+    This has to be done by processing the namedtuple class template and
+    evaluating the result.
+
+    .. note:: |jedi| only supports namedtuples on Python >2.6.
+
+    """
+    # Namedtuples are not supported on Python 2.6
+    if not hasattr(collections, '_class_template'):
+        return set()
+
+    # Process arguments
+    # TODO here we only use one of the types, we should use all.
+    name = list(_follow_param(evaluator, arguments, 0))[0].obj
+    _fields = list(_follow_param(evaluator, arguments, 1))[0]
+    if isinstance(_fields, compiled.CompiledObject):
+        fields = _fields.obj.replace(',', ' ').split()
+    elif isinstance(_fields, iterable.Array):
+        try:
+            fields = [v.obj for v in unite(_fields.py__iter__())]
+        except AttributeError:
+            return set()
+    else:
+        return set()
+
+    # Build source
+    source = collections._class_template.format(
+        typename=name,
+        field_names=fields,
+        num_fields=len(fields),
+        arg_list=', '.join(fields),
+        repr_fmt=', '.join(collections._repr_template.format(name=name) for name in fields),
+        field_defs='\n'.join(collections._field_template.format(index=index, name=name)
+                             for index, name in enumerate(fields))
+    )
+
+    # Parse source
+    generated_class = ParserWithRecovery(evaluator.grammar, unicode(source)).module.subscopes[0]
+    return set([er.Class(evaluator, generated_class)])
+
+
+@argument_clinic('first, /')
+def _return_first_param(evaluator, firsts):
+    return firsts
+
+
+_implemented = {
+    'builtins': {
+        'getattr': builtins_getattr,
+        'type': builtins_type,
+        'super': builtins_super,
+        'reversed': builtins_reversed,
+        'isinstance': builtins_isinstance,
+    },
+    'copy': {
+        'copy': _return_first_param,
+        'deepcopy': _return_first_param,
+    },
+    'json': {
+        'load': lambda *args: set(),
+        'loads': lambda *args: set(),
+    },
+    'collections': {
+        'namedtuple': collections_namedtuple,
+    },
+}

vim-plugins/bundle/jedi-vim/jedi/jedi/evaluate/sys_path.py

@@ -0,0 +1,284 @@
+import glob
+import os
+import sys
+from jedi.evaluate.site import addsitedir
+
+from jedi._compatibility import exec_function, unicode
+from jedi.parser import tree
+from jedi.parser import ParserWithRecovery
+from jedi.evaluate.cache import memoize_default
+from jedi import debug
+from jedi import common
+from jedi.parser.utils import load_parser, save_parser
+
+
+def get_venv_path(venv):
+    """Get sys.path for specified virtual environment."""
+    sys_path = _get_venv_path_dirs(venv)
+    with common.ignored(ValueError):
+        sys_path.remove('')
+    sys_path = _get_sys_path_with_egglinks(sys_path)
+    # As of now, get_venv_path_dirs does not scan built-in pythonpath and
+    # user-local site-packages, let's approximate them using path from Jedi
+    # interpreter.
+    return sys_path + sys.path
+
+
+
+def _get_sys_path_with_egglinks(sys_path):
+    """Find all paths including those referenced by egg-links.
+
+    Egg-link-referenced directories are inserted into path immediately before
+    the directory on which their links were found.  Such directories are not
+    taken into consideration by normal import mechanism, but they are traversed
+    when doing pkg_resources.require.
+    """
+    result = []
+    for p in sys_path:
+        # pkg_resources does not define a specific order for egg-link files
+        # using os.listdir to enumerate them, we're sorting them to have
+        # reproducible tests.
+        for egg_link in sorted(glob.glob(os.path.join(p, '*.egg-link'))):
+            with open(egg_link) as fd:
+                for line in fd:
+                    line = line.strip()
+                    if line:
+                        result.append(os.path.join(p, line))
+                        # pkg_resources package only interprets the first
+                        # non-empty line in egg-link files.
+                        break
+        result.append(p)
+    return result
+
+
+def _get_venv_path_dirs(venv):
+    """Get sys.path for venv without starting up the interpreter."""
+    venv = os.path.abspath(venv)
+    sitedir = _get_venv_sitepackages(venv)
+    sys_path = []
+    addsitedir(sys_path, sitedir)
+    return sys_path
+
+
+def _get_venv_sitepackages(venv):
+    if os.name == 'nt':
+        p = os.path.join(venv, 'lib', 'site-packages')
+    else:
+        p = os.path.join(venv, 'lib', 'python%d.%d' % sys.version_info[:2],
+                         'site-packages')
+    return p
+
+
+def _execute_code(module_path, code):
+    c = "import os; from os.path import *; result=%s"
+    variables = {'__file__': module_path}
+    try:
+        exec_function(c % code, variables)
+    except Exception:
+        debug.warning('sys.path manipulation detected, but failed to evaluate.')
+    else:
+        try:
+            res = variables['result']
+            if isinstance(res, str):
+                return [os.path.abspath(res)]
+        except KeyError:
+            pass
+    return []
+
+
+def _paths_from_assignment(evaluator, expr_stmt):
+    """
+    Extracts the assigned strings from an assignment that looks as follows::
+
+    >>> sys.path[0:0] = ['module/path', 'another/module/path']
+
+    This function is in general pretty tolerant (and therefore 'buggy').
+    However, it's not a big issue usually to add more paths to Jedi's sys_path,
+    because it will only affect Jedi in very random situations and by adding
+    more paths than necessary, it usually benefits the general user.
+    """
+    for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]):
+        try:
+            assert operator in ['=', '+=']
+            assert tree.is_node(assignee, 'power', 'atom_expr') and \
+                len(assignee.children) > 1
+            c = assignee.children
+            assert c[0].type == 'name' and c[0].value == 'sys'
+            trailer = c[1]
+            assert trailer.children[0] == '.' and trailer.children[1].value == 'path'
+            # TODO Essentially we're not checking details on sys.path
+            # manipulation. Both assigment of the sys.path and changing/adding
+            # parts of the sys.path are the same: They get added to the current
+            # sys.path.
+            """
+            execution = c[2]
+            assert execution.children[0] == '['
+            subscript = execution.children[1]
+            assert subscript.type == 'subscript'
+            assert ':' in subscript.children
+            """
+        except AssertionError:
+            continue
+
+        from jedi.evaluate.iterable import py__iter__
+        from jedi.evaluate.precedence import is_string
+        types = evaluator.eval_element(expr_stmt)
+        for types in py__iter__(evaluator, types, expr_stmt):
+            for typ in types:
+                if is_string(typ):
+                    yield typ.obj
+
+
+def _paths_from_list_modifications(module_path, trailer1, trailer2):
+    """ extract the path from either "sys.path.append" or "sys.path.insert" """
+    # Guarantee that both are trailers, the first one a name and the second one
+    # a function execution with at least one param.
+    if not (tree.is_node(trailer1, 'trailer') and trailer1.children[0] == '.'
+            and tree.is_node(trailer2, 'trailer') and trailer2.children[0] == '('
+            and len(trailer2.children) == 3):
+        return []
+
+    name = trailer1.children[1].value
+    if name not in ['insert', 'append']:
+        return []
+    arg = trailer2.children[1]
+    if name == 'insert' and len(arg.children) in (3, 4):  # Possible trailing comma.
+        arg = arg.children[2]
+    return _execute_code(module_path, arg.get_code())
+
+
+def _check_module(evaluator, module):
+    """
+    Detect sys.path modifications within module.
+    """
+    def get_sys_path_powers(names):
+        for name in names:
+            power = name.parent.parent
+            if tree.is_node(power, 'power', 'atom_expr'):
+                c = power.children
+                if isinstance(c[0], tree.Name) and c[0].value == 'sys' \
+                        and tree.is_node(c[1], 'trailer'):
+                    n = c[1].children[1]
+                    if isinstance(n, tree.Name) and n.value == 'path':
+                        yield name, power
+
+    sys_path = list(evaluator.sys_path)  # copy
+    try:
+        possible_names = module.used_names['path']
+    except KeyError:
+        # module.used_names is MergedNamesDict whose getitem never throws
+        # keyerror, this is superfluous.
+        pass
+    else:
+        for name, power in get_sys_path_powers(possible_names):
+            stmt = name.get_definition()
+            if len(power.children) >= 4:
+                sys_path.extend(_paths_from_list_modifications(module.path, *power.children[2:4]))
+            elif name.get_definition().type == 'expr_stmt':
+                sys_path.extend(_paths_from_assignment(evaluator, stmt))
+    return sys_path
+
+
+@memoize_default(evaluator_is_first_arg=True, default=[])
+def sys_path_with_modifications(evaluator, module):
+    if module.path is None:
+        # Support for modules without a path is bad, therefore return the
+        # normal path.
+        return list(evaluator.sys_path)
+
+    curdir = os.path.abspath(os.curdir)
+    #TODO why do we need a chdir?
+    with common.ignored(OSError):
+        os.chdir(os.path.dirname(module.path))
+
+    buildout_script_paths = set()
+
+    result = _check_module(evaluator, module)
+    result += _detect_django_path(module.path)
+    for buildout_script in _get_buildout_scripts(module.path):
+        for path in _get_paths_from_buildout_script(evaluator, buildout_script):
+            buildout_script_paths.add(path)
+    # cleanup, back to old directory
+    os.chdir(curdir)
+    return list(result) + list(buildout_script_paths)
+
+
+def _get_paths_from_buildout_script(evaluator, buildout_script):
+    def load(buildout_script):
+        try:
+            with open(buildout_script, 'rb') as f:
+                source = common.source_to_unicode(f.read())
+        except IOError:
+            debug.dbg('Error trying to read buildout_script: %s', buildout_script)
+            return
+
+        p = ParserWithRecovery(evaluator.grammar, source, buildout_script)
+        save_parser(buildout_script, p)
+        return p.module
+
+    cached = load_parser(buildout_script)
+    module = cached and cached.module or load(buildout_script)
+    if not module:
+        return
+
+    for path in _check_module(evaluator, module):
+        yield path
+
+
+def traverse_parents(path):
+    while True:
+        new = os.path.dirname(path)
+        if new == path:
+            return
+        path = new
+        yield path
+
+
+def _get_parent_dir_with_file(path, filename):
+    for parent in traverse_parents(path):
+        if os.path.isfile(os.path.join(parent, filename)):
+            return parent
+    return None
+
+
+def _detect_django_path(module_path):
+    """ Detects the path of the very well known Django library (if used) """
+    result = []
+
+    for parent in traverse_parents(module_path):
+        with common.ignored(IOError):
+            with open(parent + os.path.sep + 'manage.py'):
+                debug.dbg('Found django path: %s', module_path)
+                result.append(parent)
+    return result
+
+
+def _get_buildout_scripts(module_path):
+    """
+    if there is a 'buildout.cfg' file in one of the parent directories of the
+    given module it will return a list of all files in the buildout bin
+    directory that look like python files.
+
+    :param module_path: absolute path to the module.
+    :type module_path: str
+    """
+    project_root = _get_parent_dir_with_file(module_path, 'buildout.cfg')
+    if not project_root:
+        return []
+    bin_path = os.path.join(project_root, 'bin')
+    if not os.path.exists(bin_path):
+        return []
+    extra_module_paths = []
+    for filename in os.listdir(bin_path):
+        try:
+            filepath = os.path.join(bin_path, filename)
+            with open(filepath, 'r') as f:
+                firstline = f.readline()
+                if firstline.startswith('#!') and 'python' in firstline:
+                    extra_module_paths.append(filepath)
+        except (UnicodeDecodeError, IOError) as e:
+            # Probably a binary file; permission error or race cond. because file got deleted
+            # ignore
+            debug.warning(unicode(e))
+            continue
+    return extra_module_paths

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/__init__.py

@@ -0,0 +1,409 @@
+"""
+The ``Parser`` tries to convert the available Python code in an easy to read
+format, something like an abstract syntax tree. The classes who represent this
+tree, are sitting in the :mod:`jedi.parser.tree` module.
+
+The Python module ``tokenize`` is a very important part in the ``Parser``,
+because it splits the code into different words (tokens).  Sometimes it looks a
+bit messy. Sorry for that! You might ask now: "Why didn't you use the ``ast``
+module for this? Well, ``ast`` does a very good job understanding proper Python
+code, but fails to work as soon as there's a single line of broken code.
+
+There's one important optimization that needs to be known: Statements are not
+being parsed completely. ``Statement`` is just a representation of the tokens
+within the statement. This lowers memory usage and cpu time and reduces the
+complexity of the ``Parser`` (there's another parser sitting inside
+``Statement``, which produces ``Array`` and ``Call``).
+"""
+import os
+import re
+
+from jedi.parser import tree as pt
+from jedi.parser import tokenize
+from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER,
+                               STRING)
+from jedi.parser.pgen2.pgen import generate_grammar
+from jedi.parser.pgen2.parse import PgenParser
+
+OPERATOR_KEYWORDS = 'and', 'for', 'if', 'else', 'in', 'is', 'lambda', 'not', 'or'
+# Not used yet. In the future I intend to add something like KeywordStatement
+STATEMENT_KEYWORDS = 'assert', 'del', 'global', 'nonlocal', 'raise', \
+    'return', 'yield', 'pass', 'continue', 'break'
+
+
+_loaded_grammars = {}
+
+
+class ParseError(Exception):
+    """
+    Signals you that the code you fed the Parser was not correct Python code.
+    """
+
+
+def load_grammar(version='3.4'):
+    # For now we only support two different Python syntax versions: The latest
+    # Python 3 and Python 2. This may change.
+    if version in ('3.2', '3.3'):
+        version = '3.4'
+    elif version == '2.6':
+        version = '2.7'
+
+    file = 'grammar' + version + '.txt'
+
+    global _loaded_grammars
+    path = os.path.join(os.path.dirname(__file__), file)
+    try:
+        return _loaded_grammars[path]
+    except KeyError:
+        return _loaded_grammars.setdefault(path, generate_grammar(path))
+
+
+class ParserSyntaxError(object):
+    def __init__(self, message, position):
+        self.message = message
+        self.position = position
+
+
+class Parser(object):
+    AST_MAPPING = {
+        'expr_stmt': pt.ExprStmt,
+        'classdef': pt.Class,
+        'funcdef': pt.Function,
+        'file_input': pt.Module,
+        'import_name': pt.ImportName,
+        'import_from': pt.ImportFrom,
+        'break_stmt': pt.KeywordStatement,
+        'continue_stmt': pt.KeywordStatement,
+        'return_stmt': pt.ReturnStmt,
+        'raise_stmt': pt.KeywordStatement,
+        'yield_expr': pt.YieldExpr,
+        'del_stmt': pt.KeywordStatement,
+        'pass_stmt': pt.KeywordStatement,
+        'global_stmt': pt.GlobalStmt,
+        'nonlocal_stmt': pt.KeywordStatement,
+        'print_stmt': pt.KeywordStatement,
+        'assert_stmt': pt.AssertStmt,
+        'if_stmt': pt.IfStmt,
+        'with_stmt': pt.WithStmt,
+        'for_stmt': pt.ForStmt,
+        'while_stmt': pt.WhileStmt,
+        'try_stmt': pt.TryStmt,
+        'comp_for': pt.CompFor,
+        'decorator': pt.Decorator,
+        'lambdef': pt.Lambda,
+        'old_lambdef': pt.Lambda,
+        'lambdef_nocond': pt.Lambda,
+    }
+
+    def __init__(self, grammar, source, start_symbol='file_input',
+                 tokenizer=None, start_parsing=True):
+        # Todo Remove start_parsing (with False)
+
+        self._used_names = {}
+        self._scope_names_stack = [{}]
+        self._last_failed_start_pos = (0, 0)
+        self._global_names = []
+
+        # For the fast parser.
+        self.position_modifier = pt.PositionModifier()
+
+        self._added_newline = False
+        # The Python grammar needs a newline at the end of each statement.
+        if not source.endswith('\n') and start_symbol == 'file_input':
+            source += '\n'
+            self._added_newline = True
+
+        self.source = source
+        self._start_symbol = start_symbol
+        self._grammar = grammar
+
+        self._parsed = None
+
+        if start_parsing:
+            if tokenizer is None:
+                tokenizer = tokenize.source_tokens(source, use_exact_op_types=True)
+            self.parse(tokenizer)
+
+    def parse(self, tokenizer):
+        if self._parsed is not None:
+            return self._parsed
+
+        start_number = self._grammar.symbol2number[self._start_symbol]
+        pgen_parser = PgenParser(
+            self._grammar, self.convert_node, self.convert_leaf,
+            self.error_recovery, start_number
+        )
+
+        try:
+            self._parsed = pgen_parser.parse(tokenizer)
+        finally:
+            self.stack = pgen_parser.stack
+
+        if self._start_symbol == 'file_input' != self._parsed.type:
+            # If there's only one statement, we get back a non-module. That's
+            # not what we want, we want a module, so we add it here:
+            self._parsed = self.convert_node(self._grammar,
+                                             self._grammar.symbol2number['file_input'],
+                                             [self._parsed])
+
+        if self._added_newline:
+            self.remove_last_newline()
+
+    def get_parsed_node(self):
+        # TODO rename to get_root_node
+        return self._parsed
+
+    def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix,
+                       add_token_callback):
+        raise ParseError
+
+    def convert_node(self, grammar, type, children):
+        """
+        Convert raw node information to a Node instance.
+
+        This is passed to the parser driver which calls it whenever a reduction of a
+        grammar rule produces a new complete node, so that the tree is build
+        strictly bottom-up.
+        """
+        symbol = grammar.number2symbol[type]
+        try:
+            new_node = Parser.AST_MAPPING[symbol](children)
+        except KeyError:
+            new_node = pt.Node(symbol, children)
+
+        # We need to check raw_node always, because the same node can be
+        # returned by convert multiple times.
+        if symbol == 'global_stmt':
+            self._global_names += new_node.get_global_names()
+        elif isinstance(new_node, pt.Lambda):
+            new_node.names_dict = self._scope_names_stack.pop()
+        elif isinstance(new_node, (pt.ClassOrFunc, pt.Module)) \
+                and symbol in ('funcdef', 'classdef', 'file_input'):
+            # scope_name_stack handling
+            scope_names = self._scope_names_stack.pop()
+            if isinstance(new_node, pt.ClassOrFunc):
+                n = new_node.name
+                scope_names[n.value].remove(n)
+                # Set the func name of the current node
+                arr = self._scope_names_stack[-1].setdefault(n.value, [])
+                arr.append(n)
+            new_node.names_dict = scope_names
+        elif isinstance(new_node, pt.CompFor):
+            # The name definitions of comprehenions shouldn't be part of the
+            # current scope. They are part of the comprehension scope.
+            for n in new_node.get_defined_names():
+                self._scope_names_stack[-1][n.value].remove(n)
+        return new_node
+
+    def convert_leaf(self, grammar, type, value, prefix, start_pos):
+        # print('leaf', repr(value), token.tok_name[type])
+        if type == tokenize.NAME:
+            if value in grammar.keywords:
+                if value in ('def', 'class', 'lambda'):
+                    self._scope_names_stack.append({})
+
+                return pt.Keyword(self.position_modifier, value, start_pos, prefix)
+            else:
+                name = pt.Name(self.position_modifier, value, start_pos, prefix)
+                # Keep a listing of all used names
+                arr = self._used_names.setdefault(name.value, [])
+                arr.append(name)
+                arr = self._scope_names_stack[-1].setdefault(name.value, [])
+                arr.append(name)
+                return name
+        elif type == STRING:
+            return pt.String(self.position_modifier, value, start_pos, prefix)
+        elif type == NUMBER:
+            return pt.Number(self.position_modifier, value, start_pos, prefix)
+        elif type == NEWLINE:
+            return pt.Newline(self.position_modifier, value, start_pos, prefix)
+        elif type == INDENT:
+            return pt.Indent(self.position_modifier, value, start_pos, prefix)
+        elif type == DEDENT:
+            return pt.Dedent(self.position_modifier, value, start_pos, prefix)
+        elif type == ENDMARKER:
+            return pt.EndMarker(self.position_modifier, value, start_pos, prefix)
+        else:
+            return pt.Operator(self.position_modifier, value, start_pos, prefix)
+
+    def remove_last_newline(self):
+        """
+        In all of this we need to work with _start_pos, because if we worked
+        with start_pos, we would need to check the position_modifier as well
+        (which is accounted for in the start_pos property).
+        """
+        endmarker = self._parsed.children[-1]
+        # The newline is either in the endmarker as a prefix or the previous
+        # leaf as a newline token.
+        prefix = endmarker.prefix
+        if prefix.endswith('\n'):
+            endmarker.prefix = prefix = prefix[:-1]
+            last_end = 0
+            if '\n' not in prefix:
+                # Basically if the last line doesn't end with a newline. we
+                # have to add the previous line's end_position.
+                try:
+                    last_end = endmarker.get_previous_leaf().end_pos[1]
+                except IndexError:
+                    pass
+            last_line = re.sub('.*\n', '', prefix)
+            endmarker._start_pos = endmarker._start_pos[0] - 1, last_end + len(last_line)
+        else:
+            try:
+                newline = endmarker.get_previous_leaf()
+            except IndexError:
+                return  # This means that the parser is empty.
+            while True:
+                if newline.value == '':
+                    # Must be a DEDENT, just continue.
+                    try:
+                        newline = newline.get_previous_leaf()
+                    except IndexError:
+                        # If there's a statement that fails to be parsed, there
+                        # will be no previous leaf. So just ignore it.
+                        break
+                elif newline.value != '\n':
+                    # TODO REMOVE, error recovery was simplified.
+                    # This may happen if error correction strikes and removes
+                    # a whole statement including '\n'.
+                    break
+                else:
+                    newline.value = ''
+                    if self._last_failed_start_pos > newline._start_pos:
+                        # It may be the case that there was a syntax error in a
+                        # function. In that case error correction removes the
+                        # right newline. So we use the previously assigned
+                        # _last_failed_start_pos variable to account for that.
+                        endmarker._start_pos = self._last_failed_start_pos
+                    else:
+                        endmarker._start_pos = newline._start_pos
+                    break
+
+
+class ParserWithRecovery(Parser):
+    """
+    This class is used to parse a Python file, it then divides them into a
+    class structure of different scopes.
+
+    :param grammar: The grammar object of pgen2. Loaded by load_grammar.
+    :param source: The codebase for the parser. Must be unicode.
+    :param module_path: The path of the module in the file system, may be None.
+    :type module_path: str
+    """
+    def __init__(self, grammar, source, module_path=None, tokenizer=None,
+                 start_parsing=True):
+        self.syntax_errors = []
+
+        self._omit_dedent_list = []
+        self._indent_counter = 0
+
+        # TODO do print absolute import detection here.
+        # try:
+        #     del python_grammar_no_print_statement.keywords["print"]
+        # except KeyError:
+        #     pass  # Doesn't exist in the Python 3 grammar.
+
+        # if self.options["print_function"]:
+        #     python_grammar = pygram.python_grammar_no_print_statement
+        # else:
+        super(ParserWithRecovery, self).__init__(
+            grammar, source,
+            tokenizer=tokenizer,
+            start_parsing=start_parsing
+        )
+        if start_parsing:
+            self.module = self._parsed
+            self.module.used_names = self._used_names
+            self.module.path = module_path
+            self.module.global_names = self._global_names
+
+    def parse(self, tokenizer):
+        return super(ParserWithRecovery, self).parse(self._tokenize(self._tokenize(tokenizer)))
+
+    def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix,
+                       add_token_callback):
+        """
+        This parser is written in a dynamic way, meaning that this parser
+        allows using different grammars (even non-Python). However, error
+        recovery is purely written for Python.
+        """
+        def current_suite(stack):
+            # For now just discard everything that is not a suite or
+            # file_input, if we detect an error.
+            for index, (dfa, state, (type_, nodes)) in reversed(list(enumerate(stack))):
+                # `suite` can sometimes be only simple_stmt, not stmt.
+                symbol = grammar.number2symbol[type_]
+                if symbol == 'file_input':
+                    break
+                elif symbol == 'suite' and len(nodes) > 1:
+                    # suites without an indent in them get discarded.
+                    break
+                elif symbol == 'simple_stmt' and len(nodes) > 1:
+                    # simple_stmt can just be turned into a Node, if there are
+                    # enough statements. Ignore the rest after that.
+                    break
+            return index, symbol, nodes
+
+        index, symbol, nodes = current_suite(stack)
+        if symbol == 'simple_stmt':
+            index -= 2
+            (_, _, (type_, suite_nodes)) = stack[index]
+            symbol = grammar.number2symbol[type_]
+            suite_nodes.append(pt.Node(symbol, list(nodes)))
+            # Remove
+            nodes[:] = []
+            nodes = suite_nodes
+            stack[index]
+
+        # print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index)
+        if self._stack_removal(grammar, stack, arcs, index + 1, value, start_pos):
+            add_token_callback(typ, value, start_pos, prefix)
+        else:
+            if typ == INDENT:
+                # For every deleted INDENT we have to delete a DEDENT as well.
+                # Otherwise the parser will get into trouble and DEDENT too early.
+                self._omit_dedent_list.append(self._indent_counter)
+            else:
+                error_leaf = pt.ErrorLeaf(self.position_modifier, typ, value, start_pos, prefix)
+                stack[-1][2][1].append(error_leaf)
+
+    def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos):
+        failed_stack = []
+        found = False
+        all_nodes = []
+        for dfa, state, (typ, nodes) in stack[start_index:]:
+            if nodes:
+                found = True
+            if found:
+                symbol = grammar.number2symbol[typ]
+                failed_stack.append((symbol, nodes))
+                all_nodes += nodes
+            if nodes and nodes[0] in ('def', 'class', 'lambda'):
+                self._scope_names_stack.pop()
+        if failed_stack:
+            stack[start_index - 1][2][1].append(pt.ErrorNode(all_nodes))
+
+        self._last_failed_start_pos = start_pos
+
+        stack[start_index:] = []
+        return failed_stack
+
+    def _tokenize(self, tokenizer):
+        for typ, value, start_pos, prefix in tokenizer:
+            # print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix))
+            if typ == DEDENT:
+                # We need to count indents, because if we just omit any DEDENT,
+                # we might omit them in the wrong place.
+                o = self._omit_dedent_list
+                if o and o[-1] == self._indent_counter:
+                    o.pop()
+                    continue
+
+                self._indent_counter -= 1
+            elif typ == INDENT:
+                self._indent_counter += 1
+
+            yield typ, value, start_pos, prefix
+
+    def __repr__(self):
+        return "<%s: %s>" % (type(self).__name__, self.module)

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/fast.py

@@ -0,0 +1,621 @@
+"""
+Basically a parser that is faster, because it tries to parse only parts and if
+anything changes, it only reparses the changed parts. But because it's not
+finished (and still not working as I want), I won't document it any further.
+"""
+import re
+from itertools import chain
+
+from jedi._compatibility import use_metaclass
+from jedi import settings
+from jedi.parser import ParserWithRecovery
+from jedi.parser import tree
+from jedi.parser.utils import underscore_memoization, parser_cache
+from jedi import debug
+from jedi.parser.tokenize import (source_tokens, NEWLINE,
+                                  ENDMARKER, INDENT, DEDENT)
+
+FLOWS = 'if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally', 'for'
+
+
+class FastModule(tree.Module):
+    type = 'file_input'
+
+    def __init__(self, module_path):
+        super(FastModule, self).__init__([])
+        self.modules = []
+        self.reset_caches()
+        self.names_dict = {}
+        self.path = module_path
+
+    def reset_caches(self):
+        self.modules = []
+        try:
+            del self._used_names  # Remove the used names cache.
+        except AttributeError:
+            pass  # It was never used.
+
+    @property
+    @underscore_memoization
+    def used_names(self):
+        return MergedNamesDict([m.used_names for m in self.modules])
+
+    @property
+    def global_names(self):
+        return [name for m in self.modules for name in m.global_names]
+
+    @property
+    def error_statements(self):
+        return [e for m in self.modules for e in m.error_statements]
+
+    def __repr__(self):
+        return "<fast.%s: %s@%s-%s>" % (type(self).__name__, self.name,
+                                        self.start_pos[0], self.end_pos[0])
+
+    # To avoid issues with with the `parser.ParserWithRecovery`, we need
+    # setters that do nothing, because if pickle comes along and sets those
+    # values.
+    @global_names.setter
+    def global_names(self, value):
+        pass
+
+    @error_statements.setter
+    def error_statements(self, value):
+        pass
+
+    @used_names.setter
+    def used_names(self, value):
+        pass
+
+
+class MergedNamesDict(object):
+    def __init__(self, dicts):
+        self.dicts = dicts
+
+    def __iter__(self):
+        return iter(set(key for dct in self.dicts for key in dct))
+
+    def __getitem__(self, value):
+        return list(chain.from_iterable(dct.get(value, []) for dct in self.dicts))
+
+    def items(self):
+        dct = {}
+        for d in self.dicts:
+            for key, values in d.items():
+                try:
+                    dct_values = dct[key]
+                    dct_values += values
+                except KeyError:
+                    dct[key] = list(values)
+        return dct.items()
+
+    def values(self):
+        lst = []
+        for dct in self.dicts:
+            lst += dct.values()
+        return lst
+
+
+class CachedFastParser(type):
+    """ This is a metaclass for caching `FastParser`. """
+    def __call__(self, grammar, source, module_path=None):
+        if not settings.fast_parser:
+            return ParserWithRecovery(grammar, source, module_path)
+
+        pi = parser_cache.get(module_path, None)
+        if pi is None or isinstance(pi.parser, ParserWithRecovery):
+            p = super(CachedFastParser, self).__call__(grammar, source, module_path)
+        else:
+            p = pi.parser  # pi is a `cache.ParserCacheItem`
+            p.update(source)
+        return p
+
+
+class ParserNode(object):
+    def __init__(self, fast_module, parser, source):
+        self._fast_module = fast_module
+        self.parent = None
+        self._node_children = []
+
+        self.source = source
+        self.hash = hash(source)
+        self.parser = parser
+        if source:
+            self._end_pos = parser.module.end_pos
+        else:
+            self._end_pos = 1, 0
+
+        try:
+            # With fast_parser we have either 1 subscope or only statements.
+            self._content_scope = parser.module.subscopes[0]
+            # A parsed node's content will be in the first indent, because
+            # everything that's parsed is within this subscope.
+            self._is_class_or_def = True
+        except IndexError:
+            self._content_scope = parser.module
+            self._is_class_or_def = False
+        else:
+            self._rewrite_last_newline()
+
+        # We need to be able to reset the original children of a parser.
+        self._old_children = list(self._content_scope.children)
+
+    def is_root_node(self):
+        return self.parent is None
+
+    def _rewrite_last_newline(self):
+        """
+        The ENDMARKER can contain a newline in the prefix. However this prefix
+        really belongs to the function - respectively to the next function or
+        parser node. If we don't rewrite that newline, we end up with a newline
+        in the wrong position, i.d. at the end of the file instead of in the
+        middle.
+        """
+        c = self._content_scope.children
+        if tree.is_node(c[-1], 'suite'):  # In a simple_stmt there's no DEDENT.
+            end_marker = self.parser.module.children[-1]
+            # Set the DEDENT prefix instead of the ENDMARKER.
+            c[-1].children[-1].prefix = end_marker.prefix
+            end_marker.prefix = ''
+
+    def __repr__(self):
+        module = self.parser.module
+        try:
+            return '<%s: %s-%s>' % (type(self).__name__, module.start_pos, module.end_pos)
+        except IndexError:
+            # There's no module yet.
+            return '<%s: empty>' % type(self).__name__
+
+    @property
+    def end_pos(self):
+        return self._end_pos[0] + self.parser.position_modifier.line, self._end_pos[1]
+
+    def reset_node(self):
+        """
+        Removes changes that were applied in this class.
+        """
+        self._node_children = []
+        scope = self._content_scope
+        scope.children = list(self._old_children)
+        try:
+            # This works if it's a MergedNamesDict.
+            # We are correcting it, because the MergedNamesDicts are artificial
+            # and can change after closing a node.
+            scope.names_dict = scope.names_dict.dicts[0]
+        except AttributeError:
+            pass
+
+    def close(self):
+        """
+        Closes the current parser node. This means that after this no further
+        nodes should be added anymore.
+        """
+        # We only need to replace the dict if multiple dictionaries are used:
+        if self._node_children:
+            dcts = [n.parser.module.names_dict for n in self._node_children]
+            # Need to insert the own node as well.
+            dcts.insert(0, self._content_scope.names_dict)
+            self._content_scope.names_dict = MergedNamesDict(dcts)
+            endmarker = self.parser.get_parsed_node().children[-1]
+            assert endmarker.type == 'endmarker'
+            last_parser = self._node_children[-1].parser
+            endmarker.start_pos = last_parser.get_parsed_node().end_pos
+
+    @property
+    def _indent(self):
+        if self.is_root_node():
+            return 0
+
+        return self.parser.module.children[0].start_pos[1]
+
+    def add_node(self, node, start_line, indent):
+        """
+        Adding a node means adding a node that was either just parsed or one
+        that can be reused.
+        """
+        # Content that is not a subscope can never be part of the current node,
+        # because it's basically a sister node, that sits next to it and not
+        # within it.
+        if (self._indent >= indent or not self._is_class_or_def) and \
+                not self.is_root_node():
+            self.close()
+            return self.parent.add_node(node, start_line, indent)
+
+        # Changing the line offsets is very important, because if they don't
+        # fit, all the start_pos values will be wrong.
+        m = node.parser.module
+        node.parser.position_modifier.line = start_line - 1
+        self._fast_module.modules.append(m)
+        node.parent = self
+
+        self._node_children.append(node)
+
+        # Insert parser objects into current structure. We only need to set the
+        # parents and children in a good way.
+        scope = self._content_scope
+        for child in m.children:
+            child.parent = scope
+            scope.children.append(child)
+
+        return node
+
+    def all_sub_nodes(self):
+        """
+        Returns all nodes including nested ones.
+        """
+        for n in self._node_children:
+            yield n
+            for y in n.all_sub_nodes():
+                yield y
+
+    @underscore_memoization  # Should only happen once!
+    def remove_last_newline(self):
+        self.parser.remove_last_newline()
+
+
+class FastParser(use_metaclass(CachedFastParser)):
+    _FLOWS_NEED_SPACE = 'if', 'elif', 'while', 'with', 'except', 'for'
+    _FLOWS_NEED_COLON = 'else', 'try', 'except', 'finally'
+    _keyword_re = re.compile('^[ \t]*(def |class |@|(?:%s)|(?:%s)\s*:)'
+                             % ('|'.join(_FLOWS_NEED_SPACE),
+                                '|'.join(_FLOWS_NEED_COLON)))
+
+    def __init__(self, grammar, source, module_path=None):
+        # set values like `tree.Module`.
+        self._grammar = grammar
+        self.module_path = module_path
+        self._reset_caches()
+        self.update(source)
+
+    def _reset_caches(self):
+        self.module = FastModule(self.module_path)
+        self.root_node = self.current_node = ParserNode(self.module, self, '')
+
+    def get_parsed_node(self):
+        return self.module
+
+    def update(self, source):
+        # Variables for testing purposes: It is important that the number of
+        # parsers used can be minimized. With these variables we can test
+        # against that.
+        self.number_parsers_used = 0
+        self.number_of_splits = 0
+        self.number_of_misses = 0
+        self.module.reset_caches()
+        self.source = source
+        try:
+            self._parse(source)
+        except:
+            # FastParser is cached, be careful with exceptions.
+            self._reset_caches()
+            raise
+
+    def _split_parts(self, source):
+        """
+        Split the source code into different parts. This makes it possible to
+        parse each part seperately and therefore cache parts of the file and
+        not everything.
+        """
+        def gen_part():
+            text = ''.join(current_lines)
+            del current_lines[:]
+            self.number_of_splits += 1
+            return text
+
+        def just_newlines(current_lines):
+            for line in current_lines:
+                line = line.lstrip('\t \n\r')
+                if line and line[0] != '#':
+                    return False
+            return True
+
+        # Split only new lines. Distinction between \r\n is the tokenizer's
+        # job.
+        # It seems like there's no problem with form feed characters here,
+        # because we're not counting lines.
+        self._lines = source.splitlines(True)
+        current_lines = []
+        is_decorator = False
+        # Use -1, because that indent is always smaller than any other.
+        indent_list = [-1, 0]
+        new_indent = False
+        parentheses_level = 0
+        flow_indent = None
+        previous_line = None
+        # All things within flows are simply being ignored.
+        for i, l in enumerate(self._lines):
+            # Handle backslash newline escaping.
+            if l.endswith('\\\n') or l.endswith('\\\r\n'):
+                if previous_line is not None:
+                    previous_line += l
+                else:
+                    previous_line = l
+                continue
+            if previous_line is not None:
+                l = previous_line + l
+                previous_line = None
+
+            # check for dedents
+            s = l.lstrip('\t \n\r')
+            indent = len(l) - len(s)
+            if not s or s[0] == '#':
+                current_lines.append(l)  # Just ignore comments and blank lines
+                continue
+
+            if new_indent and not parentheses_level:
+                if indent > indent_list[-2]:
+                    # Set the actual indent, not just the random old indent + 1.
+                    indent_list[-1] = indent
+                new_indent = False
+
+            while indent < indent_list[-1]:  # -> dedent
+                indent_list.pop()
+                # This automatically resets the flow_indent if there was a
+                # dedent or a flow just on one line (with one simple_stmt).
+                new_indent = False
+                if flow_indent is None and current_lines and not parentheses_level:
+                    yield gen_part()
+                flow_indent = None
+
+            # Check lines for functions/classes and split the code there.
+            if flow_indent is None:
+                m = self._keyword_re.match(l)
+                if m:
+                    # Strip whitespace and colon from flows as a check.
+                    if m.group(1).strip(' \t\r\n:') in FLOWS:
+                        if not parentheses_level:
+                            flow_indent = indent
+                    else:
+                        if not is_decorator and not just_newlines(current_lines):
+                            yield gen_part()
+                    is_decorator = '@' == m.group(1)
+                    if not is_decorator:
+                        parentheses_level = 0
+                        # The new indent needs to be higher
+                        indent_list.append(indent + 1)
+                        new_indent = True
+                elif is_decorator:
+                    is_decorator = False
+
+            parentheses_level = \
+                max(0, (l.count('(') + l.count('[') + l.count('{') -
+                        l.count(')') - l.count(']') - l.count('}')))
+
+            current_lines.append(l)
+
+        if previous_line is not None:
+            current_lines.append(previous_line)
+        if current_lines:
+            yield gen_part()
+
+    def _parse(self, source):
+        """ :type source: str """
+        added_newline = False
+        if not source or source[-1] != '\n':
+            # To be compatible with Pythons grammar, we need a newline at the
+            # end. The parser would handle it, but since the fast parser abuses
+            # the normal parser in various ways, we need to care for this
+            # ourselves.
+            source += '\n'
+            added_newline = True
+
+        next_code_part_end_line = code_part_end_line = 1
+        start = 0
+        nodes = list(self.root_node.all_sub_nodes())
+        # Now we can reset the node, because we have all the old nodes.
+        self.root_node.reset_node()
+        self.current_node = self.root_node
+        last_end_line = 1
+
+        for code_part in self._split_parts(source):
+            next_code_part_end_line += code_part.count('\n')
+            # If the last code part parsed isn't equal to the current end_pos,
+            # we know that the parser went further (`def` start in a
+            # docstring). So just parse the next part.
+            if code_part_end_line == last_end_line:
+                self._parse_part(code_part, source[start:], code_part_end_line, nodes)
+            else:
+                self.number_of_misses += 1
+                # Means that some lines where not fully parsed. Parse it now.
+                # This is a very rare case. Should only happens with very
+                # strange code bits.
+                while last_end_line < next_code_part_end_line:
+                    code_part_end_line = last_end_line
+                    # We could calculate the src in a more complicated way to
+                    # make caching here possible as well. However, this is
+                    # complicated and error-prone. Since this is not very often
+                    # called - just ignore it.
+                    src = ''.join(self._lines[code_part_end_line - 1:])
+                    self._parse_part(code_part, src, code_part_end_line, nodes)
+                    last_end_line = self.current_node.end_pos[0]
+                debug.dbg("While parsing %s, starting with line %s wasn't included in split.",
+                          self.module_path, code_part_end_line)
+                #assert code_part_end_line > last_end_line
+                # This means that the parser parsed faster than the last given
+                # `code_part`.
+                debug.dbg('While parsing %s, line %s slowed down the fast parser.',
+                          self.module_path, code_part_end_line)
+
+            code_part_end_line = next_code_part_end_line
+            start += len(code_part)
+
+            last_end_line = self.current_node.end_pos[0]
+
+        if added_newline:
+            self.current_node.remove_last_newline()
+
+        # Now that the for loop is finished, we still want to close all nodes.
+        node = self.current_node
+        while node is not None:
+            node.close()
+            node = node.parent
+
+        debug.dbg('Parsed %s, with %s parsers in %s splits.'
+                  % (self.module_path, self.number_parsers_used,
+                     self.number_of_splits))
+
+    def _parse_part(self, source, parser_code, code_part_end_line, nodes):
+        """
+        Side effect: Alters the list of nodes.
+        """
+        h = hash(source)
+        for index, node in enumerate(nodes):
+            if node.hash == h and node.source == source:
+                node.reset_node()
+                nodes.remove(node)
+                parser_code = source
+                break
+        else:
+            tokenizer = FastTokenizer(parser_code)
+            self.number_parsers_used += 1
+            p = ParserWithRecovery(self._grammar, parser_code, self.module_path, tokenizer=tokenizer)
+
+            end = code_part_end_line - 1 + p.module.end_pos[0]
+            used_lines = self._lines[code_part_end_line - 1:end - 1]
+            code_part_actually_used = ''.join(used_lines)
+
+            node = ParserNode(self.module, p, code_part_actually_used)
+
+        indent = len(parser_code) - len(parser_code.lstrip('\t '))
+
+        self.current_node.add_node(node, code_part_end_line, indent)
+        self.current_node = node
+
+
+class FastTokenizer(object):
+    """
+    Breaks when certain conditions are met, i.e. a new function or class opens.
+    """
+    def __init__(self, source):
+        self.source = source
+        self._gen = source_tokens(source, use_exact_op_types=True)
+        self._closed = False
+
+        # fast parser options
+        self.current = self.previous = NEWLINE, '', (0, 0)
+        self._in_flow = False
+        self._is_decorator = False
+        self._first_stmt = True
+        self._parentheses_level = 0
+        self._indent_counter = 0
+        self._flow_indent_counter = 0
+        self._returned_endmarker = False
+        self._expect_indent = False
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        """ Python 2 Compatibility """
+        return self.__next__()
+
+    def __next__(self):
+        if self._closed:
+            return self._finish_dedents()
+
+        typ, value, start_pos, prefix = current = next(self._gen)
+        if typ == ENDMARKER:
+            self._closed = True
+            self._returned_endmarker = True
+            return current
+
+        self.previous = self.current
+        self.current = current
+
+        if typ == INDENT:
+            self._indent_counter += 1
+            if not self._expect_indent and not self._first_stmt and not self._in_flow:
+                # This does not mean that there is an actual flow, it means
+                # that the INDENT is syntactically wrong.
+                self._flow_indent_counter = self._indent_counter - 1
+                self._in_flow = True
+            self._expect_indent = False
+        elif typ == DEDENT:
+            self._indent_counter -= 1
+            if self._in_flow:
+                if self._indent_counter == self._flow_indent_counter:
+                    self._in_flow = False
+            else:
+                self._closed = True
+            return current
+
+        previous_type = self.previous[0]
+        if value in ('def', 'class') and self._parentheses_level:
+            # Account for the fact that an open parentheses before a function
+            # will reset the parentheses counter, but new lines before will
+            # still be ignored. So check the prefix.
+
+            # TODO what about flow parentheses counter resets in the tokenizer?
+            self._parentheses_level = 0
+            # We need to simulate a newline before the indent, because the
+            # open parentheses ignored them.
+            if re.search('\n\s*', prefix):
+                previous_type = NEWLINE
+
+        # Parentheses ignore the indentation rules. The other three stand for
+        # new lines.
+        if previous_type in (NEWLINE, INDENT, DEDENT) \
+                and not self._parentheses_level and typ not in (INDENT, DEDENT):
+            if not self._in_flow:
+                if value in FLOWS:
+                    self._flow_indent_counter = self._indent_counter
+                    self._first_stmt = False
+                elif value in ('def', 'class', '@'):
+                    # The values here are exactly the same check as in
+                    # _split_parts, but this time with tokenize and therefore
+                    # precise.
+                    if not self._first_stmt and not self._is_decorator:
+                        return self._close()
+
+                    self._is_decorator = '@' == value
+                    if not self._is_decorator:
+                        self._first_stmt = False
+                        self._expect_indent = True
+                elif self._expect_indent:
+                    return self._close()
+                else:
+                    self._first_stmt = False
+
+        if value in '([{' and value:
+            self._parentheses_level += 1
+        elif value in ')]}' and value:
+            # Ignore closing parentheses, because they are all
+            # irrelevant for the indentation.
+            self._parentheses_level = max(self._parentheses_level - 1, 0)
+        return current
+
+    def _close(self):
+        if self._first_stmt:
+            # Continue like nothing has happened, because we want to enter
+            # the first class/function.
+            if self.current[1] != '@':
+                self._first_stmt = False
+            return self.current
+        else:
+            self._closed = True
+            return self._finish_dedents()
+
+    def _finish_dedents(self):
+        if self._indent_counter:
+            self._indent_counter -= 1
+            return DEDENT, '', self.current[2], ''
+        elif not self._returned_endmarker:
+            self._returned_endmarker = True
+            return ENDMARKER, '', self.current[2], self._get_prefix()
+        else:
+            raise StopIteration
+
+    def _get_prefix(self):
+        """
+        We're using the current prefix for the endmarker to not loose any
+        information. However we care about "lost" lines. The prefix of the
+        current line (indent) will always be included in the current line.
+        """
+        cur = self.current
+        while cur[0] == DEDENT:
+            cur = next(self._gen)
+        prefix = cur[3]
+
+        # \Z for the end of the string. $ is bugged, because it has the
+        # same behavior with or without re.MULTILINE.
+        return re.sub(r'[^\n]+\Z', '', prefix)

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/grammar2.7.txt

@@ -0,0 +1,152 @@
+# Grammar for 2to3. This grammar supports Python 2.x and 3.x.
+
+# Note:  Changing the grammar specified in this file will most likely
+#        require corresponding changes in the parser module
+#        (../Modules/parsermodule.c).  If you can't make the changes to
+#        that module yourself, please co-ordinate the required changes
+#        with someone who can; ask around on python-dev for help.  Fred
+#        Drake <fdrake@acm.org> will probably be listening there.
+
+# NOTE WELL: You should also follow all the steps listed in PEP 306,
+# "How to Change Python's Grammar"
+
+
+# Start symbols for the grammar:
+#	    file_input is a module or sequence of commands read from an input file;
+#	    single_input is a single interactive statement;
+#	    eval_input is the input for the eval() and input() functions.
+# NB: compound_stmt in single_input is followed by extra NEWLINE!
+file_input: (NEWLINE | stmt)* ENDMARKER
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+eval_input: testlist NEWLINE* ENDMARKER
+
+decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+decorators: decorator+
+decorated: decorators (classdef | funcdef)
+funcdef: 'def' NAME parameters ['->' test] ':' suite
+parameters: '(' [typedargslist] ')'
+typedargslist: ((tfpdef ['=' test] ',')*
+                ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname)
+                | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+tname: NAME [':' test]
+tfpdef: tname | '(' tfplist ')'
+tfplist: tfpdef (',' tfpdef)* [',']
+varargslist: ((vfpdef ['=' test] ',')*
+              ('*' [vname] (',' vname ['=' test])*  [',' '**' vname] | '**' vname)
+              | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+vname: NAME
+vfpdef: vname | '(' vfplist ')'
+vfplist: vfpdef (',' vfpdef)* [',']
+
+stmt: simple_stmt | compound_stmt
+simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+small_stmt: (expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
+             import_stmt | global_stmt | exec_stmt | assert_stmt)
+expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
+augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+            '<<=' | '>>=' | '**=' | '//=')
+# For normal assignments, additional restrictions enforced by the interpreter
+print_stmt: 'print' ( [ test (',' test)* [','] ] |
+                      '>>' test [ (',' test)+ [','] ] )
+del_stmt: 'del' exprlist
+pass_stmt: 'pass'
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
+break_stmt: 'break'
+continue_stmt: 'continue'
+return_stmt: 'return' [testlist]
+yield_stmt: yield_expr
+raise_stmt: 'raise' [test [',' test [',' test]]]
+import_stmt: import_name | import_from
+import_name: 'import' dotted_as_names
+# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+              'import' ('*' | '(' import_as_names ')' | import_as_names))
+import_as_name: NAME ['as' NAME]
+dotted_as_name: dotted_name ['as' NAME]
+import_as_names: import_as_name (',' import_as_name)* [',']
+dotted_as_names: dotted_as_name (',' dotted_as_name)*
+dotted_name: NAME ('.' NAME)*
+global_stmt: 'global' NAME (',' NAME)*
+exec_stmt: 'exec' expr ['in' test [',' test]]
+assert_stmt: 'assert' test [',' test]
+
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' test ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+try_stmt: ('try' ':' suite
+           ((except_clause ':' suite)+
+	        ['else' ':' suite]
+	        ['finally' ':' suite] |
+	       'finally' ':' suite))
+with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_item: test ['as' expr]
+with_var: 'as' expr
+# NB compile.c makes sure that the default except clause is last
+except_clause: 'except' [test [(',' | 'as') test]]
+# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows
+# classes and functions to be empty, which is beneficial for autocompletion.
+suite: simple_stmt | NEWLINE INDENT stmt* DEDENT
+
+# Backward compatibility cruft to support:
+# [ x for x in lambda: True, lambda: False if x() ]
+# even while also allowing:
+# lambda x: 5 if x else 2
+# (But not a mix of the two)
+testlist_safe: old_test [(',' old_test)+ [',']]
+old_test: or_test | old_lambdef
+old_lambdef: 'lambda' [varargslist] ':' old_test
+
+test: or_test ['if' or_test 'else' test] | lambdef
+or_test: and_test ('or' and_test)*
+and_test: not_test ('and' not_test)*
+not_test: 'not' not_test | comparison
+comparison: expr (comp_op expr)*
+comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+star_expr: '*' expr
+expr: xor_expr ('|' xor_expr)*
+xor_expr: and_expr ('^' and_expr)*
+and_expr: shift_expr ('&' shift_expr)*
+shift_expr: arith_expr (('<<'|'>>') arith_expr)*
+arith_expr: term (('+'|'-') term)*
+term: factor (('*'|'/'|'%'|'//') factor)*
+factor: ('+'|'-'|'~') factor | power
+power: atom trailer* ['**' factor]
+atom: ('(' [yield_expr|testlist_comp] ')' |
+       '[' [testlist_comp] ']' |
+       '{' [dictorsetmaker] '}' |
+       '`' testlist1 '`' |
+       NAME | NUMBER | STRING+ | '.' '.' '.')
+# Modification by David Halter, remove `testlist_gexp` and `listmaker`
+testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+lambdef: 'lambda' [varargslist] ':' test
+trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+subscriptlist: subscript (',' subscript)* [',']
+subscript: test | [test] ':' [test] [sliceop]
+sliceop: ':' [test]
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
+testlist: test (',' test)* [',']
+# Modification by David Halter, dictsetmaker -> dictorsetmaker (so that it's
+# the same as in the 3.4 grammar).
+dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
+                (test (comp_for | (',' test)* [','])) )
+
+classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
+
+arglist: (argument ',')* (argument [',']
+                         |'*' test (',' argument)* [',' '**' test] 
+                         |'**' test)
+argument: test [comp_for] | test '=' test  # Really [keyword '='] test
+
+comp_iter: comp_for | comp_if
+comp_for: 'for' exprlist 'in' testlist_safe [comp_iter]
+comp_if: 'if' old_test [comp_iter]
+
+testlist1: test (',' test)*
+
+# not used in grammar, but may appear in "node" passed from Parser to Compiler
+encoding_decl: NAME
+
+yield_expr: 'yield' [testlist]

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/grammar3.4.txt

@@ -0,0 +1,135 @@
+# Grammar for Python
+
+# Note:  Changing the grammar specified in this file will most likely
+#        require corresponding changes in the parser module
+#        (../Modules/parsermodule.c).  If you can't make the changes to
+#        that module yourself, please co-ordinate the required changes
+#        with someone who can; ask around on python-dev for help.  Fred
+#        Drake <fdrake@acm.org> will probably be listening there.
+
+# NOTE WELL: You should also follow all the steps listed in PEP 306,
+# "How to Change Python's Grammar"
+
+# Start symbols for the grammar:
+#       single_input is a single interactive statement;
+#       file_input is a module or sequence of commands read from an input file;
+#       eval_input is the input for the eval() functions.
+# NB: compound_stmt in single_input is followed by extra NEWLINE!
+file_input: (NEWLINE | stmt)* ENDMARKER
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+eval_input: testlist NEWLINE* ENDMARKER
+
+decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+decorators: decorator+
+decorated: decorators (classdef | funcdef)
+funcdef: 'def' NAME parameters ['->' test] ':' suite
+parameters: '(' [typedargslist] ')'
+typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
+       ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
+     |  '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+tfpdef: NAME [':' test]
+varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
+       ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]
+     |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
+vfpdef: NAME
+
+stmt: simple_stmt | compound_stmt
+simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
+expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
+augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+            '<<=' | '>>=' | '**=' | '//=')
+# For normal assignments, additional restrictions enforced by the interpreter
+del_stmt: 'del' exprlist
+pass_stmt: 'pass'
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
+break_stmt: 'break'
+continue_stmt: 'continue'
+return_stmt: 'return' [testlist]
+yield_stmt: yield_expr
+raise_stmt: 'raise' [test ['from' test]]
+import_stmt: import_name | import_from
+import_name: 'import' dotted_as_names
+# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+              'import' ('*' | '(' import_as_names ')' | import_as_names))
+import_as_name: NAME ['as' NAME]
+dotted_as_name: dotted_name ['as' NAME]
+import_as_names: import_as_name (',' import_as_name)* [',']
+dotted_as_names: dotted_as_name (',' dotted_as_name)*
+dotted_name: NAME ('.' NAME)*
+global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
+assert_stmt: 'assert' test [',' test]
+
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' test ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+try_stmt: ('try' ':' suite
+           ((except_clause ':' suite)+
+            ['else' ':' suite]
+            ['finally' ':' suite] |
+           'finally' ':' suite))
+with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_item: test ['as' expr]
+# NB compile.c makes sure that the default except clause is last
+except_clause: 'except' [test ['as' NAME]]
+# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows
+# classes and functions to be empty, which is beneficial for autocompletion.
+suite: simple_stmt | NEWLINE INDENT stmt* DEDENT
+
+test: or_test ['if' or_test 'else' test] | lambdef
+test_nocond: or_test | lambdef_nocond
+lambdef: 'lambda' [varargslist] ':' test
+lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
+or_test: and_test ('or' and_test)*
+and_test: not_test ('and' not_test)*
+not_test: 'not' not_test | comparison
+comparison: expr (comp_op expr)*
+# <> isn't actually a valid comparison operator in Python. It's here for the
+# sake of a __future__ import described in PEP 401
+comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+star_expr: '*' expr
+expr: xor_expr ('|' xor_expr)*
+xor_expr: and_expr ('^' and_expr)*
+and_expr: shift_expr ('&' shift_expr)*
+shift_expr: arith_expr (('<<'|'>>') arith_expr)*
+arith_expr: term (('+'|'-') term)*
+term: factor (('*'|'/'|'%'|'//') factor)*
+factor: ('+'|'-'|'~') factor | power
+power: atom trailer* ['**' factor]
+atom: ('(' [yield_expr|testlist_comp] ')' |
+       '[' [testlist_comp] ']' |
+       '{' [dictorsetmaker] '}' |
+       NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+subscriptlist: subscript (',' subscript)* [',']
+subscript: test | [test] ':' [test] [sliceop]
+sliceop: ':' [test]
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
+testlist: test (',' test)* [',']
+dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
+                  (test (comp_for | (',' test)* [','])) )
+
+classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
+
+arglist: (argument ',')* (argument [',']
+                         |'*' test (',' argument)* [',' '**' test] 
+                         |'**' test)
+# The reason that keywords are test nodes instead of NAME is that using NAME
+# results in an ambiguity. ast.c makes sure it's a NAME.
+argument: test [comp_for] | test '=' test  # Really [keyword '='] test
+comp_iter: comp_for | comp_if
+comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_if: 'if' test_nocond [comp_iter]
+
+# not used in grammar, but may appear in "node" passed from Parser to Compiler
+encoding_decl: NAME
+
+yield_expr: 'yield' [yield_arg]
+yield_arg: 'from' test | testlist

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/grammar3.5.txt

@@ -0,0 +1,154 @@
+# Grammar for Python
+
+# Note:  Changing the grammar specified in this file will most likely
+#        require corresponding changes in the parser module
+#        (../Modules/parsermodule.c).  If you can't make the changes to
+#        that module yourself, please co-ordinate the required changes
+#        with someone who can; ask around on python-dev for help.  Fred
+#        Drake <fdrake@acm.org> will probably be listening there.
+
+# NOTE WELL: You should also follow all the steps listed at
+# https://docs.python.org/devguide/grammar.html
+
+# Start symbols for the grammar:
+#       single_input is a single interactive statement;
+#       file_input is a module or sequence of commands read from an input file;
+#       eval_input is the input for the eval() functions.
+# NB: compound_stmt in single_input is followed by extra NEWLINE!
+file_input: (NEWLINE | stmt)* ENDMARKER
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+eval_input: testlist NEWLINE* ENDMARKER
+
+decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+decorators: decorator+
+decorated: decorators (classdef | funcdef | async_funcdef)
+
+# NOTE: Reinoud Elhorst, using ASYNC/AWAIT keywords instead of tokens
+# skipping python3.5 compatibility, in favour of 3.7 solution
+async_funcdef: 'async' funcdef
+funcdef: 'def' NAME parameters ['->' test] ':' suite
+
+parameters: '(' [typedargslist] ')'
+typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
+       ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
+     |  '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+tfpdef: NAME [':' test]
+varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
+       ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]
+     |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
+vfpdef: NAME
+
+stmt: simple_stmt | compound_stmt
+simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
+expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
+augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+            '<<=' | '>>=' | '**=' | '//=')
+# For normal assignments, additional restrictions enforced by the interpreter
+del_stmt: 'del' exprlist
+pass_stmt: 'pass'
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
+break_stmt: 'break'
+continue_stmt: 'continue'
+return_stmt: 'return' [testlist]
+yield_stmt: yield_expr
+raise_stmt: 'raise' [test ['from' test]]
+import_stmt: import_name | import_from
+import_name: 'import' dotted_as_names
+# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+              'import' ('*' | '(' import_as_names ')' | import_as_names))
+import_as_name: NAME ['as' NAME]
+dotted_as_name: dotted_name ['as' NAME]
+import_as_names: import_as_name (',' import_as_name)* [',']
+dotted_as_names: dotted_as_name (',' dotted_as_name)*
+dotted_name: NAME ('.' NAME)*
+global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
+assert_stmt: 'assert' test [',' test]
+
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
+if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' test ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+try_stmt: ('try' ':' suite
+           ((except_clause ':' suite)+
+            ['else' ':' suite]
+            ['finally' ':' suite] |
+           'finally' ':' suite))
+with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_item: test ['as' expr]
+# NB compile.c makes sure that the default except clause is last
+except_clause: 'except' [test ['as' NAME]]
+# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows
+# classes and functions to be empty, which is beneficial for autocompletion.
+suite: simple_stmt | NEWLINE INDENT stmt* DEDENT
+
+test: or_test ['if' or_test 'else' test] | lambdef
+test_nocond: or_test | lambdef_nocond
+lambdef: 'lambda' [varargslist] ':' test
+lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
+or_test: and_test ('or' and_test)*
+and_test: not_test ('and' not_test)*
+not_test: 'not' not_test | comparison
+comparison: expr (comp_op expr)*
+# <> isn't actually a valid comparison operator in Python. It's here for the
+# sake of a __future__ import described in PEP 401 (which really works :-)
+comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+star_expr: '*' expr
+expr: xor_expr ('|' xor_expr)*
+xor_expr: and_expr ('^' and_expr)*
+and_expr: shift_expr ('&' shift_expr)*
+shift_expr: arith_expr (('<<'|'>>') arith_expr)*
+arith_expr: term (('+'|'-') term)*
+term: factor (('*'|'@'|'/'|'%'|'//') factor)*
+factor: ('+'|'-'|'~') factor | power
+power: atom_expr ['**' factor]
+atom_expr: ['await'] atom trailer*
+atom: ('(' [yield_expr|testlist_comp] ')' |
+       '[' [testlist_comp] ']' |
+       '{' [dictorsetmaker] '}' |
+       NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+subscriptlist: subscript (',' subscript)* [',']
+subscript: test | [test] ':' [test] [sliceop]
+sliceop: ':' [test]
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
+testlist: test (',' test)* [',']
+dictorsetmaker: ( ((test ':' test | '**' expr)
+                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                  ((test | star_expr)
+                   (comp_for | (',' (test | star_expr))* [','])) )
+
+classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
+
+arglist: argument (',' argument)*  [',']
+
+# The reason that keywords are test nodes instead of NAME is that using NAME
+# results in an ambiguity. ast.c makes sure it's a NAME.
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguements are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+            test '=' test |
+            '**' test |
+            '*' test )
+
+comp_iter: comp_for | comp_if
+comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_if: 'if' test_nocond [comp_iter]
+
+# not used in grammar, but may appear in "node" passed from Parser to Compiler
+encoding_decl: NAME
+
+yield_expr: 'yield' [yield_arg]
+yield_arg: 'from' test | testlist

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/pgen2/__init__.py

@@ -0,0 +1,8 @@
+# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+# Modifications:
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+# Copyright 2014 David Halter. Integration into Jedi.
+# Modifications are dual-licensed: MIT and PSF.

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/pgen2/grammar.py

@@ -0,0 +1,125 @@
+# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+# Modifications:
+# Copyright 2014 David Halter. Integration into Jedi.
+# Modifications are dual-licensed: MIT and PSF.
+
+"""This module defines the data structures used to represent a grammar.
+
+These are a bit arcane because they are derived from the data
+structures used by Python's 'pgen' parser generator.
+
+There's also a table here mapping operators to their names in the
+token module; the Python tokenize module reports all operators as the
+fallback token code OP, but the parser needs the actual token code.
+
+"""
+
+# Python imports
+import pickle
+
+
+class Grammar(object):
+    """Pgen parsing tables conversion class.
+
+    Once initialized, this class supplies the grammar tables for the
+    parsing engine implemented by parse.py.  The parsing engine
+    accesses the instance variables directly.  The class here does not
+    provide initialization of the tables; several subclasses exist to
+    do this (see the conv and pgen modules).
+
+    The load() method reads the tables from a pickle file, which is
+    much faster than the other ways offered by subclasses.  The pickle
+    file is written by calling dump() (after loading the grammar
+    tables using a subclass).  The report() method prints a readable
+    representation of the tables to stdout, for debugging.
+
+    The instance variables are as follows:
+
+    symbol2number -- a dict mapping symbol names to numbers.  Symbol
+                     numbers are always 256 or higher, to distinguish
+                     them from token numbers, which are between 0 and
+                     255 (inclusive).
+
+    number2symbol -- a dict mapping numbers to symbol names;
+                     these two are each other's inverse.
+
+    states        -- a list of DFAs, where each DFA is a list of
+                     states, each state is a list of arcs, and each
+                     arc is a (i, j) pair where i is a label and j is
+                     a state number.  The DFA number is the index into
+                     this list.  (This name is slightly confusing.)
+                     Final states are represented by a special arc of
+                     the form (0, j) where j is its own state number.
+
+    dfas          -- a dict mapping symbol numbers to (DFA, first)
+                     pairs, where DFA is an item from the states list
+                     above, and first is a set of tokens that can
+                     begin this grammar rule (represented by a dict
+                     whose values are always 1).
+
+    labels        -- a list of (x, y) pairs where x is either a token
+                     number or a symbol number, and y is either None
+                     or a string; the strings are keywords.  The label
+                     number is the index in this list; label numbers
+                     are used to mark state transitions (arcs) in the
+                     DFAs.
+
+    start         -- the number of the grammar's start symbol.
+
+    keywords      -- a dict mapping keyword strings to arc labels.
+
+    tokens        -- a dict mapping token numbers to arc labels.
+
+    """
+
+    def __init__(self):
+        self.symbol2number = {}
+        self.number2symbol = {}
+        self.states = []
+        self.dfas = {}
+        self.labels = [(0, "EMPTY")]
+        self.keywords = {}
+        self.tokens = {}
+        self.symbol2label = {}
+        self.start = 256
+
+    def dump(self, filename):
+        """Dump the grammar tables to a pickle file."""
+        with open(filename, "wb") as f:
+            pickle.dump(self.__dict__, f, 2)
+
+    def load(self, filename):
+        """Load the grammar tables from a pickle file."""
+        with open(filename, "rb") as f:
+            d = pickle.load(f)
+        self.__dict__.update(d)
+
+    def copy(self):
+        """
+        Copy the grammar.
+        """
+        new = self.__class__()
+        for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
+                          "tokens", "symbol2label"):
+            setattr(new, dict_attr, getattr(self, dict_attr).copy())
+        new.labels = self.labels[:]
+        new.states = self.states[:]
+        new.start = self.start
+        return new
+
+    def report(self):
+        """Dump the grammar tables to standard output, for debugging."""
+        from pprint import pprint
+        print("s2n")
+        pprint(self.symbol2number)
+        print("n2s")
+        pprint(self.number2symbol)
+        print("states")
+        pprint(self.states)
+        print("dfas")
+        pprint(self.dfas)
+        print("labels")
+        pprint(self.labels)
+        print("start", self.start)

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/pgen2/parse.py

@@ -0,0 +1,213 @@
+# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+# Modifications:
+# Copyright 2014 David Halter. Integration into Jedi.
+# Modifications are dual-licensed: MIT and PSF.
+
+"""
+Parser engine for the grammar tables generated by pgen.
+
+The grammar table must be loaded first.
+
+See Parser/parser.c in the Python distribution for additional info on
+how this parsing engine works.
+"""
+
+# Local imports
+from jedi.parser import tokenize
+
+
+class InternalParseError(Exception):
+    """
+    Exception to signal the parser is stuck and error recovery didn't help.
+    Basically this shouldn't happen. It's a sign that something is really
+    wrong.
+    """
+
+    def __init__(self, msg, type, value, start_pos):
+        Exception.__init__(self, "%s: type=%r, value=%r, start_pos=%r" %
+                           (msg, tokenize.tok_name[type], value, start_pos))
+        self.msg = msg
+        self.type = type
+        self.value = value
+        self.start_pos = start_pos
+
+
+def token_to_ilabel(grammar, type_, value):
+    # Map from token to label
+    if type_ == tokenize.NAME:
+        # Check for reserved words (keywords)
+        try:
+            return grammar.keywords[value]
+        except KeyError:
+            pass
+
+    try:
+        return grammar.tokens[type_]
+    except KeyError:
+        return None
+
+
+class PgenParser(object):
+    """Parser engine.
+
+    The proper usage sequence is:
+
+    p = Parser(grammar, [converter])  # create instance
+    p.setup([start])                  # prepare for parsing
+    <for each input token>:
+        if p.addtoken(...):           # parse a token
+            break
+    root = p.rootnode                 # root of abstract syntax tree
+
+    A Parser instance may be reused by calling setup() repeatedly.
+
+    A Parser instance contains state pertaining to the current token
+    sequence, and should not be used concurrently by different threads
+    to parse separate token sequences.
+
+    See driver.py for how to get input tokens by tokenizing a file or
+    string.
+
+    Parsing is complete when addtoken() returns True; the root of the
+    abstract syntax tree can then be retrieved from the rootnode
+    instance variable.  When a syntax error occurs, error_recovery()
+    is called. There is no error recovery; the parser cannot be used
+    after a syntax error was reported (but it can be reinitialized by
+    calling setup()).
+
+    """
+
+    def __init__(self, grammar, convert_node, convert_leaf, error_recovery, start):
+        """Constructor.
+
+        The grammar argument is a grammar.Grammar instance; see the
+        grammar module for more information.
+
+        The parser is not ready yet for parsing; you must call the
+        setup() method to get it started.
+
+        The optional convert argument is a function mapping concrete
+        syntax tree nodes to abstract syntax tree nodes.  If not
+        given, no conversion is done and the syntax tree produced is
+        the concrete syntax tree.  If given, it must be a function of
+        two arguments, the first being the grammar (a grammar.Grammar
+        instance), and the second being the concrete syntax tree node
+        to be converted.  The syntax tree is converted from the bottom
+        up.
+
+        A concrete syntax tree node is a (type, nodes) tuple, where
+        type is the node type (a token or symbol number) and nodes
+        is a list of children for symbols, and None for tokens.
+
+        An abstract syntax tree node may be anything; this is entirely
+        up to the converter function.
+
+        """
+        self.grammar = grammar
+        self.convert_node = convert_node
+        self.convert_leaf = convert_leaf
+
+        # Each stack entry is a tuple: (dfa, state, node).
+        # A node is a tuple: (type, children),
+        # where children is a list of nodes or None
+        newnode = (start, [])
+        stackentry = (self.grammar.dfas[start], 0, newnode)
+        self.stack = [stackentry]
+        self.rootnode = None
+        self.error_recovery = error_recovery
+
+    def parse(self, tokenizer):
+        for type_, value, start_pos, prefix in tokenizer:
+            if self.addtoken(type_, value, start_pos, prefix):
+                break
+        else:
+            # We never broke out -- EOF is too soon -- Unfinished statement.
+            # However, the error recovery might have added the token again, if
+            # the stack is empty, we're fine.
+            if self.stack:
+                raise InternalParseError("incomplete input", type_, value, start_pos)
+        return self.rootnode
+
+    def addtoken(self, type_, value, start_pos, prefix):
+        """Add a token; return True if this is the end of the program."""
+        ilabel = token_to_ilabel(self.grammar, type_, value)
+
+        # Loop until the token is shifted; may raise exceptions
+        while True:
+            dfa, state, node = self.stack[-1]
+            states, first = dfa
+            arcs = states[state]
+            # Look for a state with this label
+            for i, newstate in arcs:
+                t, v = self.grammar.labels[i]
+                if ilabel == i:
+                    # Look it up in the list of labels
+                    assert t < 256
+                    # Shift a token; we're done with it
+                    self.shift(type_, value, newstate, prefix, start_pos)
+                    # Pop while we are in an accept-only state
+                    state = newstate
+                    while states[state] == [(0, state)]:
+                        self.pop()
+                        if not self.stack:
+                            # Done parsing!
+                            return True
+                        dfa, state, node = self.stack[-1]
+                        states, first = dfa
+                    # Done with this token
+                    return False
+                elif t >= 256:
+                    # See if it's a symbol and if we're in its first set
+                    itsdfa = self.grammar.dfas[t]
+                    itsstates, itsfirst = itsdfa
+                    if ilabel in itsfirst:
+                        # Push a symbol
+                        self.push(t, itsdfa, newstate)
+                        break  # To continue the outer while loop
+            else:
+                if (0, state) in arcs:
+                    # An accepting state, pop it and try something else
+                    self.pop()
+                    if not self.stack:
+                        # Done parsing, but another token is input
+                        raise InternalParseError("too much input", type_, value, start_pos)
+                else:
+                    self.error_recovery(self.grammar, self.stack, arcs, type_,
+                                        value, start_pos, prefix, self.addtoken)
+                    break
+
+    def shift(self, type_, value, newstate, prefix, start_pos):
+        """Shift a token.  (Internal)"""
+        dfa, state, node = self.stack[-1]
+        newnode = self.convert_leaf(self.grammar, type_, value, prefix, start_pos)
+        node[-1].append(newnode)
+        self.stack[-1] = (dfa, newstate, node)
+
+    def push(self, type_, newdfa, newstate):
+        """Push a nonterminal.  (Internal)"""
+        dfa, state, node = self.stack[-1]
+        newnode = (type_, [])
+        self.stack[-1] = (dfa, newstate, node)
+        self.stack.append((newdfa, 0, newnode))
+
+    def pop(self):
+        """Pop a nonterminal.  (Internal)"""
+        popdfa, popstate, (type_, children) = self.stack.pop()
+        # If there's exactly one child, return that child instead of creating a
+        # new node.  We still create expr_stmt and file_input though, because a
+        # lot of Jedi depends on its logic.
+        if len(children) == 1:
+            newnode = children[0]
+        else:
+            newnode = self.convert_node(self.grammar, type_, children)
+
+        try:
+            # Equal to:
+            # dfa, state, node = self.stack[-1]
+            # symbol, children = node
+            self.stack[-1][2][1].append(newnode)
+        except IndexError:
+            # Stack is empty, set the rootnode.
+            self.rootnode = newnode

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/pgen2/pgen.py

@@ -0,0 +1,394 @@
+# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+# Modifications:
+# Copyright 2014 David Halter. Integration into Jedi.
+# Modifications are dual-licensed: MIT and PSF.
+
+# Pgen imports
+from . import grammar
+from jedi.parser import token
+from jedi.parser import tokenize
+
+
+class ParserGenerator(object):
+    def __init__(self, filename, stream=None):
+        close_stream = None
+        if stream is None:
+            stream = open(filename)
+            close_stream = stream.close
+        self.filename = filename
+        self.stream = stream
+        self.generator = tokenize.generate_tokens(stream.readline)
+        self.gettoken()  # Initialize lookahead
+        self.dfas, self.startsymbol = self.parse()
+        if close_stream is not None:
+            close_stream()
+        self.first = {}  # map from symbol name to set of tokens
+        self.addfirstsets()
+
+    def make_grammar(self):
+        c = grammar.Grammar()
+        names = list(self.dfas.keys())
+        names.sort()
+        names.remove(self.startsymbol)
+        names.insert(0, self.startsymbol)
+        for name in names:
+            i = 256 + len(c.symbol2number)
+            c.symbol2number[name] = i
+            c.number2symbol[i] = name
+        for name in names:
+            dfa = self.dfas[name]
+            states = []
+            for state in dfa:
+                arcs = []
+                for label, next in state.arcs.items():
+                    arcs.append((self.make_label(c, label), dfa.index(next)))
+                if state.isfinal:
+                    arcs.append((0, dfa.index(state)))
+                states.append(arcs)
+            c.states.append(states)
+            c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
+        c.start = c.symbol2number[self.startsymbol]
+        return c
+
+    def make_first(self, c, name):
+        rawfirst = self.first[name]
+        first = {}
+        for label in rawfirst:
+            ilabel = self.make_label(c, label)
+            ##assert ilabel not in first # XXX failed on <> ... !=
+            first[ilabel] = 1
+        return first
+
+    def make_label(self, c, label):
+        # XXX Maybe this should be a method on a subclass of converter?
+        ilabel = len(c.labels)
+        if label[0].isalpha():
+            # Either a symbol name or a named token
+            if label in c.symbol2number:
+                # A symbol name (a non-terminal)
+                if label in c.symbol2label:
+                    return c.symbol2label[label]
+                else:
+                    c.labels.append((c.symbol2number[label], None))
+                    c.symbol2label[label] = ilabel
+                    return ilabel
+            else:
+                # A named token (NAME, NUMBER, STRING)
+                itoken = getattr(token, label, None)
+                assert isinstance(itoken, int), label
+                assert itoken in token.tok_name, label
+                if itoken in c.tokens:
+                    return c.tokens[itoken]
+                else:
+                    c.labels.append((itoken, None))
+                    c.tokens[itoken] = ilabel
+                    return ilabel
+        else:
+            # Either a keyword or an operator
+            assert label[0] in ('"', "'"), label
+            value = eval(label)
+            if value[0].isalpha():
+                # A keyword
+                if value in c.keywords:
+                    return c.keywords[value]
+                else:
+                    c.labels.append((token.NAME, value))
+                    c.keywords[value] = ilabel
+                    return ilabel
+            else:
+                # An operator (any non-numeric token)
+                itoken = token.opmap[value]  # Fails if unknown token
+                if itoken in c.tokens:
+                    return c.tokens[itoken]
+                else:
+                    c.labels.append((itoken, None))
+                    c.tokens[itoken] = ilabel
+                    return ilabel
+
+    def addfirstsets(self):
+        names = list(self.dfas.keys())
+        names.sort()
+        for name in names:
+            if name not in self.first:
+                self.calcfirst(name)
+            #print name, self.first[name].keys()
+
+    def calcfirst(self, name):
+        dfa = self.dfas[name]
+        self.first[name] = None  # dummy to detect left recursion
+        state = dfa[0]
+        totalset = {}
+        overlapcheck = {}
+        for label, next in state.arcs.items():
+            if label in self.dfas:
+                if label in self.first:
+                    fset = self.first[label]
+                    if fset is None:
+                        raise ValueError("recursion for rule %r" % name)
+                else:
+                    self.calcfirst(label)
+                    fset = self.first[label]
+                totalset.update(fset)
+                overlapcheck[label] = fset
+            else:
+                totalset[label] = 1
+                overlapcheck[label] = {label: 1}
+        inverse = {}
+        for label, itsfirst in overlapcheck.items():
+            for symbol in itsfirst:
+                if symbol in inverse:
+                    raise ValueError("rule %s is ambiguous; %s is in the"
+                                     " first sets of %s as well as %s" %
+                                     (name, symbol, label, inverse[symbol]))
+                inverse[symbol] = label
+        self.first[name] = totalset
+
+    def parse(self):
+        dfas = {}
+        startsymbol = None
+        # MSTART: (NEWLINE | RULE)* ENDMARKER
+        while self.type != token.ENDMARKER:
+            while self.type == token.NEWLINE:
+                self.gettoken()
+            # RULE: NAME ':' RHS NEWLINE
+            name = self.expect(token.NAME)
+            self.expect(token.OP, ":")
+            a, z = self.parse_rhs()
+            self.expect(token.NEWLINE)
+            #self.dump_nfa(name, a, z)
+            dfa = self.make_dfa(a, z)
+            #self.dump_dfa(name, dfa)
+            # oldlen = len(dfa)
+            self.simplify_dfa(dfa)
+            # newlen = len(dfa)
+            dfas[name] = dfa
+            #print name, oldlen, newlen
+            if startsymbol is None:
+                startsymbol = name
+        return dfas, startsymbol
+
+    def make_dfa(self, start, finish):
+        # To turn an NFA into a DFA, we define the states of the DFA
+        # to correspond to *sets* of states of the NFA.  Then do some
+        # state reduction.  Let's represent sets as dicts with 1 for
+        # values.
+        assert isinstance(start, NFAState)
+        assert isinstance(finish, NFAState)
+
+        def closure(state):
+            base = {}
+            addclosure(state, base)
+            return base
+
+        def addclosure(state, base):
+            assert isinstance(state, NFAState)
+            if state in base:
+                return
+            base[state] = 1
+            for label, next in state.arcs:
+                if label is None:
+                    addclosure(next, base)
+
+        states = [DFAState(closure(start), finish)]
+        for state in states:  # NB states grows while we're iterating
+            arcs = {}
+            for nfastate in state.nfaset:
+                for label, next in nfastate.arcs:
+                    if label is not None:
+                        addclosure(next, arcs.setdefault(label, {}))
+            for label, nfaset in arcs.items():
+                for st in states:
+                    if st.nfaset == nfaset:
+                        break
+                else:
+                    st = DFAState(nfaset, finish)
+                    states.append(st)
+                state.addarc(st, label)
+        return states  # List of DFAState instances; first one is start
+
+    def dump_nfa(self, name, start, finish):
+        print("Dump of NFA for", name)
+        todo = [start]
+        for i, state in enumerate(todo):
+            print("  State", i, state is finish and "(final)" or "")
+            for label, next in state.arcs:
+                if next in todo:
+                    j = todo.index(next)
+                else:
+                    j = len(todo)
+                    todo.append(next)
+                if label is None:
+                    print("    -> %d" % j)
+                else:
+                    print("    %s -> %d" % (label, j))
+
+    def dump_dfa(self, name, dfa):
+        print("Dump of DFA for", name)
+        for i, state in enumerate(dfa):
+            print("  State", i, state.isfinal and "(final)" or "")
+            for label, next in state.arcs.items():
+                print("    %s -> %d" % (label, dfa.index(next)))
+
+    def simplify_dfa(self, dfa):
+        # This is not theoretically optimal, but works well enough.
+        # Algorithm: repeatedly look for two states that have the same
+        # set of arcs (same labels pointing to the same nodes) and
+        # unify them, until things stop changing.
+
+        # dfa is a list of DFAState instances
+        changes = True
+        while changes:
+            changes = False
+            for i, state_i in enumerate(dfa):
+                for j in range(i + 1, len(dfa)):
+                    state_j = dfa[j]
+                    if state_i == state_j:
+                        #print "  unify", i, j
+                        del dfa[j]
+                        for state in dfa:
+                            state.unifystate(state_j, state_i)
+                        changes = True
+                        break
+
+    def parse_rhs(self):
+        # RHS: ALT ('|' ALT)*
+        a, z = self.parse_alt()
+        if self.value != "|":
+            return a, z
+        else:
+            aa = NFAState()
+            zz = NFAState()
+            aa.addarc(a)
+            z.addarc(zz)
+            while self.value == "|":
+                self.gettoken()
+                a, z = self.parse_alt()
+                aa.addarc(a)
+                z.addarc(zz)
+            return aa, zz
+
+    def parse_alt(self):
+        # ALT: ITEM+
+        a, b = self.parse_item()
+        while (self.value in ("(", "[") or
+               self.type in (token.NAME, token.STRING)):
+            c, d = self.parse_item()
+            b.addarc(c)
+            b = d
+        return a, b
+
+    def parse_item(self):
+        # ITEM: '[' RHS ']' | ATOM ['+' | '*']
+        if self.value == "[":
+            self.gettoken()
+            a, z = self.parse_rhs()
+            self.expect(token.OP, "]")
+            a.addarc(z)
+            return a, z
+        else:
+            a, z = self.parse_atom()
+            value = self.value
+            if value not in ("+", "*"):
+                return a, z
+            self.gettoken()
+            z.addarc(a)
+            if value == "+":
+                return a, z
+            else:
+                return a, a
+
+    def parse_atom(self):
+        # ATOM: '(' RHS ')' | NAME | STRING
+        if self.value == "(":
+            self.gettoken()
+            a, z = self.parse_rhs()
+            self.expect(token.OP, ")")
+            return a, z
+        elif self.type in (token.NAME, token.STRING):
+            a = NFAState()
+            z = NFAState()
+            a.addarc(z, self.value)
+            self.gettoken()
+            return a, z
+        else:
+            self.raise_error("expected (...) or NAME or STRING, got %s/%s",
+                             self.type, self.value)
+
+    def expect(self, type, value=None):
+        if self.type != type or (value is not None and self.value != value):
+            self.raise_error("expected %s/%s, got %s/%s",
+                             type, value, self.type, self.value)
+        value = self.value
+        self.gettoken()
+        return value
+
+    def gettoken(self):
+        tup = next(self.generator)
+        while tup[0] in (token.COMMENT, token.NL):
+            tup = next(self.generator)
+        self.type, self.value, self.begin, prefix = tup
+        #print tokenize.tok_name[self.type], repr(self.value)
+
+    def raise_error(self, msg, *args):
+        if args:
+            try:
+                msg = msg % args
+            except:
+                msg = " ".join([msg] + list(map(str, args)))
+        line = open(self.filename).readlines()[self.begin[0]]
+        raise SyntaxError(msg, (self.filename, self.begin[0],
+                                self.begin[1], line))
+
+
+class NFAState(object):
+    def __init__(self):
+        self.arcs = []  # list of (label, NFAState) pairs
+
+    def addarc(self, next, label=None):
+        assert label is None or isinstance(label, str)
+        assert isinstance(next, NFAState)
+        self.arcs.append((label, next))
+
+
+class DFAState(object):
+    def __init__(self, nfaset, final):
+        assert isinstance(nfaset, dict)
+        assert isinstance(next(iter(nfaset)), NFAState)
+        assert isinstance(final, NFAState)
+        self.nfaset = nfaset
+        self.isfinal = final in nfaset
+        self.arcs = {}  # map from label to DFAState
+
+    def addarc(self, next, label):
+        assert isinstance(label, str)
+        assert label not in self.arcs
+        assert isinstance(next, DFAState)
+        self.arcs[label] = next
+
+    def unifystate(self, old, new):
+        for label, next in self.arcs.items():
+            if next is old:
+                self.arcs[label] = new
+
+    def __eq__(self, other):
+        # Equality test -- ignore the nfaset instance variable
+        assert isinstance(other, DFAState)
+        if self.isfinal != other.isfinal:
+            return False
+        # Can't just return self.arcs == other.arcs, because that
+        # would invoke this method recursively, with cycles...
+        if len(self.arcs) != len(other.arcs):
+            return False
+        for label, next in self.arcs.items():
+            if next is not other.arcs.get(label):
+                return False
+        return True
+
+    __hash__ = None  # For Py3 compatibility.
+
+
+def generate_grammar(filename="Grammar.txt"):
+    p = ParserGenerator(filename)
+    return p.make_grammar()

vim-plugins/bundle/jedi-vim/jedi/jedi/parser/token.py

@@ -0,0 +1,90 @@
+from __future__ import absolute_import
+
+from jedi._compatibility import is_py3, is_py35
+from token import *
+
+
+COMMENT = N_TOKENS
+tok_name[COMMENT] = 'COMMENT'
+N_TOKENS += 1
+
+NL = N_TOKENS
+tok_name[NL] = 'NL'
+N_TOKENS += 1
+
+if is_py3:
+    BACKQUOTE = N_TOKENS
+    tok_name[BACKQUOTE] = 'BACKQUOTE'
+    N_TOKENS += 1
+else:
+    RARROW = N_TOKENS
+    tok_name[RARROW] = 'RARROW'
+    N_TOKENS += 1
+    ELLIPSIS = N_TOKENS
+    tok_name[ELLIPSIS] = 'ELLIPSIS'
+    N_TOKENS += 1
+
+if not is_py35:
+    ATEQUAL = N_TOKENS
+    tok_name[ATEQUAL] = 'ATEQUAL'
+    N_TOKENS += 1
+
+
+
+# Map from operator to number (since tokenize doesn't do this)
+
+opmap_raw = """\
+( LPAR
+) RPAR
+[ LSQB
+] RSQB
+: COLON
+, COMMA
+; SEMI
++ PLUS
+- MINUS
+* STAR
+/ SLASH
+| VBAR
+& AMPER
+< LESS
+> GREATER
+= EQUAL
+. DOT
+% PERCENT
+` BACKQUOTE
+{ LBRACE
+} RBRACE
+@ AT
+== EQEQUAL
+!= NOTEQUAL
+<> NOTEQUAL
+<= LESSEQUAL
+>= GREATEREQUAL
+~ TILDE
+^ CIRCUMFLEX
+<< LEFTSHIFT
+>> RIGHTSHIFT
+** DOUBLESTAR
++= PLUSEQUAL
+-= MINEQUAL
+*= STAREQUAL
+/= SLASHEQUAL
+%= PERCENTEQUAL
+&= AMPEREQUAL
+|= VBAREQUAL
+@= ATEQUAL
+^= CIRCUMFLEXEQUAL
+<<= LEFTSHIFTEQUAL
+>>= RIGHTSHIFTEQUAL
+**= DOUBLESTAREQUAL
+// DOUBLESLASH
+//= DOUBLESLASHEQUAL
+-> RARROW
+... ELLIPSIS
+"""
+
+opmap = {}
+for line in opmap_raw.splitlines():
+    op, name = line.split()
+    opmap[op] = globals()[name]