Uploaded Test files
This commit is contained in:
parent
f584ad9d97
commit
2e81cb7d99
16627 changed files with 2065359 additions and 102444 deletions
|
|
@ -0,0 +1,404 @@
|
|||
"""
|
||||
Makes it possible to do the compiled analysis in a subprocess. This has two
|
||||
goals:
|
||||
|
||||
1. Making it safer - Segfaults and RuntimeErrors as well as stdout/stderr can
|
||||
be ignored and dealt with.
|
||||
2. Make it possible to handle different Python versions as well as virtualenvs.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import socket
|
||||
import errno
|
||||
import traceback
|
||||
from functools import partial
|
||||
from threading import Thread
|
||||
try:
|
||||
from queue import Queue, Empty
|
||||
except ImportError:
|
||||
from Queue import Queue, Empty # python 2.7
|
||||
|
||||
from jedi._compatibility import queue, is_py3, force_unicode, \
|
||||
pickle_dump, pickle_load, GeneralizedPopen, weakref
|
||||
from jedi import debug
|
||||
from jedi.cache import memoize_method
|
||||
from jedi.inference.compiled.subprocess import functions
|
||||
from jedi.inference.compiled.access import DirectObjectAccess, AccessPath, \
|
||||
SignatureParam
|
||||
from jedi.api.exceptions import InternalError
|
||||
|
||||
|
||||
_MAIN_PATH = os.path.join(os.path.dirname(__file__), '__main__.py')
|
||||
|
||||
|
||||
def _enqueue_output(out, queue):
|
||||
for line in iter(out.readline, b''):
|
||||
queue.put(line)
|
||||
|
||||
|
||||
def _add_stderr_to_debug(stderr_queue):
|
||||
while True:
|
||||
# Try to do some error reporting from the subprocess and print its
|
||||
# stderr contents.
|
||||
try:
|
||||
line = stderr_queue.get_nowait()
|
||||
line = line.decode('utf-8', 'replace')
|
||||
debug.warning('stderr output: %s' % line.rstrip('\n'))
|
||||
except Empty:
|
||||
break
|
||||
|
||||
|
||||
def _get_function(name):
|
||||
return getattr(functions, name)
|
||||
|
||||
|
||||
def _cleanup_process(process, thread):
|
||||
try:
|
||||
process.kill()
|
||||
process.wait()
|
||||
except OSError:
|
||||
# Raised if the process is already killed.
|
||||
pass
|
||||
thread.join()
|
||||
for stream in [process.stdin, process.stdout, process.stderr]:
|
||||
try:
|
||||
stream.close()
|
||||
except OSError:
|
||||
# Raised if the stream is broken.
|
||||
pass
|
||||
|
||||
|
||||
class _InferenceStateProcess(object):
|
||||
def __init__(self, inference_state):
|
||||
self._inference_state_weakref = weakref.ref(inference_state)
|
||||
self._inference_state_id = id(inference_state)
|
||||
self._handles = {}
|
||||
|
||||
def get_or_create_access_handle(self, obj):
|
||||
id_ = id(obj)
|
||||
try:
|
||||
return self.get_access_handle(id_)
|
||||
except KeyError:
|
||||
access = DirectObjectAccess(self._inference_state_weakref(), obj)
|
||||
handle = AccessHandle(self, access, id_)
|
||||
self.set_access_handle(handle)
|
||||
return handle
|
||||
|
||||
def get_access_handle(self, id_):
|
||||
return self._handles[id_]
|
||||
|
||||
def set_access_handle(self, handle):
|
||||
self._handles[handle.id] = handle
|
||||
|
||||
|
||||
class InferenceStateSameProcess(_InferenceStateProcess):
|
||||
"""
|
||||
Basically just an easy access to functions.py. It has the same API
|
||||
as InferenceStateSubprocess and does the same thing without using a subprocess.
|
||||
This is necessary for the Interpreter process.
|
||||
"""
|
||||
def __getattr__(self, name):
|
||||
return partial(_get_function(name), self._inference_state_weakref())
|
||||
|
||||
|
||||
class InferenceStateSubprocess(_InferenceStateProcess):
|
||||
def __init__(self, inference_state, compiled_subprocess):
|
||||
super(InferenceStateSubprocess, self).__init__(inference_state)
|
||||
self._used = False
|
||||
self._compiled_subprocess = compiled_subprocess
|
||||
|
||||
def __getattr__(self, name):
|
||||
func = _get_function(name)
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
self._used = True
|
||||
|
||||
result = self._compiled_subprocess.run(
|
||||
self._inference_state_weakref(),
|
||||
func,
|
||||
args=args,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
# IMO it should be possible to create a hook in pickle.load to
|
||||
# mess with the loaded objects. However it's extremely complicated
|
||||
# to work around this so just do it with this call. ~ dave
|
||||
return self._convert_access_handles(result)
|
||||
|
||||
return wrapper
|
||||
|
||||
def _convert_access_handles(self, obj):
|
||||
if isinstance(obj, SignatureParam):
|
||||
return SignatureParam(*self._convert_access_handles(tuple(obj)))
|
||||
elif isinstance(obj, tuple):
|
||||
return tuple(self._convert_access_handles(o) for o in obj)
|
||||
elif isinstance(obj, list):
|
||||
return [self._convert_access_handles(o) for o in obj]
|
||||
elif isinstance(obj, AccessHandle):
|
||||
try:
|
||||
# Rewrite the access handle to one we're already having.
|
||||
obj = self.get_access_handle(obj.id)
|
||||
except KeyError:
|
||||
obj.add_subprocess(self)
|
||||
self.set_access_handle(obj)
|
||||
elif isinstance(obj, AccessPath):
|
||||
return AccessPath(self._convert_access_handles(obj.accesses))
|
||||
return obj
|
||||
|
||||
def __del__(self):
|
||||
if self._used and not self._compiled_subprocess.is_crashed:
|
||||
self._compiled_subprocess.delete_inference_state(self._inference_state_id)
|
||||
|
||||
|
||||
class CompiledSubprocess(object):
|
||||
is_crashed = False
|
||||
# Start with 2, gets set after _get_info.
|
||||
_pickle_protocol = 2
|
||||
|
||||
def __init__(self, executable, env_vars=None):
|
||||
self._executable = executable
|
||||
self._env_vars = env_vars
|
||||
self._inference_state_deletion_queue = queue.deque()
|
||||
self._cleanup_callable = lambda: None
|
||||
|
||||
def __repr__(self):
|
||||
pid = os.getpid()
|
||||
return '<%s _executable=%r, _pickle_protocol=%r, is_crashed=%r, pid=%r>' % (
|
||||
self.__class__.__name__,
|
||||
self._executable,
|
||||
self._pickle_protocol,
|
||||
self.is_crashed,
|
||||
pid,
|
||||
)
|
||||
|
||||
@memoize_method
|
||||
def _get_process(self):
|
||||
debug.dbg('Start environment subprocess %s', self._executable)
|
||||
parso_path = sys.modules['parso'].__file__
|
||||
args = (
|
||||
self._executable,
|
||||
_MAIN_PATH,
|
||||
os.path.dirname(os.path.dirname(parso_path)),
|
||||
'.'.join(str(x) for x in sys.version_info[:3]),
|
||||
)
|
||||
process = GeneralizedPopen(
|
||||
args,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
# Use system default buffering on Python 2 to improve performance
|
||||
# (this is already the case on Python 3).
|
||||
bufsize=-1,
|
||||
env=self._env_vars
|
||||
)
|
||||
self._stderr_queue = Queue()
|
||||
self._stderr_thread = t = Thread(
|
||||
target=_enqueue_output,
|
||||
args=(process.stderr, self._stderr_queue)
|
||||
)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
# Ensure the subprocess is properly cleaned up when the object
|
||||
# is garbage collected.
|
||||
self._cleanup_callable = weakref.finalize(self,
|
||||
_cleanup_process,
|
||||
process,
|
||||
t)
|
||||
return process
|
||||
|
||||
def run(self, inference_state, function, args=(), kwargs={}):
|
||||
# Delete old inference_states.
|
||||
while True:
|
||||
try:
|
||||
inference_state_id = self._inference_state_deletion_queue.pop()
|
||||
except IndexError:
|
||||
break
|
||||
else:
|
||||
self._send(inference_state_id, None)
|
||||
|
||||
assert callable(function)
|
||||
return self._send(id(inference_state), function, args, kwargs)
|
||||
|
||||
def get_sys_path(self):
|
||||
return self._send(None, functions.get_sys_path, (), {})
|
||||
|
||||
def _kill(self):
|
||||
self.is_crashed = True
|
||||
self._cleanup_callable()
|
||||
|
||||
def _send(self, inference_state_id, function, args=(), kwargs={}):
|
||||
if self.is_crashed:
|
||||
raise InternalError("The subprocess %s has crashed." % self._executable)
|
||||
|
||||
if not is_py3:
|
||||
# Python 2 compatibility
|
||||
kwargs = {force_unicode(key): value for key, value in kwargs.items()}
|
||||
|
||||
data = inference_state_id, function, args, kwargs
|
||||
try:
|
||||
pickle_dump(data, self._get_process().stdin, self._pickle_protocol)
|
||||
except (socket.error, IOError) as e:
|
||||
# Once Python2 will be removed we can just use `BrokenPipeError`.
|
||||
# Also, somehow in windows it returns EINVAL instead of EPIPE if
|
||||
# the subprocess dies.
|
||||
if e.errno not in (errno.EPIPE, errno.EINVAL):
|
||||
# Not a broken pipe
|
||||
raise
|
||||
self._kill()
|
||||
raise InternalError("The subprocess %s was killed. Maybe out of memory?"
|
||||
% self._executable)
|
||||
|
||||
try:
|
||||
is_exception, traceback, result = pickle_load(self._get_process().stdout)
|
||||
except EOFError as eof_error:
|
||||
try:
|
||||
stderr = self._get_process().stderr.read().decode('utf-8', 'replace')
|
||||
except Exception as exc:
|
||||
stderr = '<empty/not available (%r)>' % exc
|
||||
self._kill()
|
||||
_add_stderr_to_debug(self._stderr_queue)
|
||||
raise InternalError(
|
||||
"The subprocess %s has crashed (%r, stderr=%s)." % (
|
||||
self._executable,
|
||||
eof_error,
|
||||
stderr,
|
||||
))
|
||||
|
||||
_add_stderr_to_debug(self._stderr_queue)
|
||||
|
||||
if is_exception:
|
||||
# Replace the attribute error message with a the traceback. It's
|
||||
# way more informative.
|
||||
result.args = (traceback,)
|
||||
raise result
|
||||
return result
|
||||
|
||||
def delete_inference_state(self, inference_state_id):
|
||||
"""
|
||||
Currently we are not deleting inference_state instantly. They only get
|
||||
deleted once the subprocess is used again. It would probably a better
|
||||
solution to move all of this into a thread. However, the memory usage
|
||||
of a single inference_state shouldn't be that high.
|
||||
"""
|
||||
# With an argument - the inference_state gets deleted.
|
||||
self._inference_state_deletion_queue.append(inference_state_id)
|
||||
|
||||
|
||||
class Listener(object):
|
||||
def __init__(self, pickle_protocol):
|
||||
self._inference_states = {}
|
||||
# TODO refactor so we don't need to process anymore just handle
|
||||
# controlling.
|
||||
self._process = _InferenceStateProcess(Listener)
|
||||
self._pickle_protocol = pickle_protocol
|
||||
|
||||
def _get_inference_state(self, function, inference_state_id):
|
||||
from jedi.inference import InferenceState
|
||||
|
||||
try:
|
||||
inference_state = self._inference_states[inference_state_id]
|
||||
except KeyError:
|
||||
from jedi import InterpreterEnvironment
|
||||
inference_state = InferenceState(
|
||||
# The project is not actually needed. Nothing should need to
|
||||
# access it.
|
||||
project=None,
|
||||
environment=InterpreterEnvironment()
|
||||
)
|
||||
self._inference_states[inference_state_id] = inference_state
|
||||
return inference_state
|
||||
|
||||
def _run(self, inference_state_id, function, args, kwargs):
|
||||
if inference_state_id is None:
|
||||
return function(*args, **kwargs)
|
||||
elif function is None:
|
||||
del self._inference_states[inference_state_id]
|
||||
else:
|
||||
inference_state = self._get_inference_state(function, inference_state_id)
|
||||
|
||||
# Exchange all handles
|
||||
args = list(args)
|
||||
for i, arg in enumerate(args):
|
||||
if isinstance(arg, AccessHandle):
|
||||
args[i] = inference_state.compiled_subprocess.get_access_handle(arg.id)
|
||||
for key, value in kwargs.items():
|
||||
if isinstance(value, AccessHandle):
|
||||
kwargs[key] = inference_state.compiled_subprocess.get_access_handle(value.id)
|
||||
|
||||
return function(inference_state, *args, **kwargs)
|
||||
|
||||
def listen(self):
|
||||
stdout = sys.stdout
|
||||
# Mute stdout. Nobody should actually be able to write to it,
|
||||
# because stdout is used for IPC.
|
||||
sys.stdout = open(os.devnull, 'w')
|
||||
stdin = sys.stdin
|
||||
if sys.version_info[0] > 2:
|
||||
stdout = stdout.buffer
|
||||
stdin = stdin.buffer
|
||||
# Python 2 opens streams in text mode on Windows. Set stdout and stdin
|
||||
# to binary mode.
|
||||
elif sys.platform == 'win32':
|
||||
import msvcrt
|
||||
msvcrt.setmode(stdout.fileno(), os.O_BINARY)
|
||||
msvcrt.setmode(stdin.fileno(), os.O_BINARY)
|
||||
|
||||
while True:
|
||||
try:
|
||||
payload = pickle_load(stdin)
|
||||
except EOFError:
|
||||
# It looks like the parent process closed.
|
||||
# Don't make a big fuss here and just exit.
|
||||
exit(0)
|
||||
try:
|
||||
result = False, None, self._run(*payload)
|
||||
except Exception as e:
|
||||
result = True, traceback.format_exc(), e
|
||||
|
||||
pickle_dump(result, stdout, self._pickle_protocol)
|
||||
|
||||
|
||||
class AccessHandle(object):
|
||||
def __init__(self, subprocess, access, id_):
|
||||
self.access = access
|
||||
self._subprocess = subprocess
|
||||
self.id = id_
|
||||
|
||||
def add_subprocess(self, subprocess):
|
||||
self._subprocess = subprocess
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
detail = self.access
|
||||
except AttributeError:
|
||||
detail = '#' + str(self.id)
|
||||
return '<%s of %s>' % (self.__class__.__name__, detail)
|
||||
|
||||
def __getstate__(self):
|
||||
return self.id
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.id = state
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name in ('id', 'access') or name.startswith('_'):
|
||||
raise AttributeError("Something went wrong with unpickling")
|
||||
|
||||
# if not is_py3: print >> sys.stderr, name
|
||||
# print('getattr', name, file=sys.stderr)
|
||||
return partial(self._workaround, force_unicode(name))
|
||||
|
||||
def _workaround(self, name, *args, **kwargs):
|
||||
"""
|
||||
TODO Currently we're passing slice objects around. This should not
|
||||
happen. They are also the only unhashable objects that we're passing
|
||||
around.
|
||||
"""
|
||||
if args and isinstance(args[0], slice):
|
||||
return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
|
||||
return self._cached_results(name, *args, **kwargs)
|
||||
|
||||
@memoize_method
|
||||
def _cached_results(self, name, *args, **kwargs):
|
||||
return self._subprocess.get_compiled_method_return(self.id, name, *args, **kwargs)
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def _get_paths():
|
||||
# Get the path to jedi.
|
||||
_d = os.path.dirname
|
||||
_jedi_path = _d(_d(_d(_d(_d(__file__)))))
|
||||
_parso_path = sys.argv[1]
|
||||
# The paths are the directory that jedi and parso lie in.
|
||||
return {'jedi': _jedi_path, 'parso': _parso_path}
|
||||
|
||||
|
||||
# Remove the first entry, because it's simply a directory entry that equals
|
||||
# this directory.
|
||||
del sys.path[0]
|
||||
|
||||
if sys.version_info > (3, 4):
|
||||
from importlib.machinery import PathFinder
|
||||
|
||||
class _ExactImporter(object):
|
||||
def __init__(self, path_dct):
|
||||
self._path_dct = path_dct
|
||||
|
||||
def find_module(self, fullname, path=None):
|
||||
if path is None and fullname in self._path_dct:
|
||||
p = self._path_dct[fullname]
|
||||
loader = PathFinder.find_module(fullname, path=[p])
|
||||
return loader
|
||||
return None
|
||||
|
||||
# Try to import jedi/parso.
|
||||
sys.meta_path.insert(0, _ExactImporter(_get_paths()))
|
||||
from jedi.inference.compiled import subprocess # NOQA
|
||||
sys.meta_path.pop(0)
|
||||
else:
|
||||
import imp
|
||||
|
||||
def load(name):
|
||||
paths = list(_get_paths().values())
|
||||
fp, pathname, description = imp.find_module(name, paths)
|
||||
return imp.load_module(name, fp, pathname, description)
|
||||
|
||||
load('parso')
|
||||
load('jedi')
|
||||
from jedi.inference.compiled import subprocess # NOQA
|
||||
|
||||
from jedi._compatibility import highest_pickle_protocol # noqa: E402
|
||||
|
||||
|
||||
# Retrieve the pickle protocol.
|
||||
host_sys_version = [int(x) for x in sys.argv[2].split('.')]
|
||||
pickle_protocol = highest_pickle_protocol([sys.version_info, host_sys_version])
|
||||
# And finally start the client.
|
||||
subprocess.Listener(pickle_protocol=pickle_protocol).listen()
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,115 @@
|
|||
from __future__ import print_function
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import inspect
|
||||
|
||||
from jedi._compatibility import find_module, cast_path, force_unicode, \
|
||||
all_suffixes, scandir
|
||||
from jedi.inference.compiled import access
|
||||
from jedi import debug
|
||||
from jedi import parser_utils
|
||||
|
||||
|
||||
def get_sys_path():
|
||||
return list(map(cast_path, sys.path))
|
||||
|
||||
|
||||
def load_module(inference_state, **kwargs):
|
||||
return access.load_module(inference_state, **kwargs)
|
||||
|
||||
|
||||
def get_compiled_method_return(inference_state, id, attribute, *args, **kwargs):
|
||||
handle = inference_state.compiled_subprocess.get_access_handle(id)
|
||||
return getattr(handle.access, attribute)(*args, **kwargs)
|
||||
|
||||
|
||||
def create_simple_object(inference_state, obj):
|
||||
return access.create_access_path(inference_state, obj)
|
||||
|
||||
|
||||
def get_module_info(inference_state, sys_path=None, full_name=None, **kwargs):
|
||||
"""
|
||||
Returns Tuple[Union[NamespaceInfo, FileIO, None], Optional[bool]]
|
||||
"""
|
||||
if sys_path is not None:
|
||||
sys.path, temp = sys_path, sys.path
|
||||
try:
|
||||
return find_module(full_name=full_name, **kwargs)
|
||||
except ImportError:
|
||||
return None, None
|
||||
finally:
|
||||
if sys_path is not None:
|
||||
sys.path = temp
|
||||
|
||||
|
||||
def get_builtin_module_names(inference_state):
|
||||
return list(map(force_unicode, sys.builtin_module_names))
|
||||
|
||||
|
||||
def _test_raise_error(inference_state, exception_type):
|
||||
"""
|
||||
Raise an error to simulate certain problems for unit tests.
|
||||
"""
|
||||
raise exception_type
|
||||
|
||||
|
||||
def _test_print(inference_state, stderr=None, stdout=None):
|
||||
"""
|
||||
Force some prints in the subprocesses. This exists for unit tests.
|
||||
"""
|
||||
if stderr is not None:
|
||||
print(stderr, file=sys.stderr)
|
||||
sys.stderr.flush()
|
||||
if stdout is not None:
|
||||
print(stdout)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _get_init_path(directory_path):
|
||||
"""
|
||||
The __init__ file can be searched in a directory. If found return it, else
|
||||
None.
|
||||
"""
|
||||
for suffix in all_suffixes():
|
||||
path = os.path.join(directory_path, '__init__' + suffix)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def safe_literal_eval(inference_state, value):
|
||||
return parser_utils.safe_literal_eval(value)
|
||||
|
||||
|
||||
def iter_module_names(*args, **kwargs):
|
||||
return list(_iter_module_names(*args, **kwargs))
|
||||
|
||||
|
||||
def _iter_module_names(inference_state, paths):
|
||||
# Python modules/packages
|
||||
for path in paths:
|
||||
try:
|
||||
dirs = scandir(path)
|
||||
except OSError:
|
||||
# The file might not exist or reading it might lead to an error.
|
||||
debug.warning("Not possible to list directory: %s", path)
|
||||
continue
|
||||
for dir_entry in dirs:
|
||||
name = dir_entry.name
|
||||
# First Namespaces then modules/stubs
|
||||
if dir_entry.is_dir():
|
||||
# pycache is obviously not an interestin namespace. Also the
|
||||
# name must be a valid identifier.
|
||||
# TODO use str.isidentifier, once Python 2 is removed
|
||||
if name != '__pycache__' and not re.search(r'\W|^\d', name):
|
||||
yield name
|
||||
else:
|
||||
if name.endswith('.pyi'): # Stub files
|
||||
modname = name[:-4]
|
||||
else:
|
||||
modname = inspect.getmodulename(name)
|
||||
|
||||
if modname and '.' not in modname:
|
||||
if modname != '__init__':
|
||||
yield modname
|
||||
Loading…
Add table
Add a link
Reference in a new issue