Uploaded Test files
This commit is contained in:
parent
f584ad9d97
commit
2e81cb7d99
16627 changed files with 2065359 additions and 102444 deletions
274
venv/Lib/site-packages/joblib/externals/loky/backend/semlock.py
vendored
Normal file
274
venv/Lib/site-packages/joblib/externals/loky/backend/semlock.py
vendored
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
###############################################################################
|
||||
# Ctypes implementation for posix semaphore.
|
||||
#
|
||||
# author: Thomas Moreau and Olivier Grisel
|
||||
#
|
||||
# adapted from cpython/Modules/_multiprocessing/semaphore.c (17/02/2017)
|
||||
# * use ctypes to access pthread semaphores and provide a full python
|
||||
# semaphore management.
|
||||
# * For OSX, as no sem_getvalue is not implemented, Semaphore with value > 1
|
||||
# are not guaranteed to work.
|
||||
# * Only work with LokyProcess on posix
|
||||
#
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import errno
|
||||
import ctypes
|
||||
import tempfile
|
||||
import threading
|
||||
from ctypes.util import find_library
|
||||
|
||||
# As we need to use ctypes return types for semlock object, failure value
|
||||
# needs to be cast to proper python value. Unix failure convention is to
|
||||
# return 0, whereas OSX returns -1
|
||||
SEM_FAILURE = ctypes.c_void_p(0).value
|
||||
if sys.platform == 'darwin':
|
||||
SEM_FAILURE = ctypes.c_void_p(-1).value
|
||||
|
||||
# Semaphore types
|
||||
RECURSIVE_MUTEX = 0
|
||||
SEMAPHORE = 1
|
||||
|
||||
# Semaphore constants
|
||||
SEM_OFLAG = ctypes.c_int(os.O_CREAT | os.O_EXCL)
|
||||
SEM_PERM = ctypes.c_int(384)
|
||||
|
||||
|
||||
class timespec(ctypes.Structure):
|
||||
_fields_ = [("tv_sec", ctypes.c_long), ("tv_nsec", ctypes.c_long)]
|
||||
|
||||
|
||||
if sys.platform != 'win32':
|
||||
pthread = ctypes.CDLL(find_library('pthread'), use_errno=True)
|
||||
pthread.sem_open.restype = ctypes.c_void_p
|
||||
pthread.sem_close.argtypes = [ctypes.c_void_p]
|
||||
pthread.sem_wait.argtypes = [ctypes.c_void_p]
|
||||
pthread.sem_trywait.argtypes = [ctypes.c_void_p]
|
||||
pthread.sem_post.argtypes = [ctypes.c_void_p]
|
||||
pthread.sem_getvalue.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
|
||||
pthread.sem_unlink.argtypes = [ctypes.c_char_p]
|
||||
if sys.platform != "darwin":
|
||||
pthread.sem_timedwait.argtypes = [ctypes.c_void_p,
|
||||
ctypes.POINTER(timespec)]
|
||||
|
||||
try:
|
||||
from threading import get_ident
|
||||
except ImportError:
|
||||
def get_ident():
|
||||
return threading.current_thread().ident
|
||||
|
||||
|
||||
if sys.version_info[:2] < (3, 3):
|
||||
class FileExistsError(OSError):
|
||||
pass
|
||||
|
||||
class FileNotFoundError(OSError):
|
||||
pass
|
||||
|
||||
|
||||
def sem_unlink(name):
|
||||
if pthread.sem_unlink(name.encode('ascii')) < 0:
|
||||
raiseFromErrno()
|
||||
|
||||
|
||||
def _sem_open(name, value=None):
|
||||
""" Construct or retrieve a semaphore with the given name
|
||||
|
||||
If value is None, try to retrieve an existing named semaphore.
|
||||
Else create a new semaphore with the given value
|
||||
"""
|
||||
if value is None:
|
||||
handle = pthread.sem_open(ctypes.c_char_p(name), 0)
|
||||
else:
|
||||
handle = pthread.sem_open(ctypes.c_char_p(name), SEM_OFLAG, SEM_PERM,
|
||||
ctypes.c_int(value))
|
||||
|
||||
if handle == SEM_FAILURE:
|
||||
e = ctypes.get_errno()
|
||||
if e == errno.EEXIST:
|
||||
raise FileExistsError("a semaphore named %s already exists" % name)
|
||||
elif e == errno.ENOENT:
|
||||
raise FileNotFoundError('cannot find semaphore named %s' % name)
|
||||
elif e == errno.ENOSYS:
|
||||
raise NotImplementedError('No semaphore implementation on this '
|
||||
'system')
|
||||
else:
|
||||
raiseFromErrno()
|
||||
|
||||
return handle
|
||||
|
||||
|
||||
def _sem_timedwait(handle, timeout):
|
||||
t_start = time.time()
|
||||
if sys.platform != "darwin":
|
||||
sec = int(timeout)
|
||||
tv_sec = int(t_start)
|
||||
nsec = int(1e9 * (timeout - sec) + .5)
|
||||
tv_nsec = int(1e9 * (t_start - tv_sec) + .5)
|
||||
deadline = timespec(sec+tv_sec, nsec+tv_nsec)
|
||||
deadline.tv_sec += int(deadline.tv_nsec / 1000000000)
|
||||
deadline.tv_nsec %= 1000000000
|
||||
return pthread.sem_timedwait(handle, ctypes.pointer(deadline))
|
||||
|
||||
# PERFORMANCE WARNING
|
||||
# No sem_timedwait on OSX so we implement our own method. This method can
|
||||
# degrade performances has the wait can have a latency up to 20 msecs
|
||||
deadline = t_start + timeout
|
||||
delay = 0
|
||||
now = time.time()
|
||||
while True:
|
||||
# Poll the sem file
|
||||
res = pthread.sem_trywait(handle)
|
||||
if res == 0:
|
||||
return 0
|
||||
else:
|
||||
e = ctypes.get_errno()
|
||||
if e != errno.EAGAIN:
|
||||
raiseFromErrno()
|
||||
|
||||
# check for timeout
|
||||
now = time.time()
|
||||
if now > deadline:
|
||||
ctypes.set_errno(errno.ETIMEDOUT)
|
||||
return -1
|
||||
|
||||
# calculate how much time left and check the delay is not too long
|
||||
# -- maximum is 20 msecs
|
||||
difference = (deadline - now)
|
||||
delay = min(delay, 20e-3, difference)
|
||||
|
||||
# Sleep and increase delay
|
||||
time.sleep(delay)
|
||||
delay += 1e-3
|
||||
|
||||
|
||||
class SemLock(object):
|
||||
"""ctypes wrapper to the unix semaphore"""
|
||||
|
||||
_rand = tempfile._RandomNameSequence()
|
||||
|
||||
def __init__(self, kind, value, maxvalue, name=None, unlink_now=False):
|
||||
self.count = 0
|
||||
self.ident = 0
|
||||
self.kind = kind
|
||||
self.maxvalue = maxvalue
|
||||
self.name = name
|
||||
self.handle = _sem_open(self.name.encode('ascii'), value)
|
||||
|
||||
def __del__(self):
|
||||
try:
|
||||
res = pthread.sem_close(self.handle)
|
||||
assert res == 0, "Issue while closing semaphores"
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
def _is_mine(self):
|
||||
return self.count > 0 and get_ident() == self.ident
|
||||
|
||||
def acquire(self, block=True, timeout=None):
|
||||
if self.kind == RECURSIVE_MUTEX and self._is_mine():
|
||||
self.count += 1
|
||||
return True
|
||||
|
||||
if block and timeout is None:
|
||||
res = pthread.sem_wait(self.handle)
|
||||
elif not block or timeout <= 0:
|
||||
res = pthread.sem_trywait(self.handle)
|
||||
else:
|
||||
res = _sem_timedwait(self.handle, timeout)
|
||||
if res < 0:
|
||||
e = ctypes.get_errno()
|
||||
if e == errno.EINTR:
|
||||
return None
|
||||
elif e in [errno.EAGAIN, errno.ETIMEDOUT]:
|
||||
return False
|
||||
raiseFromErrno()
|
||||
self.count += 1
|
||||
self.ident = get_ident()
|
||||
return True
|
||||
|
||||
def release(self):
|
||||
if self.kind == RECURSIVE_MUTEX:
|
||||
assert self._is_mine(), (
|
||||
"attempt to release recursive lock not owned by thread")
|
||||
if self.count > 1:
|
||||
self.count -= 1
|
||||
return
|
||||
assert self.count == 1
|
||||
else:
|
||||
if sys.platform == 'darwin':
|
||||
# Handle broken get_value for mac ==> only Lock will work
|
||||
# as sem_get_value do not work properly
|
||||
if self.maxvalue == 1:
|
||||
if pthread.sem_trywait(self.handle) < 0:
|
||||
e = ctypes.get_errno()
|
||||
if e != errno.EAGAIN:
|
||||
raise OSError(e, errno.errorcode[e])
|
||||
else:
|
||||
if pthread.sem_post(self.handle) < 0:
|
||||
raiseFromErrno()
|
||||
else:
|
||||
raise ValueError(
|
||||
"semaphore or lock released too many times")
|
||||
else:
|
||||
import warnings
|
||||
warnings.warn("semaphore are broken on OSX, release might "
|
||||
"increase its maximal value", RuntimeWarning)
|
||||
else:
|
||||
value = self._get_value()
|
||||
if value >= self.maxvalue:
|
||||
raise ValueError(
|
||||
"semaphore or lock released too many times")
|
||||
|
||||
if pthread.sem_post(self.handle) < 0:
|
||||
raiseFromErrno()
|
||||
|
||||
self.count -= 1
|
||||
|
||||
def _get_value(self):
|
||||
value = ctypes.pointer(ctypes.c_int(-1))
|
||||
if pthread.sem_getvalue(self.handle, value) < 0:
|
||||
raiseFromErrno()
|
||||
return value.contents.value
|
||||
|
||||
def _count(self):
|
||||
return self.count
|
||||
|
||||
def _is_zero(self):
|
||||
if sys.platform == 'darwin':
|
||||
# Handle broken get_value for mac ==> only Lock will work
|
||||
# as sem_get_value do not work properly
|
||||
if pthread.sem_trywait(self.handle) < 0:
|
||||
e = ctypes.get_errno()
|
||||
if e == errno.EAGAIN:
|
||||
return True
|
||||
raise OSError(e, errno.errorcode[e])
|
||||
else:
|
||||
if pthread.sem_post(self.handle) < 0:
|
||||
raiseFromErrno()
|
||||
return False
|
||||
else:
|
||||
value = ctypes.pointer(ctypes.c_int(-1))
|
||||
if pthread.sem_getvalue(self.handle, value) < 0:
|
||||
raiseFromErrno()
|
||||
return value.contents.value == 0
|
||||
|
||||
def _after_fork(self):
|
||||
self.count = 0
|
||||
|
||||
@staticmethod
|
||||
def _rebuild(handle, kind, maxvalue, name):
|
||||
self = SemLock.__new__(SemLock)
|
||||
self.count = 0
|
||||
self.ident = 0
|
||||
self.kind = kind
|
||||
self.maxvalue = maxvalue
|
||||
self.name = name
|
||||
self.handle = _sem_open(name.encode('ascii'))
|
||||
return self
|
||||
|
||||
|
||||
def raiseFromErrno():
|
||||
e = ctypes.get_errno()
|
||||
raise OSError(e, errno.errorcode[e])
|
||||
Loading…
Add table
Add a link
Reference in a new issue