275 lines
8.7 KiB
Python
275 lines
8.7 KiB
Python
|
###############################################################################
|
||
|
|
# Ctypes implementation for posix semaphore.
|
||
|
|
#
|
||
|
|
# author: Thomas Moreau and Olivier Grisel
|
||
|
|
#
|
||
|
|
# adapted from cpython/Modules/_multiprocessing/semaphore.c (17/02/2017)
|
||
|
|
# * use ctypes to access pthread semaphores and provide a full python
|
||
|
|
# semaphore management.
|
||
|
|
# * For OSX, as no sem_getvalue is not implemented, Semaphore with value > 1
|
||
|
|
# are not guaranteed to work.
|
||
|
|
# * Only work with LokyProcess on posix
|
||
|
|
#
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
import errno
|
||
|
|
import ctypes
|
||
|
|
import tempfile
|
||
|
|
import threading
|
||
|
|
from ctypes.util import find_library
|
||
|
|
|
||
|
|
# As we need to use ctypes return types for semlock object, failure value
|
||
|
|
# needs to be cast to proper python value. Unix failure convention is to
|
||
|
|
# return 0, whereas OSX returns -1
|
||
|
|
SEM_FAILURE = ctypes.c_void_p(0).value
|
||
|
|
if sys.platform == 'darwin':
|
||
|
|
SEM_FAILURE = ctypes.c_void_p(-1).value
|
||
|
|
|
||
|
|
# Semaphore types
|
||
|
|
RECURSIVE_MUTEX = 0
|
||
|
|
SEMAPHORE = 1
|
||
|
|
|
||
|
|
# Semaphore constants
|
||
|
|
SEM_OFLAG = ctypes.c_int(os.O_CREAT | os.O_EXCL)
|
||
|
|
SEM_PERM = ctypes.c_int(384)
|
||
|
|
|
||
|
|
|
||
|
|
class timespec(ctypes.Structure):
|
||
|
|
_fields_ = [("tv_sec", ctypes.c_long), ("tv_nsec", ctypes.c_long)]
|
||
|
|
|
||
|
|
|
||
|
|
if sys.platform != 'win32':
|
||
|
|
pthread = ctypes.CDLL(find_library('pthread'), use_errno=True)
|
||
|
|
pthread.sem_open.restype = ctypes.c_void_p
|
||
|
|
pthread.sem_close.argtypes = [ctypes.c_void_p]
|
||
|
|
pthread.sem_wait.argtypes = [ctypes.c_void_p]
|
||
|
|
pthread.sem_trywait.argtypes = [ctypes.c_void_p]
|
||
|
|
pthread.sem_post.argtypes = [ctypes.c_void_p]
|
||
|
|
pthread.sem_getvalue.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
|
||
|
|
pthread.sem_unlink.argtypes = [ctypes.c_char_p]
|
||
|
|
if sys.platform != "darwin":
|
||
|
|
pthread.sem_timedwait.argtypes = [ctypes.c_void_p,
|
||
|
|
ctypes.POINTER(timespec)]
|
||
|
|
|
||
|
|
try:
|
||
|
|
from threading import get_ident
|
||
|
|
except ImportError:
|
||
|
|
def get_ident():
|
||
|
|
return threading.current_thread().ident
|
||
|
|
|
||
|
|
|
||
|
|
if sys.version_info[:2] < (3, 3):
|
||
|
|
class FileExistsError(OSError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
class FileNotFoundError(OSError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def sem_unlink(name):
|
||
|
|
if pthread.sem_unlink(name.encode('ascii')) < 0:
|
||
|
|
raiseFromErrno()
|
||
|
|
|
||
|
|
|
||
|
|
def _sem_open(name, value=None):
|
||
|
|
""" Construct or retrieve a semaphore with the given name
|
||
|
|
|
||
|
|
If value is None, try to retrieve an existing named semaphore.
|
||
|
|
Else create a new semaphore with the given value
|
||
|
|
"""
|
||
|
|
if value is None:
|
||
|
|
handle = pthread.sem_open(ctypes.c_char_p(name), 0)
|
||
|
|
else:
|
||
|
|
handle = pthread.sem_open(ctypes.c_char_p(name), SEM_OFLAG, SEM_PERM,
|
||
|
|
ctypes.c_int(value))
|
||
|
|
|
||
|
|
if handle == SEM_FAILURE:
|
||
|
|
e = ctypes.get_errno()
|
||
|
|
if e == errno.EEXIST:
|
||
|
|
raise FileExistsError("a semaphore named %s already exists" % name)
|
||
|
|
elif e == errno.ENOENT:
|
||
|
|
raise FileNotFoundError('cannot find semaphore named %s' % name)
|
||
|
|
elif e == errno.ENOSYS:
|
||
|
|
raise NotImplementedError('No semaphore implementation on this '
|
||
|
|
'system')
|
||
|
|
else:
|
||
|
|
raiseFromErrno()
|
||
|
|
|
||
|
|
return handle
|
||
|
|
|
||
|
|
|
||
|
|
def _sem_timedwait(handle, timeout):
|
||
|
|
t_start = time.time()
|
||
|
|
if sys.platform != "darwin":
|
||
|
|
sec = int(timeout)
|
||
|
|
tv_sec = int(t_start)
|
||
|
|
nsec = int(1e9 * (timeout - sec) + .5)
|
||
|
|
tv_nsec = int(1e9 * (t_start - tv_sec) + .5)
|
||
|
|
deadline = timespec(sec+tv_sec, nsec+tv_nsec)
|
||
|
|
deadline.tv_sec += int(deadline.tv_nsec / 1000000000)
|
||
|
|
deadline.tv_nsec %= 1000000000
|
||
|
|
return pthread.sem_timedwait(handle, ctypes.pointer(deadline))
|
||
|
|
|
||
|
|
# PERFORMANCE WARNING
|
||
|
|
# No sem_timedwait on OSX so we implement our own method. This method can
|
||
|
|
# degrade performances has the wait can have a latency up to 20 msecs
|
||
|
|
deadline = t_start + timeout
|
||
|
|
delay = 0
|
||
|
|
now = time.time()
|
||
|
|
while True:
|
||
|
|
# Poll the sem file
|
||
|
|
res = pthread.sem_trywait(handle)
|
||
|
|
if res == 0:
|
||
|
|
return 0
|
||
|
|
else:
|
||
|
|
e = ctypes.get_errno()
|
||
|
|
if e != errno.EAGAIN:
|
||
|
|
raiseFromErrno()
|
||
|
|
|
||
|
|
# check for timeout
|
||
|
|
now = time.time()
|
||
|
|
if now > deadline:
|
||
|
|
ctypes.set_errno(errno.ETIMEDOUT)
|
||
|
|
return -1
|
||
|
|
|
||
|
|
# calculate how much time left and check the delay is not too long
|
||
|
|
# -- maximum is 20 msecs
|
||
|
|
difference = (deadline - now)
|
||
|
|
delay = min(delay, 20e-3, difference)
|
||
|
|
|
||
|
|
# Sleep and increase delay
|
||
|
|
time.sleep(delay)
|
||
|
|
delay += 1e-3
|
||
|
|
|
||
|
|
|
||
|
|
class SemLock(object):
|
||
|
|
"""ctypes wrapper to the unix semaphore"""
|
||
|
|
|
||
|
|
_rand = tempfile._RandomNameSequence()
|
||
|
|
|
||
|
|
def __init__(self, kind, value, maxvalue, name=None, unlink_now=False):
|
||
|
|
self.count = 0
|
||
|
|
self.ident = 0
|
||
|
|
self.kind = kind
|
||
|
|
self.maxvalue = maxvalue
|
||
|
|
self.name = name
|
||
|
|
self.handle = _sem_open(self.name.encode('ascii'), value)
|
||
|
|
|
||
|
|
def __del__(self):
|
||
|
|
try:
|
||
|
|
res = pthread.sem_close(self.handle)
|
||
|
|
assert res == 0, "Issue while closing semaphores"
|
||
|
|
except AttributeError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
def _is_mine(self):
|
||
|
|
return self.count > 0 and get_ident() == self.ident
|
||
|
|
|
||
|
|
def acquire(self, block=True, timeout=None):
|
||
|
|
if self.kind == RECURSIVE_MUTEX and self._is_mine():
|
||
|
|
self.count += 1
|
||
|
|
return True
|
||
|
|
|
||
|
|
if block and timeout is None:
|
||
|
|
res = pthread.sem_wait(self.handle)
|
||
|
|
elif not block or timeout <= 0:
|
||
|
|
res = pthread.sem_trywait(self.handle)
|
||
|
|
else:
|
||
|
|
res = _sem_timedwait(self.handle, timeout)
|
||
|
|
if res < 0:
|
||
|
|
e = ctypes.get_errno()
|
||
|
|
if e == errno.EINTR:
|
||
|
|
return None
|
||
|
|
elif e in [errno.EAGAIN, errno.ETIMEDOUT]:
|
||
|
|
return False
|
||
|
|
raiseFromErrno()
|
||
|
|
self.count += 1
|
||
|
|
self.ident = get_ident()
|
||
|
|
return True
|
||
|
|
|
||
|
|
def release(self):
|
||
|
|
if self.kind == RECURSIVE_MUTEX:
|
||
|
|
assert self._is_mine(), (
|
||
|
|
"attempt to release recursive lock not owned by thread")
|
||
|
|
if self.count > 1:
|
||
|
|
self.count -= 1
|
||
|
|
return
|
||
|
|
assert self.count == 1
|
||
|
|
else:
|
||
|
|
if sys.platform == 'darwin':
|
||
|
|
# Handle broken get_value for mac ==> only Lock will work
|
||
|
|
# as sem_get_value do not work properly
|
||
|
|
if self.maxvalue == 1:
|
||
|
|
if pthread.sem_trywait(self.handle) < 0:
|
||
|
|
e = ctypes.get_errno()
|
||
|
|
if e != errno.EAGAIN:
|
||
|
|
raise OSError(e, errno.errorcode[e])
|
||
|
|
else:
|
||
|
|
if pthread.sem_post(self.handle) < 0:
|
||
|
|
raiseFromErrno()
|
||
|
|
else:
|
||
|
|
raise ValueError(
|
||
|
|
"semaphore or lock released too many times")
|
||
|
|
else:
|
||
|
|
import warnings
|
||
|
|
warnings.warn("semaphore are broken on OSX, release might "
|
||
|
|
"increase its maximal value", RuntimeWarning)
|
||
|
|
else:
|
||
|
|
value = self._get_value()
|
||
|
|
if value >= self.maxvalue:
|
||
|
|
raise ValueError(
|
||
|
|
"semaphore or lock released too many times")
|
||
|
|
|
||
|
|
if pthread.sem_post(self.handle) < 0:
|
||
|
|
raiseFromErrno()
|
||
|
|
|
||
|
|
self.count -= 1
|
||
|
|
|
||
|
|
def _get_value(self):
|
||
|
|
value = ctypes.pointer(ctypes.c_int(-1))
|
||
|
|
if pthread.sem_getvalue(self.handle, value) < 0:
|
||
|
|
raiseFromErrno()
|
||
|
|
return value.contents.value
|
||
|
|
|
||
|
|
def _count(self):
|
||
|
|
return self.count
|
||
|
|
|
||
|
|
def _is_zero(self):
|
||
|
|
if sys.platform == 'darwin':
|
||
|
|
# Handle broken get_value for mac ==> only Lock will work
|
||
|
|
# as sem_get_value do not work properly
|
||
|
|
if pthread.sem_trywait(self.handle) < 0:
|
||
|
|
e = ctypes.get_errno()
|
||
|
|
if e == errno.EAGAIN:
|
||
|
|
return True
|
||
|
|
raise OSError(e, errno.errorcode[e])
|
||
|
|
else:
|
||
|
|
if pthread.sem_post(self.handle) < 0:
|
||
|
|
raiseFromErrno()
|
||
|
|
return False
|
||
|
|
else:
|
||
|
|
value = ctypes.pointer(ctypes.c_int(-1))
|
||
|
|
if pthread.sem_getvalue(self.handle, value) < 0:
|
||
|
|
raiseFromErrno()
|
||
|
|
return value.contents.value == 0
|
||
|
|
|
||
|
|
def _after_fork(self):
|
||
|
|
self.count = 0
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _rebuild(handle, kind, maxvalue, name):
|
||
|
|
self = SemLock.__new__(SemLock)
|
||
|
|
self.count = 0
|
||
|
|
self.ident = 0
|
||
|
|
self.kind = kind
|
||
|
|
self.maxvalue = maxvalue
|
||
|
|
self.name = name
|
||
|
|
self.handle = _sem_open(name.encode('ascii'))
|
||
|
|
return self
|
||
|
|
|
||
|
|
|
||
|
|
def raiseFromErrno():
|
||
|
|
e = ctypes.get_errno()
|
||
|
|
raise OSError(e, errno.errorcode[e])
|