GNXSOFT.COM
This commit is contained in:
Iliyan Angelov
@@ -0,0 +1,360 @@
|
||||
"""Functional Utilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import random
|
||||
import threading
|
||||
from collections import OrderedDict, UserDict
|
||||
from collections.abc import Iterable, Mapping
|
||||
from itertools import count, repeat
|
||||
from time import sleep, time
|
||||
|
||||
from vine.utils import wraps
|
||||
|
||||
from .encoding import safe_repr as _safe_repr
|
||||
|
||||
__all__ = (
|
||||
'LRUCache', 'memoize', 'lazy', 'maybe_evaluate',
|
||||
'is_list', 'maybe_list', 'dictfilter', 'retry_over_time',
|
||||
)
|
||||
|
||||
KEYWORD_MARK = object()
|
||||
|
||||
|
||||
class ChannelPromise:
|
||||
|
||||
def __init__(self, contract):
|
||||
self.__contract__ = contract
|
||||
|
||||
def __call__(self):
|
||||
try:
|
||||
return self.__value__
|
||||
except AttributeError:
|
||||
value = self.__value__ = self.__contract__()
|
||||
return value
|
||||
|
||||
def __repr__(self):
|
||||
try:
|
||||
return repr(self.__value__)
|
||||
except AttributeError:
|
||||
return f'<promise: 0x{id(self.__contract__):x}>'
|
||||
|
||||
|
||||
class LRUCache(UserDict):
|
||||
"""LRU Cache implementation using a doubly linked list to track access.
|
||||
|
||||
Arguments:
|
||||
---------
|
||||
limit (int): The maximum number of keys to keep in the cache.
|
||||
When a new key is inserted and the limit has been exceeded,
|
||||
the *Least Recently Used* key will be discarded from the
|
||||
cache.
|
||||
"""
|
||||
|
||||
def __init__(self, limit=None):
|
||||
self.limit = limit
|
||||
self.mutex = threading.RLock()
|
||||
self.data = OrderedDict()
|
||||
|
||||
def __getitem__(self, key):
|
||||
with self.mutex:
|
||||
value = self[key] = self.data.pop(key)
|
||||
return value
|
||||
|
||||
def update(self, *args, **kwargs):
|
||||
with self.mutex:
|
||||
data, limit = self.data, self.limit
|
||||
data.update(*args, **kwargs)
|
||||
if limit and len(data) > limit:
|
||||
# pop additional items in case limit exceeded
|
||||
for _ in range(len(data) - limit):
|
||||
data.popitem(last=False)
|
||||
|
||||
def popitem(self, last=True):
|
||||
with self.mutex:
|
||||
return self.data.popitem(last)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
# remove least recently used key.
|
||||
with self.mutex:
|
||||
if self.limit and len(self.data) >= self.limit:
|
||||
self.data.pop(next(iter(self.data)))
|
||||
self.data[key] = value
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.data)
|
||||
|
||||
def _iterate_items(self):
|
||||
with self.mutex:
|
||||
for k in self:
|
||||
try:
|
||||
yield (k, self.data[k])
|
||||
except KeyError: # pragma: no cover
|
||||
pass
|
||||
iteritems = _iterate_items
|
||||
|
||||
def _iterate_values(self):
|
||||
with self.mutex:
|
||||
for k in self:
|
||||
try:
|
||||
yield self.data[k]
|
||||
except KeyError: # pragma: no cover
|
||||
pass
|
||||
|
||||
itervalues = _iterate_values
|
||||
|
||||
def _iterate_keys(self):
|
||||
# userdict.keys in py3k calls __getitem__
|
||||
with self.mutex:
|
||||
return self.data.keys()
|
||||
iterkeys = _iterate_keys
|
||||
|
||||
def incr(self, key, delta=1):
|
||||
with self.mutex:
|
||||
# this acts as memcached does- store as a string, but return a
|
||||
# integer as long as it exists and we can cast it
|
||||
newval = int(self.data.pop(key)) + delta
|
||||
self[key] = str(newval)
|
||||
return newval
|
||||
|
||||
def __getstate__(self):
|
||||
d = dict(vars(self))
|
||||
d.pop('mutex')
|
||||
return d
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.__dict__ = state
|
||||
self.mutex = threading.RLock()
|
||||
|
||||
keys = _iterate_keys
|
||||
values = _iterate_values
|
||||
items = _iterate_items
|
||||
|
||||
|
||||
def memoize(maxsize=None, keyfun=None, Cache=LRUCache):
|
||||
"""Decorator to cache function return value."""
|
||||
def _memoize(fun):
|
||||
mutex = threading.Lock()
|
||||
cache = Cache(limit=maxsize)
|
||||
|
||||
@wraps(fun)
|
||||
def _M(*args, **kwargs):
|
||||
if keyfun:
|
||||
key = keyfun(args, kwargs)
|
||||
else:
|
||||
key = args + (KEYWORD_MARK,) + tuple(sorted(kwargs.items()))
|
||||
try:
|
||||
with mutex:
|
||||
value = cache[key]
|
||||
except KeyError:
|
||||
value = fun(*args, **kwargs)
|
||||
_M.misses += 1
|
||||
with mutex:
|
||||
cache[key] = value
|
||||
else:
|
||||
_M.hits += 1
|
||||
return value
|
||||
|
||||
def clear():
|
||||
"""Clear the cache and reset cache statistics."""
|
||||
cache.clear()
|
||||
_M.hits = _M.misses = 0
|
||||
|
||||
_M.hits = _M.misses = 0
|
||||
_M.clear = clear
|
||||
_M.original_func = fun
|
||||
return _M
|
||||
|
||||
return _memoize
|
||||
|
||||
|
||||
class lazy:
|
||||
"""Holds lazy evaluation.
|
||||
|
||||
Evaluated when called or if the :meth:`evaluate` method is called.
|
||||
The function is re-evaluated on every call.
|
||||
|
||||
Overloaded operations that will evaluate the promise:
|
||||
:meth:`__str__`, :meth:`__repr__`, :meth:`__cmp__`.
|
||||
"""
|
||||
|
||||
def __init__(self, fun, *args, **kwargs):
|
||||
self._fun = fun
|
||||
self._args = args
|
||||
self._kwargs = kwargs
|
||||
|
||||
def __call__(self):
|
||||
return self.evaluate()
|
||||
|
||||
def evaluate(self):
|
||||
return self._fun(*self._args, **self._kwargs)
|
||||
|
||||
def __str__(self):
|
||||
return str(self())
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self())
|
||||
|
||||
def __eq__(self, rhs):
|
||||
return self() == rhs
|
||||
|
||||
def __ne__(self, rhs):
|
||||
return self() != rhs
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
memo[id(self)] = self
|
||||
return self
|
||||
|
||||
def __reduce__(self):
|
||||
return (self.__class__, (self._fun,), {'_args': self._args,
|
||||
'_kwargs': self._kwargs})
|
||||
|
||||
|
||||
def maybe_evaluate(value):
|
||||
"""Evaluate value only if value is a :class:`lazy` instance."""
|
||||
if isinstance(value, lazy):
|
||||
return value.evaluate()
|
||||
return value
|
||||
|
||||
|
||||
def is_list(obj, scalars=(Mapping, str), iters=(Iterable,)):
|
||||
"""Return true if the object is iterable.
|
||||
|
||||
Note:
|
||||
----
|
||||
Returns false if object is a mapping or string.
|
||||
"""
|
||||
return isinstance(obj, iters) and not isinstance(obj, scalars or ())
|
||||
|
||||
|
||||
def maybe_list(obj, scalars=(Mapping, str)):
|
||||
"""Return list of one element if ``l`` is a scalar."""
|
||||
return obj if obj is None or is_list(obj, scalars) else [obj]
|
||||
|
||||
|
||||
def dictfilter(d=None, **kw):
|
||||
"""Remove all keys from dict ``d`` whose value is :const:`None`."""
|
||||
d = kw if d is None else (dict(d, **kw) if kw else d)
|
||||
return {k: v for k, v in d.items() if v is not None}
|
||||
|
||||
|
||||
def shufflecycle(it):
|
||||
it = list(it) # don't modify callers list
|
||||
shuffle = random.shuffle
|
||||
for _ in repeat(None):
|
||||
shuffle(it)
|
||||
yield it[0]
|
||||
|
||||
|
||||
def fxrange(start=1.0, stop=None, step=1.0, repeatlast=False):
|
||||
cur = start * 1.0
|
||||
while 1:
|
||||
if not stop or cur <= stop:
|
||||
yield cur
|
||||
cur += step
|
||||
else:
|
||||
if not repeatlast:
|
||||
break
|
||||
yield cur - step
|
||||
|
||||
|
||||
def fxrangemax(start=1.0, stop=None, step=1.0, max=100.0):
|
||||
sum_, cur = 0, start * 1.0
|
||||
while 1:
|
||||
if sum_ >= max:
|
||||
break
|
||||
yield cur
|
||||
if stop:
|
||||
cur = min(cur + step, stop)
|
||||
else:
|
||||
cur += step
|
||||
sum_ += cur
|
||||
|
||||
|
||||
def retry_over_time(fun, catch, args=None, kwargs=None, errback=None,
|
||||
max_retries=None, interval_start=2, interval_step=2,
|
||||
interval_max=30, callback=None, timeout=None):
|
||||
"""Retry the function over and over until max retries is exceeded.
|
||||
|
||||
For each retry we sleep a for a while before we try again, this interval
|
||||
is increased for every retry until the max seconds is reached.
|
||||
|
||||
Arguments:
|
||||
---------
|
||||
fun (Callable): The function to try
|
||||
catch (Tuple[BaseException]): Exceptions to catch, can be either
|
||||
tuple or a single exception class.
|
||||
|
||||
Keyword Arguments:
|
||||
-----------------
|
||||
args (Tuple): Positional arguments passed on to the function.
|
||||
kwargs (Dict): Keyword arguments passed on to the function.
|
||||
errback (Callable): Callback for when an exception in ``catch``
|
||||
is raised. The callback must take three arguments:
|
||||
``exc``, ``interval_range`` and ``retries``, where ``exc``
|
||||
is the exception instance, ``interval_range`` is an iterator
|
||||
which return the time in seconds to sleep next, and ``retries``
|
||||
is the number of previous retries.
|
||||
max_retries (int): Maximum number of retries before we give up.
|
||||
If neither of this and timeout is set, we will retry forever.
|
||||
If one of this and timeout is reached, stop.
|
||||
interval_start (float): How long (in seconds) we start sleeping
|
||||
between retries.
|
||||
interval_step (float): By how much the interval is increased for
|
||||
each retry.
|
||||
interval_max (float): Maximum number of seconds to sleep
|
||||
between retries.
|
||||
timeout (int): Maximum seconds waiting before we give up.
|
||||
"""
|
||||
kwargs = {} if not kwargs else kwargs
|
||||
args = [] if not args else args
|
||||
interval_range = fxrange(interval_start,
|
||||
interval_max + interval_start,
|
||||
interval_step, repeatlast=True)
|
||||
end = time() + timeout if timeout else None
|
||||
for retries in count():
|
||||
try:
|
||||
return fun(*args, **kwargs)
|
||||
except catch as exc:
|
||||
if max_retries is not None and retries >= max_retries:
|
||||
raise
|
||||
if end and time() > end:
|
||||
raise
|
||||
if callback:
|
||||
callback()
|
||||
tts = float(errback(exc, interval_range, retries) if errback
|
||||
else next(interval_range))
|
||||
if tts:
|
||||
for _ in range(int(tts)):
|
||||
if callback:
|
||||
callback()
|
||||
sleep(1.0)
|
||||
# sleep remainder after int truncation above.
|
||||
sleep(abs(int(tts) - tts))
|
||||
|
||||
|
||||
def reprkwargs(kwargs, sep=', ', fmt='{0}={1}'):
|
||||
return sep.join(fmt.format(k, _safe_repr(v)) for k, v in kwargs.items())
|
||||
|
||||
|
||||
def reprcall(name, args=(), kwargs=None, sep=', '):
|
||||
kwargs = {} if not kwargs else kwargs
|
||||
return '{}({}{}{})'.format(
|
||||
name, sep.join(map(_safe_repr, args or ())),
|
||||
(args and kwargs) and sep or '',
|
||||
reprkwargs(kwargs, sep),
|
||||
)
|
||||
|
||||
|
||||
def accepts_argument(func, argument_name):
|
||||
argument_spec = inspect.getfullargspec(func)
|
||||
return (
|
||||
argument_name in argument_spec.args or
|
||||
argument_name in argument_spec.kwonlyargs
|
||||
)
|
||||
|
||||
|
||||
# Compat names (before kombu 3.0)
|
||||
promise = lazy
|
||||
maybe_promise = maybe_evaluate
|
||||
Reference in New Issue
Block a user