This commit is contained in:
Iliyan Angelov
2025-09-14 23:24:25 +03:00
commit c67067a2a4
71311 changed files with 6800714 additions and 0 deletions

View File

@@ -0,0 +1,75 @@
"""Low-level AMQP client for Python (fork of amqplib)."""
# Copyright (C) 2007-2008 Barry Pederson <bp@barryp.org>
import re
from collections import namedtuple
__version__ = '5.3.1'
__author__ = 'Barry Pederson'
__maintainer__ = 'Asif Saif Uddin, Matus Valo'
__contact__ = 'auvipy@gmail.com'
__homepage__ = 'http://github.com/celery/py-amqp'
__docformat__ = 'restructuredtext'
# -eof meta-
version_info_t = namedtuple('version_info_t', (
'major', 'minor', 'micro', 'releaselevel', 'serial',
))
# bumpversion can only search for {current_version}
# so we have to parse the version here.
_temp = re.match(
r'(\d+)\.(\d+).(\d+)(.+)?', __version__).groups()
VERSION = version_info = version_info_t(
int(_temp[0]), int(_temp[1]), int(_temp[2]), _temp[3] or '', '')
del(_temp)
del(re)
from .basic_message import Message # noqa
from .channel import Channel # noqa
from .connection import Connection # noqa
from .exceptions import (AccessRefused, AMQPError, # noqa
AMQPNotImplementedError, ChannelError, ChannelNotOpen,
ConnectionError, ConnectionForced, ConsumerCancelled,
ContentTooLarge, FrameError, FrameSyntaxError,
InternalError, InvalidCommand, InvalidPath,
IrrecoverableChannelError,
IrrecoverableConnectionError, NoConsumers, NotAllowed,
NotFound, PreconditionFailed, RecoverableChannelError,
RecoverableConnectionError, ResourceError,
ResourceLocked, UnexpectedFrame, error_for_code)
from .utils import promise # noqa
__all__ = (
'Connection',
'Channel',
'Message',
'promise',
'AMQPError',
'ConnectionError',
'RecoverableConnectionError',
'IrrecoverableConnectionError',
'ChannelError',
'RecoverableChannelError',
'IrrecoverableChannelError',
'ConsumerCancelled',
'ContentTooLarge',
'NoConsumers',
'ConnectionForced',
'InvalidPath',
'AccessRefused',
'NotFound',
'ResourceLocked',
'PreconditionFailed',
'FrameError',
'FrameSyntaxError',
'InvalidCommand',
'ChannelNotOpen',
'UnexpectedFrame',
'ResourceError',
'NotAllowed',
'AMQPNotImplementedError',
'InternalError',
'error_for_code',
)

View File

@@ -0,0 +1,163 @@
"""Code common to Connection and Channel objects."""
# Copyright (C) 2007-2008 Barry Pederson <bp@barryp.org>)
import logging
from vine import ensure_promise, promise
from .exceptions import AMQPNotImplementedError, RecoverableConnectionError
from .serialization import dumps, loads
__all__ = ('AbstractChannel',)
AMQP_LOGGER = logging.getLogger('amqp')
IGNORED_METHOD_DURING_CHANNEL_CLOSE = """\
Received method %s during closing channel %s. This method will be ignored\
"""
class AbstractChannel:
"""Superclass for Connection and Channel.
The connection is treated as channel 0, then comes
user-created channel objects.
The subclasses must have a _METHOD_MAP class property, mapping
between AMQP method signatures and Python methods.
"""
def __init__(self, connection, channel_id):
self.is_closing = False
self.connection = connection
self.channel_id = channel_id
connection.channels[channel_id] = self
self.method_queue = [] # Higher level queue for methods
self.auto_decode = False
self._pending = {}
self._callbacks = {}
self._setup_listeners()
__slots__ = (
"is_closing",
"connection",
"channel_id",
"method_queue",
"auto_decode",
"_pending",
"_callbacks",
# adding '__dict__' to get dynamic assignment
"__dict__",
"__weakref__",
)
def __enter__(self):
return self
def __exit__(self, *exc_info):
self.close()
def send_method(self, sig,
format=None, args=None, content=None,
wait=None, callback=None, returns_tuple=False):
p = promise()
conn = self.connection
if conn is None:
raise RecoverableConnectionError('connection already closed')
args = dumps(format, args) if format else ''
try:
conn.frame_writer(1, self.channel_id, sig, args, content)
except StopIteration:
raise RecoverableConnectionError('connection already closed')
# TODO temp: callback should be after write_method ... ;)
if callback:
p.then(callback)
p()
if wait:
return self.wait(wait, returns_tuple=returns_tuple)
return p
def close(self):
"""Close this Channel or Connection."""
raise NotImplementedError('Must be overridden in subclass')
def wait(self, method, callback=None, timeout=None, returns_tuple=False):
p = ensure_promise(callback)
pending = self._pending
prev_p = []
if not isinstance(method, list):
method = [method]
for m in method:
prev_p.append(pending.get(m))
pending[m] = p
try:
while not p.ready:
self.connection.drain_events(timeout=timeout)
if p.value:
args, kwargs = p.value
args = args[1:] # We are not returning method back
return args if returns_tuple else (args and args[0])
finally:
for i, m in enumerate(method):
if prev_p[i] is not None:
pending[m] = prev_p[i]
else:
pending.pop(m, None)
def dispatch_method(self, method_sig, payload, content):
if self.is_closing and method_sig not in (
self._ALLOWED_METHODS_WHEN_CLOSING
):
# When channel.close() was called we must ignore all methods except
# Channel.close and Channel.CloseOk
AMQP_LOGGER.warning(
IGNORED_METHOD_DURING_CHANNEL_CLOSE,
method_sig, self.channel_id
)
return
if content and \
self.auto_decode and \
hasattr(content, 'content_encoding'):
try:
content.body = content.body.decode(content.content_encoding)
except Exception:
pass
try:
amqp_method = self._METHODS[method_sig]
except KeyError:
raise AMQPNotImplementedError(
f'Unknown AMQP method {method_sig!r}')
try:
listeners = [self._callbacks[method_sig]]
except KeyError:
listeners = []
one_shot = None
try:
one_shot = self._pending.pop(method_sig)
except KeyError:
if not listeners:
return
args = []
if amqp_method.args:
args, _ = loads(amqp_method.args, payload, 4)
if amqp_method.content:
args.append(content)
for listener in listeners:
listener(*args)
if one_shot:
one_shot(method_sig, *args)
#: Placeholder, the concrete implementations will have to
#: supply their own versions of _METHOD_MAP
_METHODS = {}

View File

@@ -0,0 +1,122 @@
"""AMQP Messages."""
# Copyright (C) 2007-2008 Barry Pederson <bp@barryp.org>
from .serialization import GenericContent
# Intended to fix #85: ImportError: cannot import name spec
# Encountered on python 2.7.3
# "The submodules often need to refer to each other. For example, the
# surround [sic] module might use the echo module. In fact, such
# references are so common that the import statement first looks in
# the containing package before looking in the standard module search
# path."
# Source:
# http://stackoverflow.com/a/14216937/4982251
from .spec import Basic
__all__ = ('Message',)
class Message(GenericContent):
"""A Message for use with the Channel.basic_* methods.
Expected arg types
body: string
children: (not supported)
Keyword properties may include:
content_type: shortstr
MIME content type
content_encoding: shortstr
MIME content encoding
application_headers: table
Message header field table, a dict with string keys,
and string | int | Decimal | datetime | dict values.
delivery_mode: octet
Non-persistent (1) or persistent (2)
priority: octet
The message priority, 0 to 9
correlation_id: shortstr
The application correlation identifier
reply_to: shortstr
The destination to reply to
expiration: shortstr
Message expiration specification
message_id: shortstr
The application message identifier
timestamp: unsigned long
The message timestamp
type: shortstr
The message type name
user_id: shortstr
The creating user id
app_id: shortstr
The creating application id
cluster_id: shortstr
Intra-cluster routing identifier
Unicode bodies are encoded according to the 'content_encoding'
argument. If that's None, it's set to 'UTF-8' automatically.
Example::
msg = Message('hello world',
content_type='text/plain',
application_headers={'foo': 7})
"""
CLASS_ID = Basic.CLASS_ID
#: Instances of this class have these attributes, which
#: are passed back and forth as message properties between
#: client and server
PROPERTIES = [
('content_type', 's'),
('content_encoding', 's'),
('application_headers', 'F'),
('delivery_mode', 'o'),
('priority', 'o'),
('correlation_id', 's'),
('reply_to', 's'),
('expiration', 's'),
('message_id', 's'),
('timestamp', 'L'),
('type', 's'),
('user_id', 's'),
('app_id', 's'),
('cluster_id', 's')
]
def __init__(self, body='', children=None, channel=None, **properties):
super().__init__(**properties)
#: set by basic_consume/basic_get
self.delivery_info = None
self.body = body
self.channel = channel
__slots__ = (
"delivery_info",
"body",
"channel",
)
@property
def headers(self):
return self.properties.get('application_headers')
@property
def delivery_tag(self):
return self.delivery_info.get('delivery_tag')

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,784 @@
"""AMQP Connections."""
# Copyright (C) 2007-2008 Barry Pederson <bp@barryp.org>
import logging
import socket
import uuid
import warnings
from array import array
from time import monotonic
from vine import ensure_promise
from . import __version__, sasl, spec
from .abstract_channel import AbstractChannel
from .channel import Channel
from .exceptions import (AMQPDeprecationWarning, ChannelError, ConnectionError,
ConnectionForced, MessageNacked, RecoverableChannelError,
RecoverableConnectionError, ResourceError,
error_for_code)
from .method_framing import frame_handler, frame_writer
from .transport import Transport
try:
from ssl import SSLError
except ImportError: # pragma: no cover
class SSLError(Exception): # noqa
pass
W_FORCE_CONNECT = """\
The .{attr} attribute on the connection was accessed before
the connection was established. This is supported for now, but will
be deprecated in amqp 2.2.0.
Since amqp 2.0 you have to explicitly call Connection.connect()
before using the connection.
"""
START_DEBUG_FMT = """
Start from server, version: %d.%d, properties: %s, mechanisms: %s, locales: %s
""".strip()
__all__ = ('Connection',)
AMQP_LOGGER = logging.getLogger('amqp')
AMQP_HEARTBEAT_LOGGER = logging.getLogger(
'amqp.connection.Connection.heartbeat_tick'
)
#: Default map for :attr:`Connection.library_properties`
LIBRARY_PROPERTIES = {
'product': 'py-amqp',
'product_version': __version__,
}
#: Default map for :attr:`Connection.negotiate_capabilities`
NEGOTIATE_CAPABILITIES = {
'consumer_cancel_notify': True,
'connection.blocked': True,
'authentication_failure_close': True,
}
class Connection(AbstractChannel):
"""AMQP Connection.
The connection class provides methods for a client to establish a
network connection to a server, and for both peers to operate the
connection thereafter.
GRAMMAR::
connection = open-connection *use-connection close-connection
open-connection = C:protocol-header
S:START C:START-OK
*challenge
S:TUNE C:TUNE-OK
C:OPEN S:OPEN-OK
challenge = S:SECURE C:SECURE-OK
use-connection = *channel
close-connection = C:CLOSE S:CLOSE-OK
/ S:CLOSE C:CLOSE-OK
Create a connection to the specified host, which should be
a 'host[:port]', such as 'localhost', or '1.2.3.4:5672'
(defaults to 'localhost', if a port is not specified then
5672 is used)
Authentication can be controlled by passing one or more
`amqp.sasl.SASL` instances as the `authentication` parameter, or
setting the `login_method` string to one of the supported methods:
'GSSAPI', 'EXTERNAL', 'AMQPLAIN', or 'PLAIN'.
Otherwise authentication will be performed using any supported method
preferred by the server. Userid and passwords apply to AMQPLAIN and
PLAIN authentication, whereas on GSSAPI only userid will be used as the
client name. For EXTERNAL authentication both userid and password are
ignored.
The 'ssl' parameter may be simply True/False, or
a dictionary of options to pass to :class:`ssl.SSLContext` such as
requiring certain certificates. For details, refer ``ssl`` parameter of
:class:`~amqp.transport.SSLTransport`.
The "socket_settings" parameter is a dictionary defining tcp
settings which will be applied as socket options.
When "confirm_publish" is set to True, the channel is put to
confirm mode. In this mode, each published message is
confirmed using Publisher confirms RabbitMQ extension.
"""
Channel = Channel
#: Mapping of protocol extensions to enable.
#: The server will report these in server_properties[capabilities],
#: and if a key in this map is present the client will tell the
#: server to either enable or disable the capability depending
#: on the value set in this map.
#: For example with:
#: negotiate_capabilities = {
#: 'consumer_cancel_notify': True,
#: }
#: The client will enable this capability if the server reports
#: support for it, but if the value is False the client will
#: disable the capability.
negotiate_capabilities = NEGOTIATE_CAPABILITIES
#: These are sent to the server to announce what features
#: we support, type of client etc.
library_properties = LIBRARY_PROPERTIES
#: Final heartbeat interval value (in float seconds) after negotiation
heartbeat = None
#: Original heartbeat interval value proposed by client.
client_heartbeat = None
#: Original heartbeat interval proposed by server.
server_heartbeat = None
#: Time of last heartbeat sent (in monotonic time, if available).
last_heartbeat_sent = 0
#: Time of last heartbeat received (in monotonic time, if available).
last_heartbeat_received = 0
#: Number of successful writes to socket.
bytes_sent = 0
#: Number of successful reads from socket.
bytes_recv = 0
#: Number of bytes sent to socket at the last heartbeat check.
prev_sent = None
#: Number of bytes received from socket at the last heartbeat check.
prev_recv = None
_METHODS = {
spec.method(spec.Connection.Start, 'ooFSS'),
spec.method(spec.Connection.OpenOk),
spec.method(spec.Connection.Secure, 's'),
spec.method(spec.Connection.Tune, 'BlB'),
spec.method(spec.Connection.Close, 'BsBB'),
spec.method(spec.Connection.Blocked),
spec.method(spec.Connection.Unblocked),
spec.method(spec.Connection.CloseOk),
}
_METHODS = {m.method_sig: m for m in _METHODS}
_ALLOWED_METHODS_WHEN_CLOSING = (
spec.Connection.Close, spec.Connection.CloseOk
)
connection_errors = (
ConnectionError,
socket.error,
IOError,
OSError,
)
channel_errors = (ChannelError,)
recoverable_connection_errors = (
RecoverableConnectionError,
MessageNacked,
socket.error,
IOError,
OSError,
)
recoverable_channel_errors = (
RecoverableChannelError,
)
def __init__(self, host='localhost:5672', userid='guest', password='guest',
login_method=None, login_response=None,
authentication=(),
virtual_host='/', locale='en_US', client_properties=None,
ssl=False, connect_timeout=None, channel_max=None,
frame_max=None, heartbeat=0, on_open=None, on_blocked=None,
on_unblocked=None, confirm_publish=False,
on_tune_ok=None, read_timeout=None, write_timeout=None,
socket_settings=None, frame_handler=frame_handler,
frame_writer=frame_writer, **kwargs):
self._connection_id = uuid.uuid4().hex
channel_max = channel_max or 65535
frame_max = frame_max or 131072
if authentication:
if isinstance(authentication, sasl.SASL):
authentication = (authentication,)
self.authentication = authentication
elif login_method is not None:
if login_method == 'GSSAPI':
auth = sasl.GSSAPI(userid)
elif login_method == 'EXTERNAL':
auth = sasl.EXTERNAL()
elif login_method == 'AMQPLAIN':
if userid is None or password is None:
raise ValueError(
"Must supply authentication or userid/password")
auth = sasl.AMQPLAIN(userid, password)
elif login_method == 'PLAIN':
if userid is None or password is None:
raise ValueError(
"Must supply authentication or userid/password")
auth = sasl.PLAIN(userid, password)
elif login_response is not None:
auth = sasl.RAW(login_method, login_response)
else:
raise ValueError("Invalid login method", login_method)
self.authentication = (auth,)
else:
self.authentication = (sasl.GSSAPI(userid, fail_soft=True),
sasl.EXTERNAL(),
sasl.AMQPLAIN(userid, password),
sasl.PLAIN(userid, password))
self.client_properties = dict(
self.library_properties, **client_properties or {}
)
self.locale = locale
self.host = host
self.virtual_host = virtual_host
self.on_tune_ok = ensure_promise(on_tune_ok)
self.frame_handler_cls = frame_handler
self.frame_writer_cls = frame_writer
self._handshake_complete = False
self.channels = {}
# The connection object itself is treated as channel 0
super().__init__(self, 0)
self._frame_writer = None
self._on_inbound_frame = None
self._transport = None
# Properties set in the Tune method
self.channel_max = channel_max
self.frame_max = frame_max
self.client_heartbeat = heartbeat
self.confirm_publish = confirm_publish
self.ssl = ssl
self.read_timeout = read_timeout
self.write_timeout = write_timeout
self.socket_settings = socket_settings
# Callbacks
self.on_blocked = on_blocked
self.on_unblocked = on_unblocked
self.on_open = ensure_promise(on_open)
self._used_channel_ids = array('H')
# Properties set in the Start method
self.version_major = 0
self.version_minor = 0
self.server_properties = {}
self.mechanisms = []
self.locales = []
self.connect_timeout = connect_timeout
def __repr__(self):
if self._transport:
return f'<AMQP Connection: {self.host}/{self.virtual_host} '\
f'using {self._transport} at {id(self):#x}>'
else:
return f'<AMQP Connection: {self.host}/{self.virtual_host} '\
f'(disconnected) at {id(self):#x}>'
def __enter__(self):
self.connect()
return self
def __exit__(self, *eargs):
self.close()
def then(self, on_success, on_error=None):
return self.on_open.then(on_success, on_error)
def _setup_listeners(self):
self._callbacks.update({
spec.Connection.Start: self._on_start,
spec.Connection.OpenOk: self._on_open_ok,
spec.Connection.Secure: self._on_secure,
spec.Connection.Tune: self._on_tune,
spec.Connection.Close: self._on_close,
spec.Connection.Blocked: self._on_blocked,
spec.Connection.Unblocked: self._on_unblocked,
spec.Connection.CloseOk: self._on_close_ok,
})
def connect(self, callback=None):
# Let the transport.py module setup the actual
# socket connection to the broker.
#
if self.connected:
return callback() if callback else None
try:
self.transport = self.Transport(
self.host, self.connect_timeout, self.ssl,
self.read_timeout, self.write_timeout,
socket_settings=self.socket_settings,
)
self.transport.connect()
self.on_inbound_frame = self.frame_handler_cls(
self, self.on_inbound_method)
self.frame_writer = self.frame_writer_cls(self, self.transport)
while not self._handshake_complete:
self.drain_events(timeout=self.connect_timeout)
except (OSError, SSLError):
self.collect()
raise
def _warn_force_connect(self, attr):
warnings.warn(AMQPDeprecationWarning(
W_FORCE_CONNECT.format(attr=attr)))
@property
def transport(self):
if self._transport is None:
self._warn_force_connect('transport')
self.connect()
return self._transport
@transport.setter
def transport(self, transport):
self._transport = transport
@property
def on_inbound_frame(self):
if self._on_inbound_frame is None:
self._warn_force_connect('on_inbound_frame')
self.connect()
return self._on_inbound_frame
@on_inbound_frame.setter
def on_inbound_frame(self, on_inbound_frame):
self._on_inbound_frame = on_inbound_frame
@property
def frame_writer(self):
if self._frame_writer is None:
self._warn_force_connect('frame_writer')
self.connect()
return self._frame_writer
@frame_writer.setter
def frame_writer(self, frame_writer):
self._frame_writer = frame_writer
def _on_start(self, version_major, version_minor, server_properties,
mechanisms, locales, argsig='FsSs'):
client_properties = self.client_properties
self.version_major = version_major
self.version_minor = version_minor
self.server_properties = server_properties
if isinstance(mechanisms, str):
mechanisms = mechanisms.encode('utf-8')
self.mechanisms = mechanisms.split(b' ')
self.locales = locales.split(' ')
AMQP_LOGGER.debug(
START_DEBUG_FMT,
self.version_major, self.version_minor,
self.server_properties, self.mechanisms, self.locales,
)
# Negotiate protocol extensions (capabilities)
scap = server_properties.get('capabilities') or {}
cap = client_properties.setdefault('capabilities', {})
cap.update({
wanted_cap: enable_cap
for wanted_cap, enable_cap in self.negotiate_capabilities.items()
if scap.get(wanted_cap)
})
if not cap:
# no capabilities, server may not react well to having
# this key present in client_properties, so we remove it.
client_properties.pop('capabilities', None)
for authentication in self.authentication:
if authentication.mechanism in self.mechanisms:
login_response = authentication.start(self)
if login_response is not NotImplemented:
break
else:
raise ConnectionError(
"Couldn't find appropriate auth mechanism "
"(can offer: {}; available: {})".format(
b", ".join(m.mechanism
for m in self.authentication
if m.mechanism).decode(),
b", ".join(self.mechanisms).decode()))
self.send_method(
spec.Connection.StartOk, argsig,
(client_properties, authentication.mechanism,
login_response, self.locale),
)
def _on_secure(self, challenge):
pass
def _on_tune(self, channel_max, frame_max, server_heartbeat, argsig='BlB'):
client_heartbeat = self.client_heartbeat or 0
self.channel_max = channel_max or self.channel_max
self.frame_max = frame_max or self.frame_max
self.server_heartbeat = server_heartbeat or 0
# negotiate the heartbeat interval to the smaller of the
# specified values
if self.server_heartbeat == 0 or client_heartbeat == 0:
self.heartbeat = max(self.server_heartbeat, client_heartbeat)
else:
self.heartbeat = min(self.server_heartbeat, client_heartbeat)
# Ignore server heartbeat if client_heartbeat is disabled
if not self.client_heartbeat:
self.heartbeat = 0
self.send_method(
spec.Connection.TuneOk, argsig,
(self.channel_max, self.frame_max, self.heartbeat),
callback=self._on_tune_sent,
)
def _on_tune_sent(self, argsig='ssb'):
self.send_method(
spec.Connection.Open, argsig, (self.virtual_host, '', False),
)
def _on_open_ok(self):
self._handshake_complete = True
self.on_open(self)
def Transport(self, host, connect_timeout,
ssl=False, read_timeout=None, write_timeout=None,
socket_settings=None, **kwargs):
return Transport(
host, connect_timeout=connect_timeout, ssl=ssl,
read_timeout=read_timeout, write_timeout=write_timeout,
socket_settings=socket_settings, **kwargs)
@property
def connected(self):
return self._transport and self._transport.connected
def collect(self):
if self._transport:
self._transport.close()
if self.channels:
# Copy all the channels except self since the channels
# dictionary changes during the collection process.
channels = [
ch for ch in self.channels.values()
if ch is not self
]
for ch in channels:
ch.collect()
self._transport = self.connection = self.channels = None
def _get_free_channel_id(self):
# Cast to a set for fast lookups, and keep stored as an array for lower memory usage.
used_channel_ids = set(self._used_channel_ids)
for channel_id in range(1, self.channel_max + 1):
if channel_id not in used_channel_ids:
self._used_channel_ids.append(channel_id)
return channel_id
raise ResourceError(
'No free channel ids, current={}, channel_max={}'.format(
len(self.channels), self.channel_max), spec.Channel.Open)
def _claim_channel_id(self, channel_id):
if channel_id in self._used_channel_ids:
raise ConnectionError(f'Channel {channel_id!r} already open')
else:
self._used_channel_ids.append(channel_id)
return channel_id
def channel(self, channel_id=None, callback=None):
"""Create new channel.
Fetch a Channel object identified by the numeric channel_id, or
create that object if it doesn't already exist.
"""
if self.channels is None:
raise RecoverableConnectionError('Connection already closed.')
try:
return self.channels[channel_id]
except KeyError:
channel = self.Channel(self, channel_id, on_open=callback)
channel.open()
return channel
def is_alive(self):
raise NotImplementedError('Use AMQP heartbeats')
def drain_events(self, timeout=None):
# read until message is ready
while not self.blocking_read(timeout):
pass
def blocking_read(self, timeout=None):
with self.transport.having_timeout(timeout):
frame = self.transport.read_frame()
return self.on_inbound_frame(frame)
def on_inbound_method(self, channel_id, method_sig, payload, content):
if self.channels is None:
raise RecoverableConnectionError('Connection already closed')
return self.channels[channel_id].dispatch_method(
method_sig, payload, content,
)
def close(self, reply_code=0, reply_text='', method_sig=(0, 0),
argsig='BsBB'):
"""Request a connection close.
This method indicates that the sender wants to close the
connection. This may be due to internal conditions (e.g. a
forced shut-down) or due to an error handling a specific
method, i.e. an exception. When a close is due to an
exception, the sender provides the class and method id of the
method which caused the exception.
RULE:
After sending this method any received method except the
Close-OK method MUST be discarded.
RULE:
The peer sending this method MAY use a counter or timeout
to detect failure of the other peer to respond correctly
with the Close-OK method.
RULE:
When a server receives the Close method from a client it
MUST delete all server-side resources associated with the
client's context. A client CANNOT reconnect to a context
after sending or receiving a Close method.
PARAMETERS:
reply_code: short
The reply code. The AMQ reply codes are defined in AMQ
RFC 011.
reply_text: shortstr
The localised reply text. This text can be logged as an
aid to resolving issues.
class_id: short
failing method class
When the close is provoked by a method exception, this
is the class of the method.
method_id: short
failing method ID
When the close is provoked by a method exception, this
is the ID of the method.
"""
if self._transport is None:
# already closed
return
try:
self.is_closing = True
return self.send_method(
spec.Connection.Close, argsig,
(reply_code, reply_text, method_sig[0], method_sig[1]),
wait=spec.Connection.CloseOk,
)
except (OSError, SSLError):
# close connection
self.collect()
raise
finally:
self.is_closing = False
def _on_close(self, reply_code, reply_text, class_id, method_id):
"""Request a connection close.
This method indicates that the sender wants to close the
connection. This may be due to internal conditions (e.g. a
forced shut-down) or due to an error handling a specific
method, i.e. an exception. When a close is due to an
exception, the sender provides the class and method id of the
method which caused the exception.
RULE:
After sending this method any received method except the
Close-OK method MUST be discarded.
RULE:
The peer sending this method MAY use a counter or timeout
to detect failure of the other peer to respond correctly
with the Close-OK method.
RULE:
When a server receives the Close method from a client it
MUST delete all server-side resources associated with the
client's context. A client CANNOT reconnect to a context
after sending or receiving a Close method.
PARAMETERS:
reply_code: short
The reply code. The AMQ reply codes are defined in AMQ
RFC 011.
reply_text: shortstr
The localised reply text. This text can be logged as an
aid to resolving issues.
class_id: short
failing method class
When the close is provoked by a method exception, this
is the class of the method.
method_id: short
failing method ID
When the close is provoked by a method exception, this
is the ID of the method.
"""
self._x_close_ok()
raise error_for_code(reply_code, reply_text,
(class_id, method_id), ConnectionError)
def _x_close_ok(self):
"""Confirm a connection close.
This method confirms a Connection.Close method and tells the
recipient that it is safe to release resources for the
connection and close the socket.
RULE:
A peer that detects a socket closure without having
received a Close-Ok handshake method SHOULD log the error.
"""
self.send_method(spec.Connection.CloseOk, callback=self._on_close_ok)
def _on_close_ok(self):
"""Confirm a connection close.
This method confirms a Connection.Close method and tells the
recipient that it is safe to release resources for the
connection and close the socket.
RULE:
A peer that detects a socket closure without having
received a Close-Ok handshake method SHOULD log the error.
"""
self.collect()
def _on_blocked(self):
"""Callback called when connection blocked.
Notes:
This is an RabbitMQ Extension.
"""
reason = 'connection blocked, see broker logs'
if self.on_blocked:
return self.on_blocked(reason)
def _on_unblocked(self):
if self.on_unblocked:
return self.on_unblocked()
def send_heartbeat(self):
self.frame_writer(8, 0, None, None, None)
def heartbeat_tick(self, rate=2):
"""Send heartbeat packets if necessary.
Raises:
~amqp.exceptions.ConnectionForvced: if none have been
received recently.
Note:
This should be called frequently, on the order of
once per second.
Keyword Arguments:
rate (int): Number of heartbeat frames to send during the heartbeat
timeout
"""
AMQP_HEARTBEAT_LOGGER.debug('heartbeat_tick : for connection %s',
self._connection_id)
if not self.heartbeat:
return
# If rate is wrong, let's use 2 as default
if rate <= 0:
rate = 2
# treat actual data exchange in either direction as a heartbeat
sent_now = self.bytes_sent
recv_now = self.bytes_recv
if self.prev_sent is None or self.prev_sent != sent_now:
self.last_heartbeat_sent = monotonic()
if self.prev_recv is None or self.prev_recv != recv_now:
self.last_heartbeat_received = monotonic()
now = monotonic()
AMQP_HEARTBEAT_LOGGER.debug(
'heartbeat_tick : Prev sent/recv: %s/%s, '
'now - %s/%s, monotonic - %s, '
'last_heartbeat_sent - %s, heartbeat int. - %s '
'for connection %s',
self.prev_sent, self.prev_recv,
sent_now, recv_now, now,
self.last_heartbeat_sent,
self.heartbeat,
self._connection_id,
)
self.prev_sent, self.prev_recv = sent_now, recv_now
# send a heartbeat if it's time to do so
if now > self.last_heartbeat_sent + self.heartbeat / rate:
AMQP_HEARTBEAT_LOGGER.debug(
'heartbeat_tick: sending heartbeat for connection %s',
self._connection_id)
self.send_heartbeat()
self.last_heartbeat_sent = monotonic()
# if we've missed two intervals' heartbeats, fail; this gives the
# server enough time to send heartbeats a little late
two_heartbeats = 2 * self.heartbeat
two_heartbeats_interval = self.last_heartbeat_received + two_heartbeats
heartbeats_missed = two_heartbeats_interval < monotonic()
if self.last_heartbeat_received and heartbeats_missed:
raise ConnectionForced('Too many heartbeats missed')
@property
def sock(self):
return self.transport.sock
@property
def server_capabilities(self):
return self.server_properties.get('capabilities') or {}

View File

@@ -0,0 +1,288 @@
"""Exceptions used by amqp."""
# Copyright (C) 2007-2008 Barry Pederson <bp@barryp.org>
from struct import pack, unpack
__all__ = (
'AMQPError',
'ConnectionError', 'ChannelError',
'RecoverableConnectionError', 'IrrecoverableConnectionError',
'RecoverableChannelError', 'IrrecoverableChannelError',
'ConsumerCancelled', 'ContentTooLarge', 'NoConsumers',
'ConnectionForced', 'InvalidPath', 'AccessRefused', 'NotFound',
'ResourceLocked', 'PreconditionFailed', 'FrameError', 'FrameSyntaxError',
'InvalidCommand', 'ChannelNotOpen', 'UnexpectedFrame', 'ResourceError',
'NotAllowed', 'AMQPNotImplementedError', 'InternalError',
'MessageNacked',
'AMQPDeprecationWarning',
)
class AMQPDeprecationWarning(UserWarning):
"""Warning for deprecated things."""
class MessageNacked(Exception):
"""Message was nacked by broker."""
class AMQPError(Exception):
"""Base class for all AMQP exceptions."""
code = 0
def __init__(self, reply_text=None, method_sig=None,
method_name=None, reply_code=None):
self.message = reply_text
self.reply_code = reply_code or self.code
self.reply_text = reply_text
self.method_sig = method_sig
self.method_name = method_name or ''
if method_sig and not self.method_name:
self.method_name = METHOD_NAME_MAP.get(method_sig, '')
Exception.__init__(self, reply_code,
reply_text, method_sig, self.method_name)
def __str__(self):
if self.method:
return '{0.method}: ({0.reply_code}) {0.reply_text}'.format(self)
return self.reply_text or '<{}: unknown error>'.format(
type(self).__name__
)
@property
def method(self):
return self.method_name or self.method_sig
class ConnectionError(AMQPError):
"""AMQP Connection Error."""
class ChannelError(AMQPError):
"""AMQP Channel Error."""
class RecoverableChannelError(ChannelError):
"""Exception class for recoverable channel errors."""
class IrrecoverableChannelError(ChannelError):
"""Exception class for irrecoverable channel errors."""
class RecoverableConnectionError(ConnectionError):
"""Exception class for recoverable connection errors."""
class IrrecoverableConnectionError(ConnectionError):
"""Exception class for irrecoverable connection errors."""
class Blocked(RecoverableConnectionError):
"""AMQP Connection Blocked Predicate."""
class ConsumerCancelled(RecoverableConnectionError):
"""AMQP Consumer Cancelled Predicate."""
class ContentTooLarge(RecoverableChannelError):
"""AMQP Content Too Large Error."""
code = 311
class NoConsumers(RecoverableChannelError):
"""AMQP No Consumers Error."""
code = 313
class ConnectionForced(RecoverableConnectionError):
"""AMQP Connection Forced Error."""
code = 320
class InvalidPath(IrrecoverableConnectionError):
"""AMQP Invalid Path Error."""
code = 402
class AccessRefused(IrrecoverableChannelError):
"""AMQP Access Refused Error."""
code = 403
class NotFound(IrrecoverableChannelError):
"""AMQP Not Found Error."""
code = 404
class ResourceLocked(RecoverableChannelError):
"""AMQP Resource Locked Error."""
code = 405
class PreconditionFailed(IrrecoverableChannelError):
"""AMQP Precondition Failed Error."""
code = 406
class FrameError(IrrecoverableConnectionError):
"""AMQP Frame Error."""
code = 501
class FrameSyntaxError(IrrecoverableConnectionError):
"""AMQP Frame Syntax Error."""
code = 502
class InvalidCommand(IrrecoverableConnectionError):
"""AMQP Invalid Command Error."""
code = 503
class ChannelNotOpen(IrrecoverableConnectionError):
"""AMQP Channel Not Open Error."""
code = 504
class UnexpectedFrame(IrrecoverableConnectionError):
"""AMQP Unexpected Frame."""
code = 505
class ResourceError(RecoverableConnectionError):
"""AMQP Resource Error."""
code = 506
class NotAllowed(IrrecoverableConnectionError):
"""AMQP Not Allowed Error."""
code = 530
class AMQPNotImplementedError(IrrecoverableConnectionError):
"""AMQP Not Implemented Error."""
code = 540
class InternalError(IrrecoverableConnectionError):
"""AMQP Internal Error."""
code = 541
ERROR_MAP = {
311: ContentTooLarge,
313: NoConsumers,
320: ConnectionForced,
402: InvalidPath,
403: AccessRefused,
404: NotFound,
405: ResourceLocked,
406: PreconditionFailed,
501: FrameError,
502: FrameSyntaxError,
503: InvalidCommand,
504: ChannelNotOpen,
505: UnexpectedFrame,
506: ResourceError,
530: NotAllowed,
540: AMQPNotImplementedError,
541: InternalError,
}
def error_for_code(code, text, method, default):
try:
return ERROR_MAP[code](text, method, reply_code=code)
except KeyError:
return default(text, method, reply_code=code)
METHOD_NAME_MAP = {
(10, 10): 'Connection.start',
(10, 11): 'Connection.start_ok',
(10, 20): 'Connection.secure',
(10, 21): 'Connection.secure_ok',
(10, 30): 'Connection.tune',
(10, 31): 'Connection.tune_ok',
(10, 40): 'Connection.open',
(10, 41): 'Connection.open_ok',
(10, 50): 'Connection.close',
(10, 51): 'Connection.close_ok',
(20, 10): 'Channel.open',
(20, 11): 'Channel.open_ok',
(20, 20): 'Channel.flow',
(20, 21): 'Channel.flow_ok',
(20, 40): 'Channel.close',
(20, 41): 'Channel.close_ok',
(30, 10): 'Access.request',
(30, 11): 'Access.request_ok',
(40, 10): 'Exchange.declare',
(40, 11): 'Exchange.declare_ok',
(40, 20): 'Exchange.delete',
(40, 21): 'Exchange.delete_ok',
(40, 30): 'Exchange.bind',
(40, 31): 'Exchange.bind_ok',
(40, 40): 'Exchange.unbind',
(40, 41): 'Exchange.unbind_ok',
(50, 10): 'Queue.declare',
(50, 11): 'Queue.declare_ok',
(50, 20): 'Queue.bind',
(50, 21): 'Queue.bind_ok',
(50, 30): 'Queue.purge',
(50, 31): 'Queue.purge_ok',
(50, 40): 'Queue.delete',
(50, 41): 'Queue.delete_ok',
(50, 50): 'Queue.unbind',
(50, 51): 'Queue.unbind_ok',
(60, 10): 'Basic.qos',
(60, 11): 'Basic.qos_ok',
(60, 20): 'Basic.consume',
(60, 21): 'Basic.consume_ok',
(60, 30): 'Basic.cancel',
(60, 31): 'Basic.cancel_ok',
(60, 40): 'Basic.publish',
(60, 50): 'Basic.return',
(60, 60): 'Basic.deliver',
(60, 70): 'Basic.get',
(60, 71): 'Basic.get_ok',
(60, 72): 'Basic.get_empty',
(60, 80): 'Basic.ack',
(60, 90): 'Basic.reject',
(60, 100): 'Basic.recover_async',
(60, 110): 'Basic.recover',
(60, 111): 'Basic.recover_ok',
(60, 120): 'Basic.nack',
(90, 10): 'Tx.select',
(90, 11): 'Tx.select_ok',
(90, 20): 'Tx.commit',
(90, 21): 'Tx.commit_ok',
(90, 30): 'Tx.rollback',
(90, 31): 'Tx.rollback_ok',
(85, 10): 'Confirm.select',
(85, 11): 'Confirm.select_ok',
}
for _method_id, _method_name in list(METHOD_NAME_MAP.items()):
METHOD_NAME_MAP[unpack('>I', pack('>HH', *_method_id))[0]] = \
_method_name

View File

@@ -0,0 +1,189 @@
"""Convert between frames and higher-level AMQP methods."""
# Copyright (C) 2007-2008 Barry Pederson <bp@barryp.org>
from collections import defaultdict
from struct import pack, pack_into, unpack_from
from . import spec
from .basic_message import Message
from .exceptions import UnexpectedFrame
from .utils import str_to_bytes
__all__ = ('frame_handler', 'frame_writer')
#: Set of methods that require both a content frame and a body frame.
_CONTENT_METHODS = frozenset([
spec.Basic.Return,
spec.Basic.Deliver,
spec.Basic.GetOk,
])
#: Number of bytes reserved for protocol in a content frame.
#: We use this to calculate when a frame exceeeds the max frame size,
#: and if it does not the message will fit into the preallocated buffer.
FRAME_OVERHEAD = 40
def frame_handler(connection, callback,
unpack_from=unpack_from, content_methods=_CONTENT_METHODS):
"""Create closure that reads frames."""
expected_types = defaultdict(lambda: 1)
partial_messages = {}
def on_frame(frame):
frame_type, channel, buf = frame
connection.bytes_recv += 1
if frame_type not in (expected_types[channel], 8):
raise UnexpectedFrame(
'Received frame {} while expecting type: {}'.format(
frame_type, expected_types[channel]),
)
elif frame_type == 1:
method_sig = unpack_from('>HH', buf, 0)
if method_sig in content_methods:
# Save what we've got so far and wait for the content-header
partial_messages[channel] = Message(
frame_method=method_sig, frame_args=buf,
)
expected_types[channel] = 2
return False
callback(channel, method_sig, buf, None)
elif frame_type == 2:
msg = partial_messages[channel]
msg.inbound_header(buf)
if not msg.ready:
# wait for the content-body
expected_types[channel] = 3
return False
# bodyless message, we're done
expected_types[channel] = 1
partial_messages.pop(channel, None)
callback(channel, msg.frame_method, msg.frame_args, msg)
elif frame_type == 3:
msg = partial_messages[channel]
msg.inbound_body(buf)
if not msg.ready:
# wait for the rest of the content-body
return False
expected_types[channel] = 1
partial_messages.pop(channel, None)
callback(channel, msg.frame_method, msg.frame_args, msg)
elif frame_type == 8:
# bytes_recv already updated
return False
return True
return on_frame
class Buffer:
def __init__(self, buf):
self.buf = buf
@property
def buf(self):
return self._buf
@buf.setter
def buf(self, buf):
self._buf = buf
# Using a memoryview allows slicing without copying underlying data.
# Slicing this is much faster than slicing the bytearray directly.
# More details: https://stackoverflow.com/a/34257357
self.view = memoryview(buf)
def frame_writer(connection, transport,
pack=pack, pack_into=pack_into, range=range, len=len,
bytes=bytes, str_to_bytes=str_to_bytes, text_t=str):
"""Create closure that writes frames."""
write = transport.write
buffer_store = Buffer(bytearray(connection.frame_max - 8))
def write_frame(type_, channel, method_sig, args, content):
chunk_size = connection.frame_max - 8
offset = 0
properties = None
args = str_to_bytes(args)
if content:
body = content.body
if isinstance(body, str):
encoding = content.properties.setdefault(
'content_encoding', 'utf-8')
body = body.encode(encoding)
properties = content._serialize_properties()
bodylen = len(body)
properties_len = len(properties) or 0
framelen = len(args) + properties_len + bodylen + FRAME_OVERHEAD
bigbody = framelen > chunk_size
else:
body, bodylen, bigbody = None, 0, 0
if bigbody:
# ## SLOW: string copy and write for every frame
frame = (b''.join([pack('>HH', *method_sig), args])
if type_ == 1 else b'') # encode method frame
framelen = len(frame)
write(pack('>BHI%dsB' % framelen,
type_, channel, framelen, frame, 0xce))
if body:
frame = b''.join([
pack('>HHQ', method_sig[0], 0, len(body)),
properties,
])
framelen = len(frame)
write(pack('>BHI%dsB' % framelen,
2, channel, framelen, frame, 0xce))
for i in range(0, bodylen, chunk_size):
frame = body[i:i + chunk_size]
framelen = len(frame)
write(pack('>BHI%dsB' % framelen,
3, channel, framelen,
frame, 0xce))
else:
# frame_max can be updated via connection._on_tune. If
# it became larger, then we need to resize the buffer
# to prevent overflow.
if chunk_size > len(buffer_store.buf):
buffer_store.buf = bytearray(chunk_size)
buf = buffer_store.buf
# ## FAST: pack into buffer and single write
frame = (b''.join([pack('>HH', *method_sig), args])
if type_ == 1 else b'')
framelen = len(frame)
pack_into('>BHI%dsB' % framelen, buf, offset,
type_, channel, framelen, frame, 0xce)
offset += 8 + framelen
if body is not None:
frame = b''.join([
pack('>HHQ', method_sig[0], 0, len(body)),
properties,
])
framelen = len(frame)
pack_into('>BHI%dsB' % framelen, buf, offset,
2, channel, framelen, frame, 0xce)
offset += 8 + framelen
bodylen = len(body)
if bodylen > 0:
framelen = bodylen
pack_into('>BHI%dsB' % framelen, buf, offset,
3, channel, framelen, body, 0xce)
offset += 8 + framelen
write(buffer_store.view[:offset])
connection.bytes_sent += 1
return write_frame

View File

@@ -0,0 +1,79 @@
"""Platform compatibility."""
import platform
import re
import sys
# Jython does not have this attribute
import typing
try:
from socket import SOL_TCP
except ImportError: # pragma: no cover
from socket import IPPROTO_TCP as SOL_TCP # noqa
RE_NUM = re.compile(r'(\d+).+')
def _linux_version_to_tuple(s: str) -> typing.Tuple[int, int, int]:
return tuple(map(_versionatom, s.split('.')[:3]))
def _versionatom(s: str) -> int:
if s.isdigit():
return int(s)
match = RE_NUM.match(s)
return int(match.groups()[0]) if match else 0
# available socket options for TCP level
KNOWN_TCP_OPTS = {
'TCP_CORK', 'TCP_DEFER_ACCEPT', 'TCP_KEEPCNT',
'TCP_KEEPIDLE', 'TCP_KEEPINTVL', 'TCP_LINGER2',
'TCP_MAXSEG', 'TCP_NODELAY', 'TCP_QUICKACK',
'TCP_SYNCNT', 'TCP_USER_TIMEOUT', 'TCP_WINDOW_CLAMP',
}
LINUX_VERSION = None
if sys.platform.startswith('linux'):
LINUX_VERSION = _linux_version_to_tuple(platform.release())
if LINUX_VERSION < (2, 6, 37):
KNOWN_TCP_OPTS.remove('TCP_USER_TIMEOUT')
# Windows Subsystem for Linux is an edge-case: the Python socket library
# returns most TCP_* enums, but they aren't actually supported
if platform.release().endswith("Microsoft"):
KNOWN_TCP_OPTS = {'TCP_NODELAY', 'TCP_KEEPIDLE', 'TCP_KEEPINTVL',
'TCP_KEEPCNT'}
elif sys.platform.startswith('darwin'):
KNOWN_TCP_OPTS.remove('TCP_USER_TIMEOUT')
elif 'bsd' in sys.platform:
KNOWN_TCP_OPTS.remove('TCP_USER_TIMEOUT')
# According to MSDN Windows platforms support getsockopt(TCP_MAXSSEG) but not
# setsockopt(TCP_MAXSEG) on IPPROTO_TCP sockets.
elif sys.platform.startswith('win'):
KNOWN_TCP_OPTS = {'TCP_NODELAY'}
elif sys.platform.startswith('cygwin'):
KNOWN_TCP_OPTS = {'TCP_NODELAY'}
# illumos does not allow to set the TCP_MAXSEG socket option,
# even if the Oracle documentation says otherwise.
# TCP_USER_TIMEOUT does not exist on Solaris 11.4
elif sys.platform.startswith('sunos'):
KNOWN_TCP_OPTS.remove('TCP_MAXSEG')
KNOWN_TCP_OPTS.remove('TCP_USER_TIMEOUT')
# aix does not allow to set the TCP_MAXSEG
# or the TCP_USER_TIMEOUT socket options.
elif sys.platform.startswith('aix'):
KNOWN_TCP_OPTS.remove('TCP_MAXSEG')
KNOWN_TCP_OPTS.remove('TCP_USER_TIMEOUT')
__all__ = (
'LINUX_VERSION',
'SOL_TCP',
'KNOWN_TCP_OPTS',
)

View File

@@ -0,0 +1,12 @@
"""Protocol data."""
from collections import namedtuple
queue_declare_ok_t = namedtuple(
'queue_declare_ok_t', ('queue', 'message_count', 'consumer_count'),
)
basic_return_t = namedtuple(
'basic_return_t',
('reply_code', 'reply_text', 'exchange', 'routing_key', 'message'),
)

View File

@@ -0,0 +1,191 @@
"""SASL mechanisms for AMQP authentication."""
import socket
import warnings
from io import BytesIO
from amqp.serialization import _write_table
class SASL:
"""The base class for all amqp SASL authentication mechanisms.
You should sub-class this if you're implementing your own authentication.
"""
@property
def mechanism(self):
"""Return a bytes containing the SASL mechanism name."""
raise NotImplementedError
def start(self, connection):
"""Return the first response to a SASL challenge as a bytes object."""
raise NotImplementedError
class PLAIN(SASL):
"""PLAIN SASL authentication mechanism.
See https://tools.ietf.org/html/rfc4616 for details
"""
mechanism = b'PLAIN'
def __init__(self, username, password):
self.username, self.password = username, password
__slots__ = (
"username",
"password",
)
def start(self, connection):
if self.username is None or self.password is None:
return NotImplemented
login_response = BytesIO()
login_response.write(b'\0')
login_response.write(self.username.encode('utf-8'))
login_response.write(b'\0')
login_response.write(self.password.encode('utf-8'))
return login_response.getvalue()
class AMQPLAIN(SASL):
"""AMQPLAIN SASL authentication mechanism.
This is a non-standard mechanism used by AMQP servers.
"""
mechanism = b'AMQPLAIN'
def __init__(self, username, password):
self.username, self.password = username, password
__slots__ = (
"username",
"password",
)
def start(self, connection):
if self.username is None or self.password is None:
return NotImplemented
login_response = BytesIO()
_write_table({b'LOGIN': self.username, b'PASSWORD': self.password},
login_response.write, [])
# Skip the length at the beginning
return login_response.getvalue()[4:]
def _get_gssapi_mechanism():
try:
import gssapi
import gssapi.raw.misc # Fail if the old python-gssapi is installed
except ImportError:
class FakeGSSAPI(SASL):
"""A no-op SASL mechanism for when gssapi isn't available."""
mechanism = None
def __init__(self, client_name=None, service=b'amqp',
rdns=False, fail_soft=False):
if not fail_soft:
raise NotImplementedError(
"You need to install the `gssapi` module for GSSAPI "
"SASL support")
def start(self): # pragma: no cover
return NotImplemented
return FakeGSSAPI
else:
class GSSAPI(SASL):
"""GSSAPI SASL authentication mechanism.
See https://tools.ietf.org/html/rfc4752 for details
"""
mechanism = b'GSSAPI'
def __init__(self, client_name=None, service=b'amqp',
rdns=False, fail_soft=False):
if client_name and not isinstance(client_name, bytes):
client_name = client_name.encode('ascii')
self.client_name = client_name
self.fail_soft = fail_soft
self.service = service
self.rdns = rdns
__slots__ = (
"client_name",
"fail_soft",
"service",
"rdns"
)
def get_hostname(self, connection):
sock = connection.transport.sock
if self.rdns and sock.family in (socket.AF_INET,
socket.AF_INET6):
peer = sock.getpeername()
hostname, _, _ = socket.gethostbyaddr(peer[0])
else:
hostname = connection.transport.host
if not isinstance(hostname, bytes):
hostname = hostname.encode('ascii')
return hostname
def start(self, connection):
try:
if self.client_name:
creds = gssapi.Credentials(
name=gssapi.Name(self.client_name))
else:
creds = None
hostname = self.get_hostname(connection)
name = gssapi.Name(b'@'.join([self.service, hostname]),
gssapi.NameType.hostbased_service)
context = gssapi.SecurityContext(name=name, creds=creds)
return context.step(None)
except gssapi.raw.misc.GSSError:
if self.fail_soft:
return NotImplemented
else:
raise
return GSSAPI
GSSAPI = _get_gssapi_mechanism()
class EXTERNAL(SASL):
"""EXTERNAL SASL mechanism.
Enables external authentication, i.e. not handled through this protocol.
Only passes 'EXTERNAL' as authentication mechanism, but no further
authentication data.
"""
mechanism = b'EXTERNAL'
def start(self, connection):
return b''
class RAW(SASL):
"""A generic custom SASL mechanism.
This mechanism takes a mechanism name and response to send to the server,
so can be used for simple custom authentication schemes.
"""
mechanism = None
def __init__(self, mechanism, response):
assert isinstance(mechanism, bytes)
assert isinstance(response, bytes)
self.mechanism, self.response = mechanism, response
warnings.warn("Passing login_method and login_response to Connection "
"is deprecated. Please implement a SASL subclass "
"instead.", DeprecationWarning)
def start(self, connection):
return self.response

View File

@@ -0,0 +1,582 @@
"""Convert between bytestreams and higher-level AMQP types.
2007-11-05 Barry Pederson <bp@barryp.org>
"""
# Copyright (C) 2007 Barry Pederson <bp@barryp.org>
import calendar
from datetime import datetime
from decimal import Decimal
from io import BytesIO
from struct import pack, unpack_from
from .exceptions import FrameSyntaxError
from .spec import Basic
from .utils import bytes_to_str as pstr_t
from .utils import str_to_bytes
ILLEGAL_TABLE_TYPE = """\
Table type {0!r} not handled by amqp.
"""
ILLEGAL_TABLE_TYPE_WITH_KEY = """\
Table type {0!r} for key {1!r} not handled by amqp. [value: {2!r}]
"""
ILLEGAL_TABLE_TYPE_WITH_VALUE = """\
Table type {0!r} not handled by amqp. [value: {1!r}]
"""
def _read_item(buf, offset):
ftype = chr(buf[offset])
offset += 1
# 'S': long string
if ftype == 'S':
slen, = unpack_from('>I', buf, offset)
offset += 4
try:
val = pstr_t(buf[offset:offset + slen])
except UnicodeDecodeError:
val = buf[offset:offset + slen]
offset += slen
# 's': short string
elif ftype == 's':
slen, = unpack_from('>B', buf, offset)
offset += 1
val = pstr_t(buf[offset:offset + slen])
offset += slen
# 'x': Bytes Array
elif ftype == 'x':
blen, = unpack_from('>I', buf, offset)
offset += 4
val = buf[offset:offset + blen]
offset += blen
# 'b': short-short int
elif ftype == 'b':
val, = unpack_from('>B', buf, offset)
offset += 1
# 'B': short-short unsigned int
elif ftype == 'B':
val, = unpack_from('>b', buf, offset)
offset += 1
# 'U': short int
elif ftype == 'U':
val, = unpack_from('>h', buf, offset)
offset += 2
# 'u': short unsigned int
elif ftype == 'u':
val, = unpack_from('>H', buf, offset)
offset += 2
# 'I': long int
elif ftype == 'I':
val, = unpack_from('>i', buf, offset)
offset += 4
# 'i': long unsigned int
elif ftype == 'i':
val, = unpack_from('>I', buf, offset)
offset += 4
# 'L': long long int
elif ftype == 'L':
val, = unpack_from('>q', buf, offset)
offset += 8
# 'l': long long unsigned int
elif ftype == 'l':
val, = unpack_from('>Q', buf, offset)
offset += 8
# 'f': float
elif ftype == 'f':
val, = unpack_from('>f', buf, offset)
offset += 4
# 'd': double
elif ftype == 'd':
val, = unpack_from('>d', buf, offset)
offset += 8
# 'D': decimal
elif ftype == 'D':
d, = unpack_from('>B', buf, offset)
offset += 1
n, = unpack_from('>i', buf, offset)
offset += 4
val = Decimal(n) / Decimal(10 ** d)
# 'F': table
elif ftype == 'F':
tlen, = unpack_from('>I', buf, offset)
offset += 4
limit = offset + tlen
val = {}
while offset < limit:
keylen, = unpack_from('>B', buf, offset)
offset += 1
key = pstr_t(buf[offset:offset + keylen])
offset += keylen
val[key], offset = _read_item(buf, offset)
# 'A': array
elif ftype == 'A':
alen, = unpack_from('>I', buf, offset)
offset += 4
limit = offset + alen
val = []
while offset < limit:
v, offset = _read_item(buf, offset)
val.append(v)
# 't' (bool)
elif ftype == 't':
val, = unpack_from('>B', buf, offset)
val = bool(val)
offset += 1
# 'T': timestamp
elif ftype == 'T':
val, = unpack_from('>Q', buf, offset)
offset += 8
val = datetime.utcfromtimestamp(val)
# 'V': void
elif ftype == 'V':
val = None
else:
raise FrameSyntaxError(
'Unknown value in table: {!r} ({!r})'.format(
ftype, type(ftype)))
return val, offset
def loads(format, buf, offset):
"""Deserialize amqp format.
bit = b
octet = o
short = B
long = l
long long = L
float = f
shortstr = s
longstr = S
table = F
array = A
timestamp = T
"""
bitcount = bits = 0
values = []
append = values.append
format = pstr_t(format)
for p in format:
if p == 'b':
if not bitcount:
bits = ord(buf[offset:offset + 1])
offset += 1
bitcount = 8
val = (bits & 1) == 1
bits >>= 1
bitcount -= 1
elif p == 'o':
bitcount = bits = 0
val, = unpack_from('>B', buf, offset)
offset += 1
elif p == 'B':
bitcount = bits = 0
val, = unpack_from('>H', buf, offset)
offset += 2
elif p == 'l':
bitcount = bits = 0
val, = unpack_from('>I', buf, offset)
offset += 4
elif p == 'L':
bitcount = bits = 0
val, = unpack_from('>Q', buf, offset)
offset += 8
elif p == 'f':
bitcount = bits = 0
val, = unpack_from('>f', buf, offset)
offset += 4
elif p == 's':
bitcount = bits = 0
slen, = unpack_from('B', buf, offset)
offset += 1
val = buf[offset:offset + slen].decode('utf-8', 'surrogatepass')
offset += slen
elif p == 'S':
bitcount = bits = 0
slen, = unpack_from('>I', buf, offset)
offset += 4
val = buf[offset:offset + slen].decode('utf-8', 'surrogatepass')
offset += slen
elif p == 'x':
blen, = unpack_from('>I', buf, offset)
offset += 4
val = buf[offset:offset + blen]
offset += blen
elif p == 'F':
bitcount = bits = 0
tlen, = unpack_from('>I', buf, offset)
offset += 4
limit = offset + tlen
val = {}
while offset < limit:
keylen, = unpack_from('>B', buf, offset)
offset += 1
key = pstr_t(buf[offset:offset + keylen])
offset += keylen
val[key], offset = _read_item(buf, offset)
elif p == 'A':
bitcount = bits = 0
alen, = unpack_from('>I', buf, offset)
offset += 4
limit = offset + alen
val = []
while offset < limit:
aval, offset = _read_item(buf, offset)
val.append(aval)
elif p == 'T':
bitcount = bits = 0
val, = unpack_from('>Q', buf, offset)
offset += 8
val = datetime.utcfromtimestamp(val)
else:
raise FrameSyntaxError(ILLEGAL_TABLE_TYPE.format(p))
append(val)
return values, offset
def _flushbits(bits, write):
if bits:
write(pack('B' * len(bits), *bits))
bits[:] = []
return 0
def dumps(format, values):
"""Serialize AMQP arguments.
Notes:
bit = b
octet = o
short = B
long = l
long long = L
shortstr = s
longstr = S
byte array = x
table = F
array = A
"""
bitcount = 0
bits = []
out = BytesIO()
write = out.write
format = pstr_t(format)
for i, val in enumerate(values):
p = format[i]
if p == 'b':
val = 1 if val else 0
shift = bitcount % 8
if shift == 0:
bits.append(0)
bits[-1] |= (val << shift)
bitcount += 1
elif p == 'o':
bitcount = _flushbits(bits, write)
write(pack('B', val))
elif p == 'B':
bitcount = _flushbits(bits, write)
write(pack('>H', int(val)))
elif p == 'l':
bitcount = _flushbits(bits, write)
write(pack('>I', val))
elif p == 'L':
bitcount = _flushbits(bits, write)
write(pack('>Q', val))
elif p == 'f':
bitcount = _flushbits(bits, write)
write(pack('>f', val))
elif p == 's':
val = val or ''
bitcount = _flushbits(bits, write)
if isinstance(val, str):
val = val.encode('utf-8', 'surrogatepass')
write(pack('B', len(val)))
write(val)
elif p == 'S' or p == 'x':
val = val or ''
bitcount = _flushbits(bits, write)
if isinstance(val, str):
val = val.encode('utf-8', 'surrogatepass')
write(pack('>I', len(val)))
write(val)
elif p == 'F':
bitcount = _flushbits(bits, write)
_write_table(val or {}, write, bits)
elif p == 'A':
bitcount = _flushbits(bits, write)
_write_array(val or [], write, bits)
elif p == 'T':
write(pack('>Q', int(calendar.timegm(val.utctimetuple()))))
_flushbits(bits, write)
return out.getvalue()
def _write_table(d, write, bits):
out = BytesIO()
twrite = out.write
for k, v in d.items():
if isinstance(k, str):
k = k.encode('utf-8', 'surrogatepass')
twrite(pack('B', len(k)))
twrite(k)
try:
_write_item(v, twrite, bits)
except ValueError:
raise FrameSyntaxError(
ILLEGAL_TABLE_TYPE_WITH_KEY.format(type(v), k, v))
table_data = out.getvalue()
write(pack('>I', len(table_data)))
write(table_data)
def _write_array(list_, write, bits):
out = BytesIO()
awrite = out.write
for v in list_:
try:
_write_item(v, awrite, bits)
except ValueError:
raise FrameSyntaxError(
ILLEGAL_TABLE_TYPE_WITH_VALUE.format(type(v), v))
array_data = out.getvalue()
write(pack('>I', len(array_data)))
write(array_data)
def _write_item(v, write, bits):
if isinstance(v, (str, bytes)):
if isinstance(v, str):
v = v.encode('utf-8', 'surrogatepass')
write(pack('>cI', b'S', len(v)))
write(v)
elif isinstance(v, bool):
write(pack('>cB', b't', int(v)))
elif isinstance(v, float):
write(pack('>cd', b'd', v))
elif isinstance(v, int):
if v > 2147483647 or v < -2147483647:
write(pack('>cq', b'L', v))
else:
write(pack('>ci', b'I', v))
elif isinstance(v, Decimal):
sign, digits, exponent = v.as_tuple()
v = 0
for d in digits:
v = (v * 10) + d
if sign:
v = -v
write(pack('>cBi', b'D', -exponent, v))
elif isinstance(v, datetime):
write(
pack('>cQ', b'T', int(calendar.timegm(v.utctimetuple()))))
elif isinstance(v, dict):
write(b'F')
_write_table(v, write, bits)
elif isinstance(v, (list, tuple)):
write(b'A')
_write_array(v, write, bits)
elif v is None:
write(b'V')
else:
raise ValueError()
def decode_properties_basic(buf, offset):
"""Decode basic properties."""
properties = {}
flags, = unpack_from('>H', buf, offset)
offset += 2
if flags & 0x8000:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['content_type'] = pstr_t(buf[offset:offset + slen])
offset += slen
if flags & 0x4000:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['content_encoding'] = pstr_t(buf[offset:offset + slen])
offset += slen
if flags & 0x2000:
_f, offset = loads('F', buf, offset)
properties['application_headers'], = _f
if flags & 0x1000:
properties['delivery_mode'], = unpack_from('>B', buf, offset)
offset += 1
if flags & 0x0800:
properties['priority'], = unpack_from('>B', buf, offset)
offset += 1
if flags & 0x0400:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['correlation_id'] = pstr_t(buf[offset:offset + slen])
offset += slen
if flags & 0x0200:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['reply_to'] = pstr_t(buf[offset:offset + slen])
offset += slen
if flags & 0x0100:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['expiration'] = pstr_t(buf[offset:offset + slen])
offset += slen
if flags & 0x0080:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['message_id'] = pstr_t(buf[offset:offset + slen])
offset += slen
if flags & 0x0040:
properties['timestamp'], = unpack_from('>Q', buf, offset)
offset += 8
if flags & 0x0020:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['type'] = pstr_t(buf[offset:offset + slen])
offset += slen
if flags & 0x0010:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['user_id'] = pstr_t(buf[offset:offset + slen])
offset += slen
if flags & 0x0008:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['app_id'] = pstr_t(buf[offset:offset + slen])
offset += slen
if flags & 0x0004:
slen, = unpack_from('>B', buf, offset)
offset += 1
properties['cluster_id'] = pstr_t(buf[offset:offset + slen])
offset += slen
return properties, offset
PROPERTY_CLASSES = {
Basic.CLASS_ID: decode_properties_basic,
}
class GenericContent:
"""Abstract base class for AMQP content.
Subclasses should override the PROPERTIES attribute.
"""
CLASS_ID = None
PROPERTIES = [('dummy', 's')]
def __init__(self, frame_method=None, frame_args=None, **props):
self.frame_method = frame_method
self.frame_args = frame_args
self.properties = props
self._pending_chunks = []
self.body_received = 0
self.body_size = 0
self.ready = False
__slots__ = (
"frame_method",
"frame_args",
"properties",
"_pending_chunks",
"body_received",
"body_size",
"ready",
# adding '__dict__' to get dynamic assignment
"__dict__",
"__weakref__",
)
def __getattr__(self, name):
# Look for additional properties in the 'properties'
# dictionary, and if present - the 'delivery_info' dictionary.
if name == '__setstate__':
# Allows pickling/unpickling to work
raise AttributeError('__setstate__')
if name in self.properties:
return self.properties[name]
raise AttributeError(name)
def _load_properties(self, class_id, buf, offset):
"""Load AMQP properties.
Given the raw bytes containing the property-flags and property-list
from a content-frame-header, parse and insert into a dictionary
stored in this object as an attribute named 'properties'.
"""
# Read 16-bit shorts until we get one with a low bit set to zero
props, offset = PROPERTY_CLASSES[class_id](buf, offset)
self.properties = props
return offset
def _serialize_properties(self):
"""Serialize AMQP properties.
Serialize the 'properties' attribute (a dictionary) into
the raw bytes making up a set of property flags and a
property list, suitable for putting into a content frame header.
"""
shift = 15
flag_bits = 0
flags = []
sformat, svalues = [], []
props = self.properties
for key, proptype in self.PROPERTIES:
val = props.get(key, None)
if val is not None:
if shift == 0:
flags.append(flag_bits)
flag_bits = 0
shift = 15
flag_bits |= (1 << shift)
if proptype != 'bit':
sformat.append(str_to_bytes(proptype))
svalues.append(val)
shift -= 1
flags.append(flag_bits)
result = BytesIO()
write = result.write
for flag_bits in flags:
write(pack('>H', flag_bits))
write(dumps(b''.join(sformat), svalues))
return result.getvalue()
def inbound_header(self, buf, offset=0):
class_id, self.body_size = unpack_from('>HxxQ', buf, offset)
offset += 12
self._load_properties(class_id, buf, offset)
if not self.body_size:
self.ready = True
return offset
def inbound_body(self, buf):
chunks = self._pending_chunks
self.body_received += len(buf)
if self.body_received >= self.body_size:
if chunks:
chunks.append(buf)
self.body = bytes().join(chunks)
chunks[:] = []
else:
self.body = buf
self.ready = True
else:
chunks.append(buf)

View File

@@ -0,0 +1,121 @@
"""AMQP Spec."""
from collections import namedtuple
method_t = namedtuple('method_t', ('method_sig', 'args', 'content'))
def method(method_sig, args=None, content=False):
"""Create amqp method specification tuple."""
return method_t(method_sig, args, content)
class Connection:
"""AMQ Connection class."""
CLASS_ID = 10
Start = (10, 10)
StartOk = (10, 11)
Secure = (10, 20)
SecureOk = (10, 21)
Tune = (10, 30)
TuneOk = (10, 31)
Open = (10, 40)
OpenOk = (10, 41)
Close = (10, 50)
CloseOk = (10, 51)
Blocked = (10, 60)
Unblocked = (10, 61)
class Channel:
"""AMQ Channel class."""
CLASS_ID = 20
Open = (20, 10)
OpenOk = (20, 11)
Flow = (20, 20)
FlowOk = (20, 21)
Close = (20, 40)
CloseOk = (20, 41)
class Exchange:
"""AMQ Exchange class."""
CLASS_ID = 40
Declare = (40, 10)
DeclareOk = (40, 11)
Delete = (40, 20)
DeleteOk = (40, 21)
Bind = (40, 30)
BindOk = (40, 31)
Unbind = (40, 40)
UnbindOk = (40, 51)
class Queue:
"""AMQ Queue class."""
CLASS_ID = 50
Declare = (50, 10)
DeclareOk = (50, 11)
Bind = (50, 20)
BindOk = (50, 21)
Purge = (50, 30)
PurgeOk = (50, 31)
Delete = (50, 40)
DeleteOk = (50, 41)
Unbind = (50, 50)
UnbindOk = (50, 51)
class Basic:
"""AMQ Basic class."""
CLASS_ID = 60
Qos = (60, 10)
QosOk = (60, 11)
Consume = (60, 20)
ConsumeOk = (60, 21)
Cancel = (60, 30)
CancelOk = (60, 31)
Publish = (60, 40)
Return = (60, 50)
Deliver = (60, 60)
Get = (60, 70)
GetOk = (60, 71)
GetEmpty = (60, 72)
Ack = (60, 80)
Nack = (60, 120)
Reject = (60, 90)
RecoverAsync = (60, 100)
Recover = (60, 110)
RecoverOk = (60, 111)
class Confirm:
"""AMQ Confirm class."""
CLASS_ID = 85
Select = (85, 10)
SelectOk = (85, 11)
class Tx:
"""AMQ Tx class."""
CLASS_ID = 90
Select = (90, 10)
SelectOk = (90, 11)
Commit = (90, 20)
CommitOk = (90, 21)
Rollback = (90, 30)
RollbackOk = (90, 31)

View File

@@ -0,0 +1,679 @@
"""Transport implementation."""
# Copyright (C) 2009 Barry Pederson <bp@barryp.org>
import errno
import os
import re
import socket
import ssl
from contextlib import contextmanager
from ssl import SSLError
from struct import pack, unpack
from .exceptions import UnexpectedFrame
from .platform import KNOWN_TCP_OPTS, SOL_TCP
from .utils import set_cloexec
_UNAVAIL = {errno.EAGAIN, errno.EINTR, errno.ENOENT, errno.EWOULDBLOCK}
AMQP_PORT = 5672
EMPTY_BUFFER = bytes()
SIGNED_INT_MAX = 0x7FFFFFFF
# Yes, Advanced Message Queuing Protocol Protocol is redundant
AMQP_PROTOCOL_HEADER = b'AMQP\x00\x00\x09\x01'
# Match things like: [fe80::1]:5432, from RFC 2732
IPV6_LITERAL = re.compile(r'\[([\.0-9a-f:]+)\](?::(\d+))?')
DEFAULT_SOCKET_SETTINGS = {
'TCP_NODELAY': 1,
'TCP_USER_TIMEOUT': 1000,
'TCP_KEEPIDLE': 60,
'TCP_KEEPINTVL': 10,
'TCP_KEEPCNT': 9,
}
def to_host_port(host, default=AMQP_PORT):
"""Convert hostname:port string to host, port tuple."""
port = default
m = IPV6_LITERAL.match(host)
if m:
host = m.group(1)
if m.group(2):
port = int(m.group(2))
else:
if ':' in host:
host, port = host.rsplit(':', 1)
port = int(port)
return host, port
class _AbstractTransport:
"""Common superclass for TCP and SSL transports.
PARAMETERS:
host: str
Broker address in format ``HOSTNAME:PORT``.
connect_timeout: int
Timeout of creating new connection.
read_timeout: int
sets ``SO_RCVTIMEO`` parameter of socket.
write_timeout: int
sets ``SO_SNDTIMEO`` parameter of socket.
socket_settings: dict
dictionary containing `optname` and ``optval`` passed to
``setsockopt(2)``.
raise_on_initial_eintr: bool
when True, ``socket.timeout`` is raised
when exception is received during first read. See ``_read()`` for
details.
"""
def __init__(self, host, connect_timeout=None,
read_timeout=None, write_timeout=None,
socket_settings=None, raise_on_initial_eintr=True, **kwargs):
self.connected = False
self.sock = None
self.raise_on_initial_eintr = raise_on_initial_eintr
self._read_buffer = EMPTY_BUFFER
self.host, self.port = to_host_port(host)
self.connect_timeout = connect_timeout
self.read_timeout = read_timeout
self.write_timeout = write_timeout
self.socket_settings = socket_settings
__slots__ = (
"connection",
"sock",
"raise_on_initial_eintr",
"_read_buffer",
"host",
"port",
"connect_timeout",
"read_timeout",
"write_timeout",
"socket_settings",
# adding '__dict__' to get dynamic assignment
"__dict__",
"__weakref__",
)
def __repr__(self):
if self.sock:
src = f'{self.sock.getsockname()[0]}:{self.sock.getsockname()[1]}'
try:
dst = f'{self.sock.getpeername()[0]}:{self.sock.getpeername()[1]}'
except (socket.error) as e:
dst = f'ERROR: {e}'
return f'<{type(self).__name__}: {src} -> {dst} at {id(self):#x}>'
else:
return f'<{type(self).__name__}: (disconnected) at {id(self):#x}>'
def connect(self):
try:
# are we already connected?
if self.connected:
return
self._connect(self.host, self.port, self.connect_timeout)
self._init_socket(
self.socket_settings, self.read_timeout, self.write_timeout,
)
# we've sent the banner; signal connect
# EINTR, EAGAIN, EWOULDBLOCK would signal that the banner
# has _not_ been sent
self.connected = True
except (OSError, SSLError):
# if not fully connected, close socket, and reraise error
if self.sock and not self.connected:
self.sock.close()
self.sock = None
raise
@contextmanager
def having_timeout(self, timeout):
if timeout is None:
yield self.sock
else:
sock = self.sock
prev = sock.gettimeout()
if prev != timeout:
sock.settimeout(timeout)
try:
yield self.sock
except SSLError as exc:
if 'timed out' in str(exc):
# http://bugs.python.org/issue10272
raise socket.timeout()
elif 'The operation did not complete' in str(exc):
# Non-blocking SSL sockets can throw SSLError
raise socket.timeout()
raise
except OSError as exc:
if exc.errno == errno.EWOULDBLOCK:
raise socket.timeout()
raise
finally:
if timeout != prev:
sock.settimeout(prev)
def _connect(self, host, port, timeout):
entries = socket.getaddrinfo(
host, port, socket.AF_UNSPEC, socket.SOCK_STREAM, SOL_TCP,
)
for i, res in enumerate(entries):
af, socktype, proto, canonname, sa = res
try:
self.sock = socket.socket(af, socktype, proto)
try:
set_cloexec(self.sock, True)
except NotImplementedError:
pass
self.sock.settimeout(timeout)
self.sock.connect(sa)
except socket.error:
if self.sock:
self.sock.close()
self.sock = None
if i + 1 >= len(entries):
raise
else:
break
def _init_socket(self, socket_settings, read_timeout, write_timeout):
self.sock.settimeout(None) # set socket back to blocking mode
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
self._set_socket_options(socket_settings)
# set socket timeouts
for timeout, interval in ((socket.SO_SNDTIMEO, write_timeout),
(socket.SO_RCVTIMEO, read_timeout)):
if interval is not None:
sec = int(interval)
usec = int((interval - sec) * 1000000)
self.sock.setsockopt(
socket.SOL_SOCKET, timeout,
pack('ll', sec, usec),
)
self._setup_transport()
self._write(AMQP_PROTOCOL_HEADER)
def _get_tcp_socket_defaults(self, sock):
tcp_opts = {}
for opt in KNOWN_TCP_OPTS:
enum = None
if opt == 'TCP_USER_TIMEOUT':
try:
from socket import TCP_USER_TIMEOUT as enum
except ImportError:
# should be in Python 3.6+ on Linux.
enum = 18
elif hasattr(socket, opt):
enum = getattr(socket, opt)
if enum:
if opt in DEFAULT_SOCKET_SETTINGS:
tcp_opts[enum] = DEFAULT_SOCKET_SETTINGS[opt]
elif hasattr(socket, opt):
tcp_opts[enum] = sock.getsockopt(
SOL_TCP, getattr(socket, opt))
return tcp_opts
def _set_socket_options(self, socket_settings):
tcp_opts = self._get_tcp_socket_defaults(self.sock)
if socket_settings:
tcp_opts.update(socket_settings)
for opt, val in tcp_opts.items():
self.sock.setsockopt(SOL_TCP, opt, val)
def _read(self, n, initial=False):
"""Read exactly n bytes from the peer."""
raise NotImplementedError('Must be overridden in subclass')
def _setup_transport(self):
"""Do any additional initialization of the class."""
pass
def _shutdown_transport(self):
"""Do any preliminary work in shutting down the connection."""
pass
def _write(self, s):
"""Completely write a string to the peer."""
raise NotImplementedError('Must be overridden in subclass')
def close(self):
if self.sock is not None:
try:
self._shutdown_transport()
except OSError:
pass
# Call shutdown first to make sure that pending messages
# reach the AMQP broker if the program exits after
# calling this method.
try:
self.sock.shutdown(socket.SHUT_RDWR)
except OSError:
pass
try:
self.sock.close()
except OSError:
pass
self.sock = None
self.connected = False
def read_frame(self, unpack=unpack):
"""Parse AMQP frame.
Frame has following format::
0 1 3 7 size+7 size+8
+------+---------+---------+ +-------------+ +-----------+
| type | channel | size | | payload | | frame-end |
+------+---------+---------+ +-------------+ +-----------+
octet short long 'size' octets octet
"""
read = self._read
read_frame_buffer = EMPTY_BUFFER
try:
frame_header = read(7, True)
read_frame_buffer += frame_header
frame_type, channel, size = unpack('>BHI', frame_header)
# >I is an unsigned int, but the argument to sock.recv is signed,
# so we know the size can be at most 2 * SIGNED_INT_MAX
if size > SIGNED_INT_MAX:
part1 = read(SIGNED_INT_MAX)
try:
part2 = read(size - SIGNED_INT_MAX)
except (socket.timeout, OSError, SSLError):
# In case this read times out, we need to make sure to not
# lose part1 when we retry the read
read_frame_buffer += part1
raise
payload = b''.join([part1, part2])
else:
payload = read(size)
read_frame_buffer += payload
frame_end = ord(read(1))
except socket.timeout:
self._read_buffer = read_frame_buffer + self._read_buffer
raise
except (OSError, SSLError) as exc:
if (
isinstance(exc, socket.error) and os.name == 'nt'
and exc.errno == errno.EWOULDBLOCK # noqa
):
# On windows we can get a read timeout with a winsock error
# code instead of a proper socket.timeout() error, see
# https://github.com/celery/py-amqp/issues/320
self._read_buffer = read_frame_buffer + self._read_buffer
raise socket.timeout()
if isinstance(exc, SSLError) and 'timed out' in str(exc):
# Don't disconnect for ssl read time outs
# http://bugs.python.org/issue10272
self._read_buffer = read_frame_buffer + self._read_buffer
raise socket.timeout()
if exc.errno not in _UNAVAIL:
self.connected = False
raise
# frame-end octet must contain '\xce' value
if frame_end == 206:
return frame_type, channel, payload
else:
raise UnexpectedFrame(
f'Received frame_end {frame_end:#04x} while expecting 0xce')
def write(self, s):
try:
self._write(s)
except socket.timeout:
raise
except OSError as exc:
if exc.errno not in _UNAVAIL:
self.connected = False
raise
class SSLTransport(_AbstractTransport):
"""Transport that works over SSL.
PARAMETERS:
host: str
Broker address in format ``HOSTNAME:PORT``.
connect_timeout: int
Timeout of creating new connection.
ssl: bool|dict
parameters of TLS subsystem.
- when ``ssl`` is not dictionary, defaults of TLS are used
- otherwise:
- if ``ssl`` dictionary contains ``context`` key,
:attr:`~SSLTransport._wrap_context` is used for wrapping
socket. ``context`` is a dictionary passed to
:attr:`~SSLTransport._wrap_context` as context parameter.
All others items from ``ssl`` argument are passed as
``sslopts``.
- if ``ssl`` dictionary does not contain ``context`` key,
:attr:`~SSLTransport._wrap_socket_sni` is used for
wrapping socket. All items in ``ssl`` argument are
passed to :attr:`~SSLTransport._wrap_socket_sni` as
parameters.
kwargs:
additional arguments of
:class:`~amqp.transport._AbstractTransport` class
"""
def __init__(self, host, connect_timeout=None, ssl=None, **kwargs):
self.sslopts = ssl if isinstance(ssl, dict) else {}
self._read_buffer = EMPTY_BUFFER
super().__init__(
host, connect_timeout=connect_timeout, **kwargs)
__slots__ = (
"sslopts",
)
def _setup_transport(self):
"""Wrap the socket in an SSL object."""
self.sock = self._wrap_socket(self.sock, **self.sslopts)
# Explicitly set a timeout here to stop any hangs on handshake.
self.sock.settimeout(self.connect_timeout)
self.sock.do_handshake()
self._quick_recv = self.sock.read
def _wrap_socket(self, sock, context=None, **sslopts):
if context:
return self._wrap_context(sock, sslopts, **context)
return self._wrap_socket_sni(sock, **sslopts)
def _wrap_context(self, sock, sslopts, check_hostname=None, **ctx_options):
"""Wrap socket without SNI headers.
PARAMETERS:
sock: socket.socket
Socket to be wrapped.
sslopts: dict
Parameters of :attr:`ssl.SSLContext.wrap_socket`.
check_hostname
Whether to match the peer certs hostname. See
:attr:`ssl.SSLContext.check_hostname` for details.
ctx_options
Parameters of :attr:`ssl.create_default_context`.
"""
ctx = ssl.create_default_context(**ctx_options)
ctx.check_hostname = check_hostname
return ctx.wrap_socket(sock, **sslopts)
def _wrap_socket_sni(self, sock, keyfile=None, certfile=None,
server_side=False, cert_reqs=None,
ca_certs=None, do_handshake_on_connect=False,
suppress_ragged_eofs=True, server_hostname=None,
ciphers=None, ssl_version=None):
"""Socket wrap with SNI headers.
stdlib :attr:`ssl.SSLContext.wrap_socket` method augmented with support
for setting the server_hostname field required for SNI hostname header.
PARAMETERS:
sock: socket.socket
Socket to be wrapped.
keyfile: str
Path to the private key
certfile: str
Path to the certificate
server_side: bool
Identifies whether server-side or client-side
behavior is desired from this socket. See
:attr:`~ssl.SSLContext.wrap_socket` for details.
cert_reqs: ssl.VerifyMode
When set to other than :attr:`ssl.CERT_NONE`, peers certificate
is checked. Possible values are :attr:`ssl.CERT_NONE`,
:attr:`ssl.CERT_OPTIONAL` and :attr:`ssl.CERT_REQUIRED`.
ca_certs: str
Path to “certification authority” (CA) certificates
used to validate other peers certificates when ``cert_reqs``
is other than :attr:`ssl.CERT_NONE`.
do_handshake_on_connect: bool
Specifies whether to do the SSL
handshake automatically. See
:attr:`~ssl.SSLContext.wrap_socket` for details.
suppress_ragged_eofs (bool):
See :attr:`~ssl.SSLContext.wrap_socket` for details.
server_hostname: str
Specifies the hostname of the service which
we are connecting to. See :attr:`~ssl.SSLContext.wrap_socket`
for details.
ciphers: str
Available ciphers for sockets created with this
context. See :attr:`ssl.SSLContext.set_ciphers`
ssl_version:
Protocol of the SSL Context. The value is one of
``ssl.PROTOCOL_*`` constants.
"""
opts = {
'sock': sock,
'server_side': server_side,
'do_handshake_on_connect': do_handshake_on_connect,
'suppress_ragged_eofs': suppress_ragged_eofs,
'server_hostname': server_hostname,
}
if ssl_version is None:
ssl_version = (
ssl.PROTOCOL_TLS_SERVER
if server_side
else ssl.PROTOCOL_TLS_CLIENT
)
context = ssl.SSLContext(ssl_version)
if certfile is not None:
context.load_cert_chain(certfile, keyfile)
if ca_certs is not None:
context.load_verify_locations(ca_certs)
if ciphers is not None:
context.set_ciphers(ciphers)
# Set SNI headers if supported.
# Must set context.check_hostname before setting context.verify_mode
# to avoid setting context.verify_mode=ssl.CERT_NONE while
# context.check_hostname is still True (the default value in context
# if client-side) which results in the following exception:
# ValueError: Cannot set verify_mode to CERT_NONE when check_hostname
# is enabled.
try:
context.check_hostname = (
ssl.HAS_SNI and server_hostname is not None
)
except AttributeError:
pass # ask forgiveness not permission
# See note above re: ordering for context.check_hostname and
# context.verify_mode assignments.
if cert_reqs is not None:
context.verify_mode = cert_reqs
if ca_certs is None and context.verify_mode != ssl.CERT_NONE:
purpose = (
ssl.Purpose.CLIENT_AUTH
if server_side
else ssl.Purpose.SERVER_AUTH
)
context.load_default_certs(purpose)
sock = context.wrap_socket(**opts)
return sock
def _shutdown_transport(self):
"""Unwrap a SSL socket, so we can call shutdown()."""
if self.sock is not None:
self.sock = self.sock.unwrap()
def _read(self, n, initial=False,
_errnos=(errno.ENOENT, errno.EAGAIN, errno.EINTR)):
# According to SSL_read(3), it can at most return 16kb of data.
# Thus, we use an internal read buffer like TCPTransport._read
# to get the exact number of bytes wanted.
recv = self._quick_recv
rbuf = self._read_buffer
try:
while len(rbuf) < n:
try:
s = recv(n - len(rbuf)) # see note above
except OSError as exc:
# ssl.sock.read may cause ENOENT if the
# operation couldn't be performed (Issue celery#1414).
if exc.errno in _errnos:
if initial and self.raise_on_initial_eintr:
raise socket.timeout()
continue
raise
if not s:
raise OSError('Server unexpectedly closed connection')
rbuf += s
except: # noqa
self._read_buffer = rbuf
raise
result, self._read_buffer = rbuf[:n], rbuf[n:]
return result
def _write(self, s):
"""Write a string out to the SSL socket fully."""
write = self.sock.write
while s:
try:
n = write(s)
except ValueError:
# AG: sock._sslobj might become null in the meantime if the
# remote connection has hung up.
# In python 3.4, a ValueError is raised is self._sslobj is
# None.
n = 0
if not n:
raise OSError('Socket closed')
s = s[n:]
class TCPTransport(_AbstractTransport):
"""Transport that deals directly with TCP socket.
All parameters are :class:`~amqp.transport._AbstractTransport` class.
"""
def _setup_transport(self):
# Setup to _write() directly to the socket, and
# do our own buffered reads.
self._write = self.sock.sendall
self._read_buffer = EMPTY_BUFFER
self._quick_recv = self.sock.recv
def _read(self, n, initial=False, _errnos=(errno.EAGAIN, errno.EINTR)):
"""Read exactly n bytes from the socket."""
recv = self._quick_recv
rbuf = self._read_buffer
try:
while len(rbuf) < n:
try:
s = recv(n - len(rbuf))
except OSError as exc:
if exc.errno in _errnos:
if initial and self.raise_on_initial_eintr:
raise socket.timeout()
continue
raise
if not s:
raise OSError('Server unexpectedly closed connection')
rbuf += s
except: # noqa
self._read_buffer = rbuf
raise
result, self._read_buffer = rbuf[:n], rbuf[n:]
return result
def Transport(host, connect_timeout=None, ssl=False, **kwargs):
"""Create transport.
Given a few parameters from the Connection constructor,
select and create a subclass of
:class:`~amqp.transport._AbstractTransport`.
PARAMETERS:
host: str
Broker address in format ``HOSTNAME:PORT``.
connect_timeout: int
Timeout of creating new connection.
ssl: bool|dict
If set, :class:`~amqp.transport.SSLTransport` is used
and ``ssl`` parameter is passed to it. Otherwise
:class:`~amqp.transport.TCPTransport` is used.
kwargs:
additional arguments of :class:`~amqp.transport._AbstractTransport`
class
"""
transport = SSLTransport if ssl else TCPTransport
return transport(host, connect_timeout=connect_timeout, ssl=ssl, **kwargs)

View File

@@ -0,0 +1,64 @@
"""Compatibility utilities."""
import logging
from logging import NullHandler
# enables celery 3.1.23 to start again
from vine import promise # noqa
from vine.utils import wraps
try:
import fcntl
except ImportError: # pragma: no cover
fcntl = None # noqa
def set_cloexec(fd, cloexec):
"""Set flag to close fd after exec."""
if fcntl is None:
return
try:
FD_CLOEXEC = fcntl.FD_CLOEXEC
except AttributeError:
raise NotImplementedError(
'close-on-exec flag not supported on this platform',
)
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
if cloexec:
flags |= FD_CLOEXEC
else:
flags &= ~FD_CLOEXEC
return fcntl.fcntl(fd, fcntl.F_SETFD, flags)
def coro(gen):
"""Decorator to mark generator as a co-routine."""
@wraps(gen)
def _boot(*args, **kwargs):
co = gen(*args, **kwargs)
next(co)
return co
return _boot
def str_to_bytes(s):
"""Convert str to bytes."""
if isinstance(s, str):
return s.encode('utf-8', 'surrogatepass')
return s
def bytes_to_str(s):
"""Convert bytes to str."""
if isinstance(s, bytes):
return s.decode('utf-8', 'surrogatepass')
return s
def get_logger(logger):
"""Get logger by name."""
if isinstance(logger, str):
logger = logging.getLogger(logger)
if not logger.handlers:
logger.addHandler(NullHandler())
return logger