Adding handler base and logrecord stuff

mathom · mathom · commit c491d6820afb · 2014-10-15T13:19:50.000-05:00
diff --git a/stackify/__init__.py b/stackify/__init__.py
@@ -11,8 +11,12 @@
 
 MAX_BATCH = 100
 
+QUEUE_SIZE = 1000
 
 import logging
+import inspect
+
+DEFAULT_LEVEL = logging.ERROR
 
 LOGGING_LEVELS = {
     logging.CRITICAL: 'CRITICAL',
@@ -31,3 +35,28 @@
 from stackify.application import ApiConfiguration
 from stackify.http import HTTPClient
 
+from stackify.handler import StackifyHandler
+
+
+def getLogger(name=None, **kwargs):
+    if not name:
+        try:
+            frame = inspect.stack()[1]
+            module = inspect.getmodule(frame[0])
+            name = module.__name__
+        except IndexError:
+            name = 'stackify-python-unknown'
+
+    logger = logging.getLogger(name)
+
+    if not [isinstance(x, StackifyHandler) for x in logger.handlers]:
+        internal_log.debug('Creating handler for logger {0}'.format(name))
+        handler = StackifyHandler(**kwargs)
+        logger.addHandler(handler)
+
+        if logger.getEffectiveLevel() == logging.NOTSET:
+            logger.setLevel(DEFAULT_LEVEL)
+
+    return logger
+
+
diff --git a/stackify/error.py b/stackify/error.py
@@ -15,9 +15,14 @@ def __init__(self):
         self.StackTrace = [] # array of TraceFrames
         self.InnerError = None # cause?
 
-    def load_stack(self):
-        type_, value, tb = sys.exc_info()
+    def load_stack(self, exc_info=None):
+        if not exc_info:
+            type_, value, tb = sys.exc_info()
+        else:
+            type_, value, tb = exc_info
+
         stacks = traceback.extract_tb(tb)
+
         self.ErrorType = type_.__name__
         self.Message = str(value)
         self.SourceMethod = stacks[-1][2]
@@ -54,13 +59,18 @@ def __init__(self):
 class StackifyError(JSONObject):
     def __init__(self):
         self.EnvironmentDetail = None # environment detail object
-        self.OccurredEpochMillis = int(time.time() * 1000)
+        self.OccurredEpochMillis = None
         self.Error = None # ErrorItem object
         self.WebRequestDetail = None # WebRequestDetail object
         self.CustomerName = None
         self.UserName = None
 
-    def load_exception(self):
+    def load_exception(self, exc_info=None):
         self.Error = ErrorItem()
-        self.Error.load_stack()
+        self.Error.load_stack(exc_info)
+
+    def from_record(self, record):
+        self.load_exception(record.exc_info)
+        self.OccurredEpochMillis = int(record.created * 1000)
+
 
diff --git a/stackify/handler.py b/stackify/handler.py
@@ -0,0 +1,64 @@
+import logging
+import threading
+
+try:
+    from logging.handlers import QueueHandler, QueueListener
+except:
+    from stackify.handler_backport import QueueHandler, QueueListener
+
+try:
+    import Queue as queue
+except ImportError:
+    import queue
+
+from stackify import QUEUE_SIZE
+from stackify.log import LogMsg
+from stackify.error import ErrorItem
+from stackify.http import HTTPClient
+
+
+
+class StackifyHandler(QueueHandler):
+    '''
+    A handler class to format and queue log messages for later
+    transmission to Stackify servers.
+    '''
+
+    def __init__(self, queue_=None):
+        if queue_ is None:
+            queue_ = queue.Queue(QUEUE_SIZE)
+
+        super(StackifyHandler, self).__init__(queue_)
+
+        self.listener = StackifyListener(queue_)
+
+    def enqueue(self, record):
+        '''
+        Put a new record on the queue. If it's full, evict an item.
+        '''
+        try:
+            self.queue.put_nowait(record)
+            logger = logging.getLogger(__name__)
+            logger.debug('put record ' + record.toJSON())
+        except queue.Full:
+            logger = logging.getLogger(__name__)
+            logger.warn('StackifyHandler queue is full, evicting oldest record')
+            self.queue.get_nowait()
+            self.queue.put_nowait(record)
+
+    def prepare(self, record):
+        print(record.__dict__)
+        msg = LogMsg()
+        msg.from_record(record)
+
+        return msg
+
+
+class StackifyListener(QueueListener):
+    '''
+    A listener to read queued log messages and send them to Stackify.
+    '''
+
+    def __init__(self, queue_):
+        super(StackifyListener, self).__init__(queue_)
+
diff --git a/stackify/handler_backport.py b/stackify/handler_backport.py
@@ -0,0 +1,200 @@
+# Copyright 2001-2013 by Vinay Sajip. All Rights Reserved.
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose and without fee is hereby granted,
+# provided that the above copyright notice appear in all copies and that
+# both that copyright notice and this permission notice appear in
+# supporting documentation, and that the name of Vinay Sajip
+# not be used in advertising or publicity pertaining to distribution
+# of the software without specific, written prior permission.
+# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
+# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
+# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
+# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+import logging
+import threading
+
+
+class QueueHandler(logging.Handler):
+    """
+    This handler sends events to a queue. Typically, it would be used together
+    with a multiprocessing Queue to centralise logging to file in one process
+    (in a multi-process application), so as to avoid file write contention
+    between processes.
+
+    This code is new in Python 3.2, but this class can be copy pasted into
+    user code for use with earlier Python versions.
+    """
+
+    def __init__(self, queue):
+        """
+        Initialise an instance, using the passed queue.
+        """
+        logging.Handler.__init__(self)
+        self.queue = queue
+
+    def enqueue(self, record):
+        """
+        Enqueue a record.
+
+        The base implementation uses put_nowait. You may want to override
+        this method if you want to use blocking, timeouts or custom queue
+        implementations.
+        """
+        self.queue.put_nowait(record)
+
+    def prepare(self, record):
+        """
+        Prepares a record for queuing. The object returned by this method is
+        enqueued.
+
+        The base implementation formats the record to merge the message
+        and arguments, and removes unpickleable items from the record
+        in-place.
+
+        You might want to override this method if you want to convert
+        the record to a dict or JSON string, or send a modified copy
+        of the record while leaving the original intact.
+        """
+        # The format operation gets traceback text into record.exc_text
+        # (if there's exception data), and also puts the message into
+        # record.message. We can then use this to replace the original
+        # msg + args, as these might be unpickleable. We also zap the
+        # exc_info attribute, as it's no longer needed and, if not None,
+        # will typically not be pickleable.
+        self.format(record)
+        record.msg = record.message
+        record.args = None
+        record.exc_info = None
+        return record
+
+    def emit(self, record):
+        """
+        Emit a record.
+
+        Writes the LogRecord to the queue, preparing it for pickling first.
+        """
+        try:
+            self.enqueue(self.prepare(record))
+        except Exception:
+            self.handleError(record)
+
+
+class QueueListener(object):
+    """
+    This class implements an internal threaded listener which watches for
+    LogRecords being added to a queue, removes them and passes them to a
+    list of handlers for processing.
+    """
+    _sentinel = None
+
+    def __init__(self, queue, *handlers):
+        """
+        Initialise an instance with the specified queue and
+        handlers.
+        """
+        self.queue = queue
+        self.handlers = handlers
+        self._stop = threading.Event()
+        self._thread = None
+
+    def dequeue(self, block):
+        """
+        Dequeue a record and return it, optionally blocking.
+
+        The base implementation uses get. You may want to override this method
+        if you want to use timeouts or work with custom queue implementations.
+        """
+        return self.queue.get(block)
+
+    def start(self):
+        """
+        Start the listener.
+
+        This starts up a background thread to monitor the queue for
+        LogRecords to process.
+        """
+        self._thread = t = threading.Thread(target=self._monitor)
+        t.setDaemon(True)
+        t.start()
+
+    def prepare(self , record):
+        """
+        Prepare a record for handling.
+
+        This method just returns the passed-in record. You may want to
+        override this method if you need to do any custom marshalling or
+        manipulation of the record before passing it to the handlers.
+        """
+        return record
+
+    def handle(self, record):
+        """
+        Handle a record.
+
+        This just loops through the handlers offering them the record
+        to handle.
+        """
+        record = self.prepare(record)
+        for handler in self.handlers:
+            handler.handle(record)
+
+    def _monitor(self):
+        """
+        Monitor the queue for records, and ask the handler
+        to deal with them.
+
+        This method runs on a separate, internal thread.
+        The thread will terminate if it sees a sentinel object in the queue.
+        """
+        q = self.queue
+        has_task_done = hasattr(q, 'task_done')
+        while not self._stop.isSet():
+            try:
+                record = self.dequeue(True)
+                if record is self._sentinel:
+                    break
+                self.handle(record)
+                if has_task_done:
+                    q.task_done()
+            except queue.Empty:
+                pass
+        # There might still be records in the queue.
+        while True:
+            try:
+                record = self.dequeue(False)
+                if record is self._sentinel:
+                    break
+                self.handle(record)
+                if has_task_done:
+                    q.task_done()
+            except queue.Empty:
+                break
+
+    def enqueue_sentinel(self):
+        """
+        This is used to enqueue the sentinel record.
+
+        The base implementation uses put_nowait. You may want to override this
+        method if you want to use timeouts or work with custom queue
+        implementations.
+        """
+        self.queue.put_nowait(self._sentinel)
+
+    def stop(self):
+        """
+        Stop the listener.
+
+        This asks the thread to terminate, and then waits for it to do so.
+        Note that if you don't call this before your application exits, there
+        may be some records still left on the queue, which won't be processed.
+        """
+        self._stop.set()
+        self.enqueue_sentinel()
+        self._thread.join()
+        self._thread = None
+
diff --git a/stackify/log.py b/stackify/log.py
