Compact (archive) Old Log Files In Python

I'm using standart logger library in Python. There are RotatingFileHandler, that can rotate log files dayily, for example. But it just renames them. Will be great, if it can not o

Solution 1:

I think your best option is to extend RotatingFileHandler something like this (not tested):

import os
from logging.handlers import RotatingFileHandler


COMPRESSION_SUPPORTED = {}

try:
   import gzip
   COMPRESSION_SUPPORTED['gz'] = gzip
except ImportError:
   passtry:
   import zipfile
   COMPRESSION_SUPPORTED['zip'] = zipfile
except ImportError:
   passclassNewRotatingFileHandler(RotatingFileHandler):

     def__init__(self, *args, **kws):
         compress_mode = kws.pop('compress_mode')

         try:
             self.compress_cls = COMPRESSION_SUPPORTED[compress_mode]
         except KeyError:
             raise ValueError('"%s" compression method not supported.' % compress_mode)

         super(NewRotatingFileHandler, self).__init__(self, *args, **kws)

     defdoRollover(self):
         super(NewRotatingFileHandler, self).doRollover()

         # Compress the old log.
         old_log = self.baseFilename + ".1"withopen(old_log) as log:
             with self.compress_cls.open(old_log + '.gz', 'wb') as comp_log:
                 comp_log.writelines(log)

         os.remove(old_log)

Solution 2:

The accepted answer will archive only 1 file-(basefile.log.1) .The other files are not archived. This code will archive all the log files other than the base file.

import os
import gzip
import logging.handlers

classNewRotatingFileHandler(logging.handlers.RotatingFileHandler):
    def__init__(self, filename, **kws):
        backupCount = kws.get('backupCount', 0)
        self.backup_count = backupCount
        logging.handlers.RotatingFileHandler.__init__(self, filename, **kws)

    defdoArchive(self, old_log):
        withopen(old_log) as log:
            with gzip.open(old_log + '.gz', 'wb') as comp_log:
                comp_log.writelines(log)
       os.remove(old_log)

   defdoRollover(self):
      if self.stream:
          self.stream.close()
          self.stream = Noneif self.backup_count > 0:
          for i inrange(self.backup_count - 1, 0, -1):
              sfn = "%s.%d.gz" % (self.baseFilename, i)
              dfn = "%s.%d.gz" % (self.baseFilename, i + 1)
              if os.path.exists(sfn):
                  if os.path.exists(dfn):
                      os.remove(dfn)
                  os.rename(sfn, dfn)
      dfn = self.baseFilename + ".1"if os.path.exists(dfn):
          os.remove(dfn)
      if os.path.exists(self.baseFilename):
          os.rename(self.baseFilename, dfn)
          self.doArchive(dfn)
      ifnot self.delay:
          self.stream = self._open()

Solution 3:

You can automatically write bz2 compressed log files by initializing the RotatingFileHandler with encoding='bz2-codec':

import logging
import logging.handlers as handlers

if __name__=='__main__':
    log_filename='log_rotate.bz2'
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    handler = handlers.RotatingFileHandler(
        log_filename, maxBytes=20, backupCount=5, encoding='bz2-codec')
    logger.addHandler(handler)
    for i inrange(20):
        logger.debug('i = %d' % i)

PS. Python3 removed 'bz2-codec' from the set of valid encodings, so this solution is specific to Python2.