root/apache_log/apache_log/parser.py

#!/usr/bin/python
# -*- coding: iso-8859-1 -*-
#
# Python script to parse Apache log file
# Creation: 21 septembre 2005
# Author: Victor Stinner

import re
from time import time
from sys import exit
from urllib import unquote
from datetime import datetime
from logging import warning
from IPy import IP
from apache_log.user_agents import userAgentTag
from apache_log.tags import DEFAULT_TAG
from sys import stdin, stderr

class ApacheRequest:
    def __init__(self, parser, raw_string, data):
        self.parser = parser
        self.raw_string = raw_string
        self.tag = DEFAULT_TAG

        self.method = data.get('method')           # str
        self.origin = data.get('origin')           # str
        self.size = data.get('size')               # int

        if 'date' in data:
            stamp = data['date']
            if "+" in stamp:
                stamp = stamp[:-6]
            self.timestamp = datetime.strptime(stamp, self.parser.date_format)
        else:
            self.timestamp = None

        if 'url' in data:
            self.url = self.unquote(data['url'])
        else:
            self.url = None
        if 'host' in data:
            self.host = IP(data['host'])
        else:
            self.host = None
        if 'referrer' in data:
            self.referrer = self.unquote(data['referrer'])
        else:
            self.referrer = None
        self.user = data.get('user', None)
        if self.user == '-':
            self.user = None
        if 'code' in data:
            self.code = int(data['code'])
        else:
            self.code = None
        if 'user_agent' in data:
            self.user_agent = self.unquote(data['user_agent'])
#            tag = userAgentTag(parser.error, self.user_agent)
#            if tag:
#                self.tag = tag
        else:
            self.user_agent = None

    def unquote(self, url):
        url = url.replace(r'\"', '"')
        url = unquote(url)
        if url != "-":
            return url
        else:
            return None

class ApacheLogParser:
    _REGEX_STRING = r'(?:[^\"]|\")*'

    # Referrer URL -> referrer="http://..."
    REGEX_REFERRER = r'"(?P<referrer>%s)"' % _REGEX_STRING

    #Â User agent: "Mozilla ..." -> user_agent="Mozilla ..."
    REGEX_USER_AGENT = r'"(?P<user_agent>%s)"' % _REGEX_STRING

    # HTTP request:
    # "GET /index.html HTTP/1.1" -> method="GET", url="/index.html", http_version="1.1"
    # "GET /" => method="GET", url="/", http_version is not set
    HTTP_METHODS = ("POST", "GET", "HEAD")
    WEBDAV_METHODS = (
        "PUT", "COPY", "MERGE", "DELETE", "CHECKOUT",
        "PROPFIND", "PROPPATCH", "CONNECT",
        "MKACTIVITY", "MKCOL",
        "REPORT", "OPTIONS",
    )
    METHODS = HTTP_METHODS + WEBDAV_METHODS
    REGEX_REQUEST = r'"(?P<method>%s) (?P<url>[^ ]+)(?: HTTP/(?P<http_version>1\.[01]))?"' % '|'.join(METHODS)

    # Date -> date="..."
    REGEX_DATE = r"\[(?P<date>[^]]+)\]"

    # Username: "haypo", "-" -> user="haypo"
    REGEX_USER = r'(?P<user>(?:[a-z]+|-|""))'

    # Origin -> origin="..."
    REGEX_ORIGIN = r"(?P<origin>[^ ]+)"

    # Host -> host="localhost"
    REGEX_HOST = r"(?P<host>[^ ]+)"

    # Code, Size: "200 -" --> code="200", size="-"
    REGEX_ANSWER = r"(?P<code>[0-9]{3}) (?P<size>[0-9]+|-)"

    MAX_ERROR = 20

    def __init__(self, syntax):
        def replace(regs):
            name = regs.group(1)
            return getattr(self, "REGEX_" + name.upper())
        regex = "^%s$" % re.sub("{([^}]+)}", replace, syntax)
        self.date_format = '%d/%b/%Y:%H:%M:%S'
        self.filename = None
        self.line_number = None
        self._regex = re.compile(regex)
        self.skip_error = True
        self.nb_error = 0
        self.first_datetime = None
        self.last_datetime = None

    def _processRequest(self, request):
        if request.timestamp:
            stamp = request.timestamp
            if self.first_datetime:
                self.first_datetime = min(self.first_datetime, stamp)
                self.last_datetime = max(self.last_datetime, stamp)
            else:
                self.first_datetime = stamp
                self.last_datetime = stamp
        self.processRequest(request)

    def processRequest(self, request):
        raise NotImplemented()

    def error(self, message):
        message = "%s:%s: %s" % (self.filename, self.line_number, message)
        if not self.skip_error or self.MAX_ERROR <= self.nb_error:
            raise SyntaxError(message)
        else:
            warning(message)
        self.nb_error += 1

    def parseLine(self, line):
        try:
            match = self._regex.search(line)
            if not match:
                self.error("Unable to parse: %s" % line)
                return
            request = ApacheRequest(self, line, match.groupdict())
            self._processRequest(request)
        except KeyboardInterrupt:
            print "Interrupt line: %s" % line
            raise

    def parseFile(self, filename):
        self.filename = filename

        self.line_number = 0
        try:
            if filename == "-":
                print >>stderr, "Read stdin..."
                input = stdin
                input.flush()
            else:
                print >>stderr, "Load file %s ..." % filename
                input = open(self.filename, "r")
            while True:
                line = input.readline()
                if not line:
                    break
                self.line_number += 1
                line = line.rstrip()
                if not line:
                    continue
                self.parseLine(line)
            print >>stderr, "File %s parsed (%s lines)." % (self.filename, self.line_number)
        except KeyboardInterrupt:
            print >>stderr, "Load interrupted (CTRL+C)."
            exit(1)

class HitCounter(dict):
    def hit(self, key):
        if key in self:
            self[key] += 1
        else:
            self[key] = 1

def acceptAll(request):
    return None

class ApacheLogParser_Stat(ApacheLogParser):
    def __init__(self, syntax, host):
        ApacheLogParser.__init__(self, syntax)
        self.host = host
        self.regex_host = re.compile("^"+host+"(.*)$")
        self.stat_page = HitCounter()
        self.stat_host = HitCounter()
        self.stat_referrer = HitCounter()
        self.ignore_handler = acceptAll
        self.ignore_reasons = HitCounter()
        self.raw_total_hits = 0
        self.total_hits = 0
        self.tag_hits = HitCounter()
        self.raw_hits = HitCounter()

    def hit(self, attr, key):
        if key in attr:
            attr[key] = attr[key] + 1
        else:
            attr[key] = 1

    def ignoreErrorCode(self, code):
        if 200 <= code <= 299:
            return False
        if 300 <= code <= 599:
            return False
        raise SyntaxError("Unknown HTTP code: %s" % code)

    def processRequest(self, request):
        # Update raw statistics
        self.raw_total_hits += 1
        self.raw_hits.hit(request.tag)

        # Clean url (remove hostname)
        match = self.regex_host.match(request.url)
        if match:
            request.url = "/"+m.group(1)

        # Ignore this hit?
        reason = self.ignore_handler(request)
        if reason:
            self.ignore_reasons.hit(reason)
            return

        # Skip redirection and error pages
        if self.ignoreErrorCode(request.code):
            return

        self.stat_page.hit(request.url)
        self.stat_host.hit(request.host)
        if request.referrer:
            self.stat_referrer.hit(request.referrer)

        # Accept request
        self.total_hits += 1
        self.tag_hits.hit(request.tag)

    def _sort_value(self, val1, val2):
        if val1 > val2:
            return -1
        elif val1 == val2:
            return 0
        else:
            return 1

    def _sort_stat_referrer(self, key1, key2):
        return self._sort_value(self.stat_referrer[key1],  self.stat_referrer[key2])

    def _sort_stat_page(self, key1, key2):
        return self._sort_value(self.stat_page[key1],  self.stat_page[key2])

    def _sort_stat_host(self, key1, key2):
        return self._sort_value(self.stat_host[key1],  self.stat_host[key2])

    def getTopReferrer(self, max=30):
        return self.getTop(self.stat_referrer, self._sort_stat_referrer, max)

    def getTopPage(self, max=30):
        return self.getTop(self.stat_page, self._sort_stat_page, max)

    def getTopHost(self, max=30):
        return self.getTop(self.stat_host, self._sort_stat_host, max)

    def getTop(self, attr, sort_func, max=30):
        keys = attr.keys()
        keys.sort(sort_func)
        top = []
        cpt = 1
        for key in keys:
            top.append( (attr[key], key,) )
            cpt += 1
            if max <= cpt:
                break
        return top