root/apache_log/apache_log/user_agents.py

from apache_log.tags import (TAG_CRAWLER, TAG_DOWNLOADER, TAG_BOT, TAG_RSS,
    TAG_SUBVERSION, TAG_BROWSER, TAG_EMPTY, TAG_RANDOM)
import re
from apache_log.mozilla import MOZILLA_PREFIX_REGEX, parseMozillaUserAgent

CRAWLER_REGEX = re.compile('^(%s)' % '|'.join((
    "msnbot/",                   # http://www.live.com/
    "Speedy Spider",             # http://www.entireweb.com/
    "Baiduspider",               # http://www.baidu.com/
    "msnbot-media/",             # http://www.live.com/
    "Yeti/",
    "shelob v1",
    "lwp-trivial/",
    "LWP::Simple/",
    "OrangeSpider",
    "Yahoo-MMCrawler/",          # http://www.yahoo.com/
    "holmes/",
    "Googlebot-(Image|Video)",   # http://www.google.com/
    "Krugle/",                   # http://www.krugle.com/
    "FuseBulb.Com",              # http://www.fusebulb.com/
    'FAST MetaWeb Crawler',      # http://www.fastsearch.com/
    'KDDI-CA23 UP.Browser/',
    'Y!J-SRD/',
    'FAST Enterprise Crawler',   # used by Virk.dk - udvikling (thomas.bentzen@capgemini.com)
    'Nokia6682/2\.0 \(3\.01\.1\) SymbianOS/8\.0 Series60/2\.6 Profile/MIDP-2\.0 configuration/CLDC-1\.1 UP\.Link/6\.3\.0\.0\.0 \(compatible;YahooSeeker/',
    'WebImages 0\.3',            # http://herbert.groot.jebbink.nl/?app=WebImages
    'webcollage/',
    "gsa-crawler ",              # Enterprise; M2-PE3CGVABCA2AB; dpeterka@operative.com
    "Pete-Spider Light/1\.0",
    "MSRBOT ",                   # http://research.microsoft.com/research/sv/msrbot/
    "TMCrawler$",
    "appie 1\.1 ",               # http://www.walhello.com
    'Wells Search II$',
    'NG/2\.0$',
    'LiteFinder/',               # http://www.litefinder.net/about.html
    'Nokia6230i/\. FAST Crawler$',
    'Bookdog/',
    'A WinHTTP Example Program',
    'wadaino.jp-crawler ',       # http://wadaino.jp/
    'Yandex/',
    'Teemer ',                   # http://www.netseer.com/crawler.html
    'MSNBOT_Mobile ',
    'Trailfire-bot/',            # http://trailfire.com
    'Wouah/',                    # http://www.wouah.eu
    'travel-search',
)))

BOT_REGEX = re.compile('^(%s)' % '|'.join((
    "freshmeat.net URI validator", # http://freshmeat.net/
    "PortUrlChecker/",
    "Debian uscan",              # http://dehs.alioth.debian.org/uscan.html
    "VadixBot",
    "NutchCVS/",
    "larbin",
    "ia_archiver",
    "CazoodleBot/",
    "woriobot",
    "Attributor.comBot",
    "PHP version tracker",
    "Gigabot/",
    "bot/",
    "LargeSmall Crawler",
    'StackRambler/',
    "SurveyBot/",                # http://www.whois.sc/
    "rbot http util",            # http://linuxbrit.co.uk/rbot/
    'LinkChecker/',              # http://linkchecker.sourceforge.net/
    'Gaisbot/',                  # http://gais.cs.ccu.edu.tw/robot.php
    'HMSE_Robot$',
    'Slurpy Verifier/',
    'psbot/',                    # http://www.picsearch.com/bot.html
    'disco/Nutch-',              # imagine@gmail.com ; nedrocks@gmail.com
    'Trailfire/',                # http://lucene.apache.org/nutch/bot.html
    'GurujiBot/',                # http://www.guruji.com/en/WebmasterFAQ.html
    'Mozilla/4\.0 compatible FurlBot/Furl Search 2\.0 \(FurlBot;', # http://www.furl.net
    'Lsearch/sondeur$',
    'MJ12bot/',                  # http://majestic12.co.uk/bot.php
    'Microsoft URL Control',
    'Portsurvey/',               # FreeBSD ports using libwww-perl
    'Mozilla/3\.0 \(compatible; Indy Library\)',
    'nrsbot/',                   # loopip.com/robot.html
    'WordPress/',
    'nicebot$',
)))

DOWNLOADER_REGEX = re.compile('^(%s)' % '|'.join((
    "FDM 2",
    "EasyDL/",
    "Wget/",
    "wget$",
    "libcurl-agent/",
)))

RSS_REGEX = re.compile('^(%s)' % '|'.join((
    "Bloglines",
    "Feedfetcher-Google",
    "RSSOwl/",
    "Akregator/",
    "Netvibes",
    "Vienna/",
    "AppleSyndication/",
    "Liferea/",
    'ELI/',                 # DAUM RSS Robot, http://ws.daum.net/aboutkr.html
    'FeedFetcher-Google',   # http://www.google.com/feedfetcher.html
    'AideRSS/',             # aiderss.com
    'Feedreader ',
)))

SUBVERSION_REGEX = re.compile(r"^(SVN/|SVNKit )")

BROWSER_REGEX = re.compile("^(%s)" % "|".join((
    "Opera/",
    "Avant Browser",
    "w3m/",
    "Lynx/",
    "Netscape",
    "DoCoMo/",
    "CFNetwork/",
    "Nokia6820/",
    'Python-urllib/',
    "Sleipnir/",
    "Jakarta Commons-HttpClient",
    "libwww-perl/",
    "Apache",
    'DataCha0s/',
    'voyager/',
    "Java/",
    'Portal Manager',
    'compatible$',
    'Mozilla/[457]\.0$',
    'mozilla 4\.0$',
    'ELinks',
    'Links ',
    'POE-Component-Client-HTTP/',
    'Mozilla/0\.91 Beta \(Windows\)$',
    'Mozilla/3\.0 \(compatible\)$',
    'Mozilla/3\.01 \(compatible;\)$',
    'Mozilla/4\.5 \[en\] \(Win98; I\)$',   # Netscape 4.5 on Windows 98
    'Mozilla/4\.5 \(compatible; HTTrack 3\.0x; Windows 98\)$',
    'Mozilla/4\.8 \[en\] \(X11; U; Linux 2\.4\.20-8 i686\)$',
    'T-Mobile Dash Mozilla/4\.0',
    'Kazehakase/',
    'HTC_P3300 Mozilla/',
)))

RANDOM_REGEX = re.compile('^(%s)$' % '|'.join((
    "cpohlltsgtodxkdggpOh8x",
    "osxsl cpwljvhyroXesvgnhmwjucblj evv",
    'Prtmaldw hujcqopg yfklsatx',
    'hahahhahhaaahaha',
    'Rvxleb mwyxfnhb wybm',
    'Sagd jqnvxh qpvxu',
    'Cvbm fyihkar oxin',
    'vsfq2ynoqPxamsaiieaqPnwkq',
    'yisd0mlomenbacmlkkixiawf',
    'vpsqkb2epevbejvjkhrnbg  eot2ojj hiq',
    'tbms7dorslbgoupwptposdscdNbucimv vj7mpd',
    'Qcdgjyexi obchu bpdnhrso',
    'Exdcjqp knudlw ralzxbgmu',
)))

REGEX_TAGS = (
    (CRAWLER_REGEX, TAG_CRAWLER),
    (DOWNLOADER_REGEX, TAG_DOWNLOADER),
    (BOT_REGEX, TAG_BOT),
    (RSS_REGEX, TAG_RSS),
    (SUBVERSION_REGEX, TAG_SUBVERSION),
    (BROWSER_REGEX, TAG_BROWSER),
    (RANDOM_REGEX, TAG_RANDOM),
)

def userAgentTag(error, text):
    for regex, tag in REGEX_TAGS:
        if regex.search(text):
            return TAG_CRAWLER

    # Empty user agent: '-'
    if not text or text == "-":
        return TAG_EMPTY

    # Parse Mozilla user agent
    match = MOZILLA_PREFIX_REGEX.match(text)
    if match:
        return parseMozillaUserAgent(error, match.group(1))

    error("Unable to parse user agent: %r" % text)
    return None