root/apache_log/apache_log/bot_hosts.py

from IPy import IP
import re

BOT_HOSTS = (
    # Google search engine hosts
    r"64\.233\.183\.104",
    r"64\.233\.169\.104",
    r"66\.102\.9\.104",
    r"72\.14\.253\.104",
    r"209\.85\.135\.104",
    r"209\.85\.129\.104",
    r"(images|www)\.google\.[a-z]{2,3}",

    # Popular search engines
    r"www.altavista.com",
    r"[a-z]+.search.yahoo.com",
    r"search\.msn\.com", "search\.live\.com",

    # Other search engines
    r"vivisimo\.com",
    r"search[0-9]-[0-9].free.fr",
    r"search.ke.voila.fr",
)

BOT_IPS = (
    IP('82.99.30.0/25'),    # IM Teknik AB (?)
    IP('216.10.74.0/25'),  # ohloh.net
)

BOT_DOMAINS = [ re.compile(re.escape(regex)+'$') for regex in (
    # Search engine crawlers (domain as string)
    '.crawl.yahoo.net',   # Yahoo
    '.search.live.com',   # live.com (Microsoft)
    '.search.msn.com',    # search.msn.com
    '.googlebot.com',     # Google
    '.google.com',        # Google
    '.exabot.com',        # Exabot
    'crawler.bloglines.com',
    '.cuill.com',
    '.picsearch.com',
    '.fastsearch.net',
    '.rambler.ru',
    '.cazoodle.com',
)] + [ re.compile(regex) for regex in (
    # Domain regex
    r'^crawler[0-9]{2}\.kaist\.ac\.kr$',
)]