robot-id: freecrawl
robot-name: Freecrawl
robot-cover-url: http://euroseek.net/
robot-owner-name: Jesper Ekhall
robot-owner-email: ekhall@freeside.net
robot-status: active
robot-purpose: indexing
robot-type: standalone
robot-platform: unix
robot-availability: none
robot-exclusion: yes
robot-exclusion-useragent: Freecrawl
robot-noindex: no
robot-host: *.freeside.net
robot-from: yes
robot-useragent: Freecrawl
robot-language: c
robot-description: The Freecrawl robot is used to build a database
robot-environment: service

robot-id:           getbot
robot-name:         GetBot
robot-cover-url:    http://www.blacktop.com.zav/bots 
robot-details-url:
robot-owner-name:   Alex Zavatone
robot-owner-url:    http://www.blacktop.com/zav
robot-owner-email:  zav@macromedia.com
robot-status:
robot-purpose:      maintenance
robot-type:         standalone
robot-platform:
robot-availability:
robot-exclusion:    no.
robot-exclusion-useragent:
robot-noindex:
robot-host:
robot-from:         no
robot-useragent:    ???
robot-language:     Shockwave/Director.
robot-description:  GetBot's purpose is to index all the sites it can find that
	contain Shockwave movies.  It is the first bot or spider
	written in Shockwave.  The bot was originally written at
	Macromedia on a hungover Sunday as a proof of concept. -
	Alex Zavatone 3/29/96
robot-history:
robot-environment:
modified-date:      Fri Mar 29 20:06:12 1996.
modified-by:


robot-id: googlebot
robot-name: Googlebot
robot-cover-url: http://www.googlebot.com/ 
robot-details-url: http://www.googlebot.com/bot.html
robot-owner-name: Google Inc.
robot-owner-url: http://www.google.com/
robot-owner-email: googlebot@google.com 
robot-status: active
robot-purpose: indexing
robot-type: standalone 
robot-platform: Linux
robot-availability: none
robot-exclusion: yes
robot-exclusion-useragent: googlebot
robot-noindex: yes
robot-host: googlebot.com
robot-from: yes 
robot-useragent: Googlebot/2.X (+http://www.googlebot.com/bot.html)
robot-language: c++
robot-description: Google's crawler
robot-history: Developed by Google Inc
robot-environment: commercial
modified-date: Thu Mar 29 21:00:07 PST 2001
modified-by: googlebot@google.com

robot-id:           htmlgobble
robot-name:         HTMLgobble
robot-cover-url:    
robot-details-url:
robot-owner-name:   Andreas Ley
robot-owner-url:    
robot-owner-email:  ley@rz.uni-karlsruhe.de
robot-status:       
robot-purpose:      mirror
robot-type:         
robot-platform:     
robot-availability: 
robot-exclusion:    
robot-exclusion-useragent:
robot-noindex:      no
robot-host:         tp70.rz.uni-karlsruhe.de
robot-from:         yes
robot-useragent:    HTMLgobble v2.2
robot-language:     
robot-description:  A mirroring robot. Configured to stay within a directory,
	sleeps between requests, and the next version will use HEAD
	to check if the entire document needs to be
	retrieved
robot-history:      
robot-environment:
modified-date:      
modified-by:

robot-id: Ilse
robot-name: Ingrid
robot-cover-url:
robot-details-url:
robot-owner-name: Ilse c.v.
robot-owner-url: http://www.ilse.nl/
robot-owner-email: ilse@ilse.nl
robot-status: Running
robot-purpose: Indexing
robot-type: Web Indexer
robot-platform: UNIX
robot-availability: Commercial as part of search engine package
robot-exclusion: Yes
robot-exclusion-useragent: INGRID/0.1
robot-noindex: Yes
robot-host: bart.ilse.nl
robot-from: Yes
robot-useragent: INGRID/0.1
robot-language: C
robot-description:  
robot-history:
robot-environment:
modified-date: 06/13/1997
modified-by: Ilse