#
# robots.txt for http://www.w3.org/
#
# $Id: robots.txt,v 1.34 2005/08/17 14:00:11 ted Exp $
#

# For use by search.w3.org
User-agent: W3C-gsa
Disallow: /Out-Of-Date

# W3C Link checker
User-agent: W3C-checklink
Disallow:

# exclude some access-controlled areas
User-agent: *
Disallow: /2004/ontaria/basic
Disallow: /Team
Disallow: /Project
Disallow: /Systems
Disallow: /Web
Disallow: /History
Disallow: /Out-Of-Date
Disallow: /2002/02/mid
Disallow: /mid/
Disallow: /People/all/
Disallow: /2003/03/Translations/byLanguage
Disallow: /2003/03/Translations/byTechnology
#Disallow: /2001/07/pubrules-checker
#shouldnt get transparent proxies but will ml links of things like pubrules
Disallow: /2000/06/webdata/xslt
Disallow: /2000/09/webdata/xslt