Commit d235bb39 authored by Kevin Atkinson's avatar Kevin Atkinson

Add robots.txt file for www.emulab.net.

parent 2a15da9b
......@@ -57,6 +57,9 @@ FILES += $(SRCDIR)/.htaccess $(SRCDIR)/error.shtml \
$(SRCDIR)/ssh-mime-winxp.exe $(SRCDIR)/rdp-mime-winxp.exe \
$(SRCDIR)/emulab-mime-winxp.zip $(SRCDIR)/emulab-mime-winxp.reg \
$(SRCDIR)/emusearch.xml
ifeq ($(TBROOT),/usr/testbed)
FILES += robots.txt
endif
AUTOICONS = $(wildcard $(SRCDIR)/autostatus-icons/*.gif)
AUTOICONS += $(wildcard $(SRCDIR)/autostatus-icons/*.png)
......
User-Agent: *
# Disallow most php pages except a select few
Allow: /index.php3
Allow: /doc.php3
Allow: /docwrapper.php3
Allow: /doc/docwrapper.php3
# Google was rejecting URL's of the form
# http://www.emulab.net/docwrapper.php3?docname=FOO
# Let's see if this helps.
Allow: /docwrapper.php3?*$
Allow: /doc/docwrapper.php3?*$
Allow: /news.php3
Allow: /pubs.php3
Allow: /expubs.php
Allow: /software.php3
Allow: /people.php3
Allow: /gallery/gallery.php3
Allow: /explist.php3
Allow: /tutorial/mobilewireless.php3
# Last in case the reason google was rejecting docwrapper pages was
# due to order
Disallow: /*.php*$
# Disallow downloads for now
Disallow: /downloads
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment