From b93077c935ac0675e804313603e357309814acf4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 9 Nov 2016 14:06:01 +0000 Subject: [PATCH] Ignore robots.txt --- gendoc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gendoc.sh b/gendoc.sh index d088e12..679308a 100755 --- a/gendoc.sh +++ b/gendoc.sh @@ -20,7 +20,7 @@ done # Scrape the pkg directory for the API docs. Scrap lib for the CSS/JS. Ignore everything else. # The output is dumped to the directory "localhost:6060". -wget -r -m -k -E -p --include-directories="/pkg,/lib" --exclude-directories="*" http://localhost:6060/pkg/github.com/matrix-org/go-neb/ +wget -r -m -k -E -p -erobots=off --include-directories="/pkg,/lib" --exclude-directories="*" http://localhost:6060/pkg/github.com/matrix-org/go-neb/ # Stop the godoc server kill -9 $DOC_PID