From 6312e177f5712b0e11e77f14ccfa7c50334713f8 Mon Sep 17 00:00:00 2001
From: Chris Blanchard <cablanchard@gmail.com>
Date: Thu, 20 Aug 2015 13:56:07 +0100
Subject: [PATCH] Prevent crawls of large data files

---
 public/robots.txt | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/public/robots.txt b/public/robots.txt
index 085187f..8958fb4 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,5 +1,7 @@
 # See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
 #
 # To ban all spiders from the entire site uncomment the next two lines:
-# User-Agent: *
-# Disallow: /
+User-Agent: *
+Disallow: /data_files
+Disallow: /directories
+Disallow: /movies
\ No newline at end of file