Changeset 391 for python/trunk/Lib/test/test_robotparser.py

Timestamp:

Mar 19, 2014, 11:31:01 PM (11 years ago)

Author:

dmik

Message:

python: Merge vendor 2.7.6 to trunk.

Location:

python/trunk

Files:

• . (modified) (1 prop)
• Lib/test/test_robotparser.py (modified) (3 diffs)

python/trunk/Lib/test/test_robotparser.py

@@ -1 +1 @@
 import unittest, StringIO, robotparser
 from test import test_support
+from urllib2 import urlopen, HTTPError
 
 class RobotTestCase(unittest.TestCase):
@@ -21 +22 @@
             agent = self.agent
         if self.good:
-            self.failUnless(self.parser.can_fetch(agent, url))
+            self.assertTrue(self.parser.can_fetch(agent, url))
         else:
-            self.failIf(self.parser.can_fetch(agent, url))
+            self.assertFalse(self.parser.can_fetch(agent, url))
 
     def __str__(self):
@@ -203 +204 @@
 
 
-
-class TestCase(unittest.TestCase):
-    def runTest(self):
+# 14. For issue #6325 (query string support)
+doc = """
+User-agent: *
+Disallow: /some/path?name=value
+"""
+
+good = ['/some/path']
+bad = ['/some/path?name=value']
+
+RobotTest(14, doc, good, bad)
+
+# 15. For issue #4108 (obey first * entry)
+doc = """
+User-agent: *
+Disallow: /some/path
+
+User-agent: *
+Disallow: /another/path
+"""
+
+good = ['/another/path']
+bad = ['/some/path']
+
+RobotTest(15, doc, good, bad)
+
+# 16. Empty query (issue #17403). Normalizing the url first.
+doc = """
+User-agent: *
+Allow: /some/path?
+Disallow: /another/path?
+"""
+
+good = ['/some/path?']
+bad = ['/another/path?']
+
+RobotTest(16, doc, good, bad)
+
+
+class NetworkTestCase(unittest.TestCase):
+
+    def testPasswordProtectedSite(self):
         test_support.requires('network')
-        # whole site is password-protected.
-        url = 'http://mueblesmoraleda.com'
-        parser = robotparser.RobotFileParser()
-        parser.set_url(url)
-        parser.read()
-        self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False)
+        with test_support.transient_internet('mueblesmoraleda.com'):
+            url = 'http://mueblesmoraleda.com'
+            robots_url = url + "/robots.txt"
+            # First check the URL is usable for our purposes, since the
+            # test site is a bit flaky.
+            try:
+                urlopen(robots_url)
+            except HTTPError as e:
+                if e.code not in {401, 403}:
+                    self.skipTest(
+                        "%r should return a 401 or 403 HTTP error, not %r"
+                        % (robots_url, e.code))
+            else:
+                self.skipTest(
+                    "%r should return a 401 or 403 HTTP error, not succeed"
+                    % (robots_url))
+            parser = robotparser.RobotFileParser()
+            parser.set_url(url)
+            try:
+                parser.read()
+            except IOError:
+                self.skipTest('%s is unavailable' % url)
+            self.assertEqual(parser.can_fetch("*", robots_url), False)
+
+    def testPythonOrg(self):
+        test_support.requires('network')
+        with test_support.transient_internet('www.python.org'):
+            parser = robotparser.RobotFileParser(
+                "http://www.python.org/robots.txt")
+            parser.read()
+            self.assertTrue(
+                parser.can_fetch("*", "http://www.python.org/robots.txt"))
+
 
 def test_main():
     test_support.run_unittest(tests)
-    TestCase().run()
+    test_support.run_unittest(NetworkTestCase)
 
 if __name__=='__main__':