0.3.9 (#16)

* fix files not having url: fixes issue #14 
* limit infinite loop on continued query: fixes issue #15

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Current
 
+### Version 0.3.9
+
+* Fix infinite loop on continued queries: [issue #15](https://github.com/barrust/mediawiki/issues/15)
+ * Check by looking at the continue variable over time; if it is the same, exit
+* Fix image with no url: [issue #14](https://github.com/barrust/mediawiki/issues/14)
+
 ### Version 0.3.8
 
 * Fix empty disambiguation list items
@@ -32,7 +38,8 @@
 
 * Update documentation
 * Better continuous integration
-
+* Better test data: [issue #4](https://github.com/barrust/mediawiki/issues/4)
+* First version on PyPi: [issue #8](https://github.com/barrust/mediawiki/issues/8)
 
 ### Version 0.3.3
 

mediawiki/mediawiki.py

@@ -18,7 +18,7 @@
 from .utilities import memoize
 
 URL = 'https://github.com/barrust/mediawiki'
-VERSION = '0.3.8'
+VERSION = '0.3.9'
 
 
 class MediaWiki(object):
@@ -841,11 +841,11 @@ def images(self):
             params = {
                 'generator': 'images',
                 'gimlimit': 'max',
-                'prop': 'imageinfo',
+                'prop': 'imageinfo',  # this will be replaced by fileinfo
                 'iiprop': 'url'
-                }
+            }
             for page in self._continued_query(params):
-                if 'imageinfo' in page:
+                if 'imageinfo' in page and 'url' in page['imageinfo'][0]:
                     self._images.append(page['imageinfo'][0]['url'])
             self._images = sorted(self._images)
         return self._images
@@ -887,7 +887,7 @@ def categories(self):
                 'prop': 'categories',
                 'cllimit': 'max',
                 'clshow': '!hidden'
-                }
+            }
             for link in self._continued_query(params):
                 if link['title'].startswith('Category:'):
                     self._categories.append(link['title'][9:])
@@ -913,7 +913,7 @@ def coordinates(self):
                 'prop': 'coordinates',
                 'colimit': 'max',
                 'titles': self.title
-                }
+            }
             request = self.mediawiki.wiki_request(params)
             res = request['query']['pages'][self.pageid]
             if 'query' in request and 'coordinates' in res:
@@ -935,7 +935,7 @@ def links(self):
                 'prop': 'links',
                 'plnamespace': 0,
                 'pllimit': 'max'
-                }
+            }
             for link in self._continued_query(params):
                 self._links.append(link['title'])
             self._links = sorted(self._links)
@@ -956,7 +956,7 @@ def redirects(self):
                 'prop': 'redirects',
                 'rdprop': 'title',
                 'rdlimit': 'max'
-                }
+            }
             for link in self._continued_query(params):
                 self._redirects.append(link['title'])
             self._redirects = sorted(self._redirects)
@@ -979,7 +979,7 @@ def backlinks(self):
                 'bllimit': 'max',
                 'blfilterredir': 'nonredirects',
                 'blnamespace': 0
-                }
+            }
             for link in self._continued_query(params, 'backlinks'):
                 self._backlinks.append(link['title'])
             self._backlinks = sorted(self._backlinks)
@@ -1176,12 +1176,12 @@ def _continued_query(self, query_params, key='pages'):
         '''
         query_params.update(self.__title_query_param())
 
-        last_continue = dict()
+        last_cont = dict()
         prop = query_params.get('prop')
 
         while True:
             params = query_params.copy()
-            params.update(last_continue)
+            params.update(last_cont)
 
             request = self.mediawiki.wiki_request(params)
 
@@ -1199,10 +1199,10 @@ def _continued_query(self, query_params, key='pages'):
                 for datum in pages[self.pageid].get(prop, list()):
                     yield datum
 
-            if 'continue' not in request:
+            if 'continue' not in request or request['continue'] == last_cont:
                 break
 
-            last_continue = request['continue']
+            last_cont = request['continue']
     # end _continued_query
 
     def __title_query_param(self):

tests/mediawiki_test.py

@@ -3,15 +3,29 @@
 '''
 # -*- coding: utf-8 -*-
 from __future__ import (unicode_literals, print_function)
+import unittest
+import json
+from datetime import timedelta
+from decimal import Decimal
+
 from mediawiki import (MediaWiki, PageError, RedirectError,
                        DisambiguationError, MediaWikiAPIURLError,
                        MediaWikiGeoCoordError, HTTPTimeoutError,
                        MediaWikiException)
 import mediawiki
-import unittest
-import json
-from datetime import timedelta
-from decimal import Decimal
+
+class FunctionUseCounter:
+    ''' decorator to keep a running count of how many
+    times function has been called; stop at 50 '''
+    def __init__(self, func):
+        self.func = func
+        self.count = 0
+
+    def __call__(self, *args, **kwargs):
+        self.count += 1
+        if self.count > 50:  # arbitrary large
+            return dict()
+        return self.func(*args, **kwargs)
 
 
 class MediaWikiOverloaded(MediaWiki):
@@ -961,3 +975,42 @@ def test_page_preload(self):
         self.assertEqual(hasattr(pag, '_coordinates'), True)
         self.assertEqual(hasattr(pag, '_backlinks'), True)
         self.assertEqual(hasattr(pag, '_categories'), True)
+
+
+class TestMediaWikiRegressions(unittest.TestCase):
+    ''' Add regression tests here for special cases '''
+
+    def test_hidden_file(self):
+        ''' test hidden file or no url: issue #14 '''
+        site = MediaWikiOverloaded()
+        res = site.responses[site.api_url]
+        page = site.page('One Two Three... Infinity')
+        try:
+            page.images
+        except KeyError:
+            self.fail("KeyError exception on hidden file")
+        self.assertEqual(page.images, res['hidden_images'])
+
+    def test_large_cont_query(self):
+        ''' test known large continued query with continue='||' '''
+        site = MediaWikiOverloaded()
+        res = site.responses[site.api_url]['large_continued_query']
+        page = site.page('List of named minor planets (numerical)')
+        self.assertEqual(page.links, res)
+
+    def test_large_cont_query_images(self):
+        ''' test known large continued query with images '''
+        site = MediaWikiOverloaded()
+        res = site.responses[site.api_url]['large_continued_query_images']
+        page = site.page('B8 polytope')
+        self.assertEqual(page.images, res)
+        self.assertEqual(len(page.images), 2213)
+
+    def test_infinit_loop_images(self):
+        ''' test known image infinite loop: issue #15 '''
+        site = MediaWikiOverloaded()
+        res = site.responses[site.api_url]['infinite_loop_images']
+        page = site.page('Rober Eryol')
+        site._get_response = FunctionUseCounter(site._get_response)
+        self.assertEqual(page.images, res)
+        self.assertEqual(site._get_response.count, 13)