Refactor launcher and uploader (#9)

* explicit path to compress

* Support wikis that have an incorrectly configured DH key (mediawiki-client-tools#224)

This makes the tool work on https://kol.coldfront.net/thekolwiki, for
example

* .md link to caps (mediawiki-client-tools#225)

One intra-documentation link gave me a 404 so I updated it to the
filename. All other markdown links look ok.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

* Update and remove URLs in test (mediawiki-client-tools#227)

I found some URLs in test file have redirect to non wiki URL so I update
and remove it for avoid unnecessary error when running check.

* [Backport] Add option to disable image file size check (close mediawiki-client-tools#170) (mediawiki-client-tools#228)

This commit is backport from
[saveweb/wikiteam3](https://github.com/saveweb/wikiteam3) all credit
goes to the original author.

Close mediawiki-client-tools#170 
Fix size mismatch error when some wiki do server-side image
resizing/compression without re-upload/update data in wiki.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

* Fix error in commit 265855d (mediawiki-client-tools#229)

In commit 265855d, I forget to change MediaWiki version in
site_info_test.py so this commit fix this problem.

Also I change function name in this file from `test_mediawiki_1_16` to
`test_mediawiki_version_match` to make it clear what it really does.

* get latest changes

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* sync with upstream (#8)

* uploader might want to compare new archive with trailing cruft in prefix filename

* disable SSL certificate verification

---------

Co-authored-by: Samuel Gaus <sam@gaus.co.uk>
Co-authored-by: E. Seiver <ellis.seiver@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: sian1468 <58017832+sian1468@users.noreply.github.com>

INSTALLATION.md

@@ -110,22 +110,22 @@ poetry update && poetry install && poetry build
 ```
 
 ```bash
-pip install --force-reinstall dist/*.whl
+pip3 install --force-reinstall dist/*.whl
 ```
 
 <details>
 <summary>For Windows Command Prompt, enter this pip command instead, (in a batch file use %%x).</summary>
 
 ```bash
-for %x in (dist\*.whl) do pip install --force-reinstall %x
+for %x in (dist\*.whl) do pip3 install --force-reinstall %x
 ```
 
 </details>
 <details>
 <summary>For Windows Powershell, enter this pip command instead.</summary>
 
 ```bash
-pip install --force-reinstall (Get-ChildItem .\dist\*.whl).FullName
+pip3 install --force-reinstall (Get-ChildItem .\dist\*.whl).FullName
 ```
 
 </details>
@@ -141,7 +141,7 @@ dumpgenerator [args]
 ### 3. Uninstalling the package and deleting the cloned repository when you're done
 
 ```shell
-pip uninstall wikiteam3
+pip3 uninstall wikiteam3
 ```
 
 ```bash
@@ -161,22 +161,22 @@ poetry update && poetry install && poetry build
 ```
 
 ```bash
-pip install --force-reinstall dist/*.whl
+pip3 install --force-reinstall dist/*.whl
 ```
 
 <details>
 <summary>For Windows Command Prompt, enter this pip command instead, (in a batch file use %%x).</summary>
 
 ```bash
-for %x in (dist\*.whl) do pip install --force-reinstall %x
+for %x in (dist\*.whl) do pip3 install --force-reinstall %x
 ```
 
 </details>
 <details>
 <summary>For Windows Powershell, enter this pip command instead.</summary>
 
 ```bash
-pip install --force-reinstall (Get-ChildItem .\dist\*.whl).FullName
+pip3 install --force-reinstall (Get-ChildItem .\dist\*.whl).FullName
 ```
 
 </details>
@@ -194,22 +194,22 @@ poetry update && poetry install && poetry build
 ```
 
 ```bash
-pip install --force-reinstall dist/*.whl
+pip3 install --force-reinstall dist/*.whl
 ```
 
 <details>
 <summary>For Windows Command Prompt, enter this pip command instead, (in a batch file use %%x).</summary>
 
 ```bash
-for %x in (dist\*.whl) do pip install --force-reinstall %x
+for %x in (dist\*.whl) do pip3 install --force-reinstall %x
 ```
 
 </details>
 <details>
 <summary>For Windows Powershell, enter this pip command instead.</summary>
 
 ```bash
-pip install --force-reinstall (Get-ChildItem .\dist\*.whl).FullName
+pip3 install --force-reinstall (Get-ChildItem .\dist\*.whl).FullName
 ```
 
 </details>

wikiteam3/dumpgenerator/dump/image/html_regexs_test.py

@@ -42,8 +42,6 @@ class TestRegexs:
     class TestRegexsOnline:
         listFiles_urls = {
             # site-date: url , `limit=` for counting the number of matches
-            "group0.mediawiki.demo.save-web.org_mediawiki-1.16.5-20230701": "http://group0.mediawiki.demo.save-web.org/mediawiki-1.16.5/index.php?title=特殊:文件列表&limit=2",
-            "group2.mediawiki.demo.save-web.org_mediawiki-1.39.1-20230701": "http://group2.mediawiki.demo.save-web.org/mediawiki-1.39.1/index.php?title=Special:ListFiles&limit=1",
             "archiveteam.org-20230701": "https://wiki.archiveteam.org/index.php?title=Special:ListFiles&sort=byname&limit=7",
             "wiki.othing.xyz-20230701": "https://wiki.othing.xyz/index.php?title=Special:ListFiles&sort=byname",
             "mediawiki.org-20230701": "https://www.mediawiki.org/w/index.php?title=Special:ListFiles&sort=byname&limit=7",

wikiteam3/dumpgenerator/dump/image/image.py

@@ -48,6 +48,7 @@ def generateImageDump(
         c_savedImageDescs = 0
 
         bypass_cdn_image_compression: bool = other["bypass_cdn_image_compression"]
+        disable_image_verify: bool = other["disable_image_verify"]
 
         def modify_params(params: Optional[Dict] = None) -> Dict:
             """bypass Cloudflare Polish (image optimization)"""
@@ -130,7 +131,11 @@ def check_response(r: requests.Response) -> None:
 
                 if r.status_code == 200:
                     try:
-                        if size == "False" or len(r.content) == int(size):
+                        if (
+                            size == "False"
+                            or len(r.content) == int(size)
+                            or disable_image_verify
+                        ):
                             # size == 'False' means size is unknown
                             with open(filename3, "wb") as imagefile:
                                 imagefile.write(r.content)
@@ -582,4 +587,4 @@ def curateImageURL(config: Config = None, url=""):
         # urls with odd chars
         url = re.sub(" ", "_", url)
 
-        return url
+        return url

wikiteam3/dumpgenerator/dump/misc/site_info_test.py

@@ -8,10 +8,10 @@
 from .site_info import saveSiteInfo
 
 
-def test_mediawiki_1_16():
-    with get_config("1.16.5") as config:
+def test_mediawiki_version_match():
+    with get_config("1.39.5") as config:
         sess = requests.Session()
         saveSiteInfo(config, sess)
         with open(f"{config.path}/siteinfo.json") as f:
             siteInfoJson = json.load(f)
-        assert siteInfoJson["query"]["general"]["generator"] == "MediaWiki 1.16.5"
+        assert siteInfoJson["query"]["general"]["generator"] == "MediaWiki 1.39.5"

wikiteam3/dumpgenerator/test/test_config.py

@@ -26,10 +26,10 @@ def _new_config_from_parameter(params):
 
 def get_config(mediawiki_ver, api=True):
     assert api == True
-    if mediawiki_ver == "1.16.5":
+    if mediawiki_ver == "1.39.5":
         return _new_config_from_parameter(
             [
                 "--api",
-                "http://group0.mediawiki.demo.save-web.org/mediawiki-1.16.5/api.php",
+                "https://testw.fandom.com/api.php",
             ]
         )

wikiteam3/launcher/launch.py

@@ -25,10 +25,10 @@
 import time
 from pathlib import Path
 
-from compress import compress_history, compress_images
 from py7zr import SevenZipFile
 
 from wikiteam3.dumpgenerator.config import Config
+from wikiteam3.launcher.compress import compress_history, compress_images
 from wikiteam3.utils.checkxml import check_xml_integrity
 from wikiteam3.utils.domain import domain2prefix