From afab72c894984409090d1fb171e526cf4edfc9a6 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Mon, 3 May 2021 14:01:04 +0800 Subject: [PATCH 1/9] Construct the cdx API query using a URI object This avoids problems related to URL encoding. Obsoletes: https://github.com/hartator/wayback-machine-downloader/pull/116 --- lib/wayback_machine_downloader/archive_api.rb | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/wayback_machine_downloader/archive_api.rb b/lib/wayback_machine_downloader/archive_api.rb index 903f42b..ef8d3a0 100644 --- a/lib/wayback_machine_downloader/archive_api.rb +++ b/lib/wayback_machine_downloader/archive_api.rb @@ -1,28 +1,29 @@ +require 'uri' + module ArchiveAPI def get_raw_list_from_api url, page_index - request_url = "https://web.archive.org/cdx/search/xd?url=" - request_url += url - request_url += parameters_for_api page_index + request_url = URI("https://web.archive.org/cdx/search/xd") + params = [["url", url]] + params += parameters_for_api page_index + request_url.query = URI.encode_www_form(params) URI.open(request_url).read end def parameters_for_api page_index - parameters = "&fl=timestamp,original&collapse=digest&gzip=false" - if @all - parameters += "" - else - parameters += "&filter=statuscode:200" + parameters = [["fl", "timestamp,original"], ["collapse", "digest"], ["gzip", "false"]] + if !@all + parameters.push(["filter", "statuscode:200"]) end if @from_timestamp and @from_timestamp != 0 - parameters += "&from=" + @from_timestamp.to_s + parameters.push(["from", @from_timestamp.to_s]) end if @to_timestamp and @to_timestamp != 0 - parameters += "&to=" + @to_timestamp.to_s + parameters.push(["to", @to_timestamp.to_s]) end if page_index - parameters += "&page=#{page_index}" + parameters.push(["page", page_index]) end parameters end From cd29f79fd049a2df1431b4d65e59ae6e82cdcd3f Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Mon, 3 May 2021 16:48:49 +0800 Subject: [PATCH 2/9] Switch to the JSON output format for easier parsing --- lib/wayback_machine_downloader.rb | 16 ++++++---------- lib/wayback_machine_downloader/archive_api.rb | 13 +++++++++++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/wayback_machine_downloader.rb b/lib/wayback_machine_downloader.rb index 04005c8..d1098e0 100644 --- a/lib/wayback_machine_downloader.rb +++ b/lib/wayback_machine_downloader.rb @@ -84,7 +84,7 @@ def get_all_snapshots_to_consider # Note: Passing a page index parameter allow us to get more snapshots, # but from a less fresh index print "Getting snapshot pages" - snapshot_list_to_consider = "" + snapshot_list_to_consider = [] snapshot_list_to_consider += get_raw_list_from_api(@base_url, nil) print "." unless @exact_url @@ -95,17 +95,15 @@ def get_all_snapshots_to_consider print "." end end - puts " found #{snapshot_list_to_consider.lines.count} snaphots to consider." + puts " found #{snapshot_list_to_consider.length} snaphots to consider." puts snapshot_list_to_consider end def get_file_list_curated file_list_curated = Hash.new - get_all_snapshots_to_consider.each_line do |line| - next unless line.include?('/') - file_timestamp = line[0..13].to_i - file_url = line[15..-2] + get_all_snapshots_to_consider.each do |file_timestamp, file_url| + next unless file_url.include?('/') file_id = file_url.split('/')[3..-1].join('/') file_id = CGI::unescape file_id file_id = file_id.tidy_bytes unless file_id == "" @@ -130,10 +128,8 @@ def get_file_list_curated def get_file_list_all_timestamps file_list_curated = Hash.new - get_all_snapshots_to_consider.each_line do |line| - next unless line.include?('/') - file_timestamp = line[0..13].to_i - file_url = line[15..-2] + get_all_snapshots_to_consider.each do |file_timestamp, file_url| + next unless file_url.include?('/') file_id = file_url.split('/')[3..-1].join('/') file_id_and_timestamp = [file_timestamp, file_id].join('/') file_id_and_timestamp = CGI::unescape file_id_and_timestamp diff --git a/lib/wayback_machine_downloader/archive_api.rb b/lib/wayback_machine_downloader/archive_api.rb index ef8d3a0..e4b3529 100644 --- a/lib/wayback_machine_downloader/archive_api.rb +++ b/lib/wayback_machine_downloader/archive_api.rb @@ -1,14 +1,23 @@ +require 'json' require 'uri' module ArchiveAPI def get_raw_list_from_api url, page_index request_url = URI("https://web.archive.org/cdx/search/xd") - params = [["url", url]] + params = [["output", "json"], ["url", url]] params += parameters_for_api page_index request_url.query = URI.encode_www_form(params) - URI.open(request_url).read + begin + json = JSON.parse(URI.open(request_url).read) + if (json[0] <=> ["timestamp","original"]) == 0 + json.shift + end + json + rescue JSON::ParserError + [] + end end def parameters_for_api page_index From 3d0c702fdfee43e661cc8d7218ea29b147ffc7e9 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Mon, 3 May 2021 17:50:26 +0800 Subject: [PATCH 3/9] Update Internet Archive URLs to https --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2bb9ea6..5d57f9a 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ Example: -f, --from TIMESTAMP -Optional. You may want to supply a from timestamp to lock your backup to a specific version of the website. Timestamps can be found inside the urls of the regular Wayback Machine website (e.g., http://web.archive.org/web/20060716231334/http://example.com). You can also use years (2006), years + month (200607), etc. It can be used in combination of To Timestamp. +Optional. You may want to supply a from timestamp to lock your backup to a specific version of the website. Timestamps can be found inside the urls of the regular Wayback Machine website (e.g., https://web.archive.org/web/20060716231334/http://example.com). You can also use years (2006), years + month (200607), etc. It can be used in combination of To Timestamp. Wayback Machine Downloader will then fetch only file versions on or after the timestamp specified. Example: @@ -89,7 +89,7 @@ Example: -t, --to TIMESTAMP -Optional. You may want to supply a to timestamp to lock your backup to a specifc version of the website. Timestamps can be found inside the urls of the regular Wayback Machine website (e.g., http://web.archive.org/web/20100916231334/http://example.com). You can also use years (2010), years + month (201009), etc. It can be used in combination of From Timestamp. +Optional. You may want to supply a to timestamp to lock your backup to a specifc version of the website. Timestamps can be found inside the urls of the regular Wayback Machine website (e.g., https://web.archive.org/web/20100916231334/http://example.com). You can also use years (2010), years + month (201009), etc. It can be used in combination of From Timestamp. Wayback Machine Downloader will then fetch only file versions on or before the timestamp specified. Example: From bec41e09ea554943a7f52930cb64382b5460e43d Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Mon, 3 May 2021 20:15:17 +0800 Subject: [PATCH 4/9] Instruct git to ignore a file generated by the tests --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index adbec99..6c6f892 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ rdoc log websites .DS_Store +.rake_tasks~ ## BUNDLER *.gem From ea15965d6d8093bc8a8af496610130eb53db8329 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Mon, 3 May 2021 20:20:09 +0800 Subject: [PATCH 5/9] Fix typos Suggested-by: codespell, spellintian --- README.md | 8 ++++---- bin/wayback_machine_downloader | 2 +- lib/wayback_machine_downloader/tidy_bytes.rb | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5d57f9a..f9277ba 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ It will download the last version of every file present on Wayback Machine to `. -x, --exclude EXCLUDE_FILTER Skip downloading of urls that match this filter (use // notation for the filter to be treated as a regex) -a, --all Expand downloading to error files (40x and 50x) and redirections (30x) - -c, --concurrency NUMBER Number of multiple files to dowload at a time + -c, --concurrency NUMBER Number of multiple files to download at a time Default is one file at a time (ie. 20) -p, --maximum-snapshot NUMBER Maximum snapshot pages to consider (Default is 100) Count an average of 150,000 snapshots per page @@ -62,7 +62,7 @@ Example: -s, --all-timestamps -Optional. This option will download all timestamps/snapshots for a given website. It will uses the timepstamp of each snapshot as directory. +Optional. This option will download all timestamps/snapshots for a given website. It will uses the timestamp of each snapshot as directory. Example: @@ -89,7 +89,7 @@ Example: -t, --to TIMESTAMP -Optional. You may want to supply a to timestamp to lock your backup to a specifc version of the website. Timestamps can be found inside the urls of the regular Wayback Machine website (e.g., https://web.archive.org/web/20100916231334/http://example.com). You can also use years (2010), years + month (201009), etc. It can be used in combination of From Timestamp. +Optional. You may want to supply a to timestamp to lock your backup to a specific version of the website. Timestamps can be found inside the urls of the regular Wayback Machine website (e.g., https://web.archive.org/web/20100916231334/http://example.com). You can also use years (2010), years + month (201009), etc. It can be used in combination of From Timestamp. Wayback Machine Downloader will then fetch only file versions on or before the timestamp specified. Example: @@ -169,7 +169,7 @@ Example: -c, --concurrency NUMBER -Optional. Specify the number of multiple files you want to download at the same time. Allows to speed up the download of a website significantly. Default is to download one file at a time. +Optional. Specify the number of multiple files you want to download at the same time. Allows one to speed up the download of a website significantly. Default is to download one file at a time. Example: diff --git a/bin/wayback_machine_downloader b/bin/wayback_machine_downloader index 8b9f2fd..4fb6d3d 100755 --- a/bin/wayback_machine_downloader +++ b/bin/wayback_machine_downloader @@ -46,7 +46,7 @@ option_parser = OptionParser.new do |opts| options[:all] = true end - opts.on("-c", "--concurrency NUMBER", Integer, "Number of multiple files to dowload at a time", "Default is one file at a time (ie. 20)") do |t| + opts.on("-c", "--concurrency NUMBER", Integer, "Number of multiple files to download at a time", "Default is one file at a time (ie. 20)") do |t| options[:threads_count] = t end diff --git a/lib/wayback_machine_downloader/tidy_bytes.rb b/lib/wayback_machine_downloader/tidy_bytes.rb index ba000b4..0d08431 100644 --- a/lib/wayback_machine_downloader/tidy_bytes.rb +++ b/lib/wayback_machine_downloader/tidy_bytes.rb @@ -70,7 +70,7 @@ def tidy_bytes(force = false) if is_unused || is_restricted bytes[i] = tidy_byte(byte) elsif is_cont - # Not expecting contination byte? Clean up. Otherwise, now expect one less. + # Not expecting continuation byte? Clean up. Otherwise, now expect one less. conts_expected == 0 ? bytes[i] = tidy_byte(byte) : conts_expected -= 1 else if conts_expected > 0 From 06e25957b6e51be923b7d544a8614487a50dd67e Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Mon, 3 May 2021 20:52:28 +0800 Subject: [PATCH 6/9] Print progress messages to stderr when printing JSON This avoids the messages breaking JSON parsing when the output is being redirected to a file and parsed. --- lib/wayback_machine_downloader.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/wayback_machine_downloader.rb b/lib/wayback_machine_downloader.rb index d1098e0..951175e 100644 --- a/lib/wayback_machine_downloader.rb +++ b/lib/wayback_machine_downloader.rb @@ -172,7 +172,10 @@ def get_file_list_by_timestamp def list_files # retrieval produces its own output + @orig_stdout = $stdout + $stdout = $stderr files = get_file_list_by_timestamp + $stdout = @orig_stdout puts "[" files.each do |file| puts file.to_json + "," From ba4ca603775025d200387ec531e2340201f06514 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Mon, 3 May 2021 20:54:29 +0800 Subject: [PATCH 7/9] Do not emit a comma for the final item in JSON output This avoids producing JSON that is not parsable. --- lib/wayback_machine_downloader.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/wayback_machine_downloader.rb b/lib/wayback_machine_downloader.rb index 951175e..bfd61ff 100644 --- a/lib/wayback_machine_downloader.rb +++ b/lib/wayback_machine_downloader.rb @@ -177,9 +177,10 @@ def list_files files = get_file_list_by_timestamp $stdout = @orig_stdout puts "[" - files.each do |file| + files[0...-1].each do |file| puts file.to_json + "," end + puts files[-1].to_json puts "]" end From 30475c5c9e1d92d63b75dc5f22a40dd16c1aa23a Mon Sep 17 00:00:00 2001 From: hartator Date: Sun, 6 Jun 2021 19:47:11 -0500 Subject: [PATCH 8/9] Make URI#open cross Ruby versions compatible --- lib/wayback_machine_downloader/archive_api.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wayback_machine_downloader/archive_api.rb b/lib/wayback_machine_downloader/archive_api.rb index e4b3529..01b36d9 100644 --- a/lib/wayback_machine_downloader/archive_api.rb +++ b/lib/wayback_machine_downloader/archive_api.rb @@ -10,7 +10,7 @@ def get_raw_list_from_api url, page_index request_url.query = URI.encode_www_form(params) begin - json = JSON.parse(URI.open(request_url).read) + json = JSON.parse(URI(request_url).open.read) if (json[0] <=> ["timestamp","original"]) == 0 json.shift end From 83b4f880b10a063357ad535e1fc6ed495df3d2e8 Mon Sep 17 00:00:00 2001 From: hartator Date: Sun, 6 Jun 2021 19:47:48 -0500 Subject: [PATCH 9/9] Bump Gem version --- lib/wayback_machine_downloader.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wayback_machine_downloader.rb b/lib/wayback_machine_downloader.rb index bfd61ff..81c1d2e 100644 --- a/lib/wayback_machine_downloader.rb +++ b/lib/wayback_machine_downloader.rb @@ -14,7 +14,7 @@ class WaybackMachineDownloader include ArchiveAPI - VERSION = "2.2.1" + VERSION = "2.3.0" attr_accessor :base_url, :exact_url, :directory, :all_timestamps, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter,