-
Notifications
You must be signed in to change notification settings - Fork 17
/
github.nim
181 lines (155 loc) · 6.54 KB
/
github.nim
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
#
# Nim package directory
# GitHub interface
#
import std/[algorithm, asyncdispatch, httpclient, json, strutils, tables, times]
import jester, statsd_client
import util, persist
type
Pkg* = JsonNode
Pkgs* = TableRef[string, Pkg]
const
github_caching_time* = 600
let conf* = load_conf()
let stats* = newStatdClient(prefix = "nim_package_directory")
let github_token_headers = newHttpHeaders({
"Authorization": "token $#" % conf.github_token})
# volatile caches
var volatile_cache_github_trending_last_update_time = 0
var volatile_cache_github_trending: seq[JsonNode] = @[]
proc fetch_from_github(url: string): Future[string] {.async.} =
## Fetch content from GitHub asychronously
log_debug "fetching ", url
try:
let ac = newAsyncHttpClient()
ac.headers = github_token_headers
return await ac.getContent(url)
except:
log_debug "failed to fetch content ", url
log_debug getCurrentExceptionMsg()
return ""
proc fetch_json*(url: string): Future[JsonNode] {.async.} =
## Fetch JSON from GitHub asynchronously
let response = await fetch_from_github(url)
return response.parseJson()
proc fetch_nimble_packages*(): Future[string] {.async.} =
## Fetch the packages.json file from GitHub
let url = "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json"
return await fetch_from_github(url)
# TODO: return strings, not newJStrings
proc fetch_github_readme*(owner, repo_name: string): Future[JsonNode] {.async.} =
## Fetch README.* from GitHub
let url = "https://api.github.com/repos/$#/$#/readme" % [owner, repo_name]
log_debug "fetching ", url
try:
let ac = newAsyncHttpClient()
ac.headers = github_token_headers
ac.headers["Accept"] = "application/vnd.github.v3.html" # necessary
let readme = await ac.getContent(url)
return newJString readme
except:
log_debug "failed to fetch content ", url
log_debug getCurrentExceptionMsg()
return newJString ""
proc fetch_github_doc_pages*(owner, repo_name: string): Future[JsonNode] {.async.} =
## Fetch documentation pages from GitHub
let url = "https://$#.github.io/$#/index.html" % [owner.toLowerAscii, repo_name]
log_debug "checking ", url
let resp = await newAsyncHttpClient().get(url)
if resp.status.startswith("200"):
return newJString url
else:
log_debug "doc not found at ", url
return newJString ""
proc fetch_github_versions*(pkg: Pkg, owner, repo_name: string) {.async.} =
## Fetch versions from GH from releases and tags
## Set github_versions, github_latest_version, github_latest_version_url
let github_tags_url = "https://api.github.com/repos/$#/$#/tags" % [owner, repo_name]
log_debug "fetching GitHub tags ", github_tags_url
var version_names = newJArray()
try:
let ac = newAsyncHttpClient()
ac.headers = github_token_headers
let rtags = await ac.getContent(github_tags_url)
let tags = parseJson(rtags)
for t in tags:
let name = t["name"].str.strip(trailing = false, chars = {'v'})
if name.len > 0:
version_names.add newJString name
except:
log_info getCurrentExceptionMsg()
pkg["github_versions"] = version_names
pkg["github_latest_version"] = newJString "none"
pkg["github_latest_version_url"] = newJString ""
return
pkg["github_versions"] = version_names
log_debug "fetched $# GH versions" % $len(version_names)
let github_api_releases_url = "https://api.github.com/repos/$#/$#/releases" % [owner, repo_name]
log_debug "fetching GH releases ", github_api_releases_url
var releases: JsonNode
try:
releases = await fetch_json(github_api_releases_url)
except:
log_debug getCurrentExceptionMsg()
releases = newJArray()
if releases.len > 0:
let (latest_version, meta) = extract_latest_version(releases)
doAssert meta != nil
pkg["github_latest_version"] = newJString latest_version
pkg["github_latest_versions_str"] = extract_latest_versions_str(releases)
pkg["github_latest_version_url"] = newJString meta["tarball_url"].str
pkg["github_latest_version_time"] = newJString meta["published_at"].str
else:
log_debug getCurrentExceptionMsg()
log_debug "No releases - falling back to tags"
var latest = "0"
for v in version_names:
if v.str > latest:
latest = v.str
if latest == "0":
pkg["github_latest_version"] = newJString "none"
pkg["github_latest_version_url"] = newJString ""
else:
pkg["github_latest_version"] = newJString latest.strip
pkg["github_latest_version_url"] = newJString(
"https://github.com/$#/$#/archive/v$#.tar.gz" % [owner, repo_name, latest]
)
pkg["github_latest_version_time"] = newJString ""
proc fetch_trending_packages*(request: Request, pkgs: Pkgs): Future[seq[Pkg]] {.async.} =
## Fetches trending repositories written in Nim from GitHub, and filters packages.json down to those
if volatile_cache_github_trending_last_update_time +
github_caching_time > epochTime().int:
return volatile_cache_github_trending
let date = utc(getTime() - 14.days).format("yyyy-MM-dd")
let url = "https://api.github.com/search/repositories?q=language:nim+pushed:>$#&per_page=$#sort=$#&page=$#" % [date, "20", "updated", "1"]
log_info "searching GH repos: '$#'" % url
let query_res = await fetch_json(url)
let github_trending_pkgs: seq[JsonNode] =
query_res["items"].elems
.sortedByIt(it["updated_at"].str).reversed()
# Filter the package list to trending packages + add GitHub info
var trending_pkgs: seq[Pkg] = @[]
for p in github_trending_pkgs:
# Many packages prefix their GitHub name with `nim-`: remove these
# (there may be false positives, but considerably fewer than otherwise)
let package_name: string =
if p["name"].str.len > 4 and p["name"].str[0..3] == "nim-":
p["name"].str[4..^1].normalize()
else:
p["name"].str.normalize()
# FIXME: checking names is not completely reliable, check urls instead
if pkgs.hasKey(package_name):
var current_pkg = pkgs[package_name]
# Add GitHub stargazer count
current_pkg.add("stargazers_count", p["stargazers_count"])
# Add last updated time (pushed_at is more accurate than updated_at)
current_pkg.add("pushed_at", p["pushed_at"])
# Add GitHub author
current_pkg.add("owner", p["owner"])
trending_pkgs.add(current_pkg)
else:
log_debug "package " & package_name & " not found"
trending_pkgs = trending_pkgs.sortedByIt(it["pushed_at"].str).reversed()
volatile_cache_github_trending = trending_pkgs
volatile_cache_github_trending_last_update_time = epochTime().int
return trending_pkgs