Skip to content
This repository has been archived by the owner on Mar 4, 2024. It is now read-only.

Commit

Permalink
Implement download statistics (closes #57)
Browse files Browse the repository at this point in the history
  • Loading branch information
noahc3 committed Apr 13, 2019
1 parent e7679be commit 3352230
Show file tree
Hide file tree
Showing 15 changed files with 283 additions and 101 deletions.
11 changes: 9 additions & 2 deletions SDSetupBackend/Controllers/v1.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;

using SDSetupCommon;


namespace SDSetupBackend.Controllers {
[Route("api/[controller]")]
Expand Down Expand Up @@ -73,9 +75,14 @@ public class v1 : ControllerBase {
files.Add(new KeyValuePair<string, string>(f.Replace((Program.Files + "/" + packageset + "/" + k + "/" + channel).AsPath(), ""), f));
}
}


}

Program.dlStats.IncrementPackageDownloadCount(k);
}

Program.dlStats.AllTimeBundles++;
DeletingFileStream stream = (DeletingFileStream) ZipFromFilestreams(files.ToArray(), uuid);

Program.generatedZips[uuid] = stream;
Expand Down Expand Up @@ -129,13 +136,13 @@ public class v1 : ControllerBase {
[HttpGet("fetch/dlstats")]
public ActionResult GetDownloadStats() {
//TODO: dont do this
return new ObjectResult(Program.dlstats.ToDataBinary(U.GetPackageListInLatestPackageset()));
return new ObjectResult(Program.dlStats.ToDataBinary(U.GetPackageListInLatestPackageset()));
}

[HttpGet("fetch/dlstatsdebug")]
public ActionResult GetDownloadStatsDebug() {
//TODO: dont do this
return new ObjectResult(JsonConvert.SerializeObject(DownloadStats.FromDataBinary(Program.dlstats.ToDataBinary(U.GetPackageListInLatestPackageset())), Formatting.Indented));
return new ObjectResult(JsonConvert.SerializeObject(DownloadStats.FromDataBinary(Program.dlStats.ToDataBinary(U.GetPackageListInLatestPackageset())), Formatting.Indented));
}

[HttpGet("get/latestpackageset")]
Expand Down
58 changes: 43 additions & 15 deletions SDSetupBackend/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using Microsoft.Extensions.Logging.Console;
using Newtonsoft.Json;
using System.Net;
using System.Timers;

using SDSetupCommon;

Expand All @@ -20,9 +21,6 @@ public class Program {

private static string ip;
private static int httpPort;
private static int httpsPort;
private static string httpsCertLocation;
private static string httpsCertKey;

public static string Temp;
public static string Files;
Expand All @@ -36,7 +34,9 @@ public class Program {
public static string latestPackageset = "default";
public static string latestAppVersion = "NO VERSION";

public static DownloadStats dlstats = new DownloadStats();
public static DownloadStats dlStats;
private static bool dlStatsInitialized = false;
private static Timer dlStatsSaveTimer;

private static string _privelegedUUID;
private static string privelegedUUID {
Expand All @@ -61,11 +61,8 @@ public class Program {
string[] hostConf = File.ReadAllLines((Config + "/host.txt").AsPath());
ip = hostConf[0];
httpPort = Convert.ToInt32(hostConf[1]);
httpsPort = Convert.ToInt32(hostConf[2]);

string[] certInfo = File.ReadAllLines((Config + "/https.txt").AsPath());
httpsCertLocation = certInfo[0];
httpsCertKey = certInfo[1];

privelegedUUID = Guid.NewGuid().ToString().Replace("-", "").ToLower();

Expand Down Expand Up @@ -95,7 +92,13 @@ public class Program {
}

public static string ReloadEverything() {
//try {
try {

//if dlstats was initialized, write them to disk before reloading.
if (dlStatsInitialized) {
File.WriteAllText((Config + "/dlstats.bin").AsPath(), dlStats.ToDataBinary(U.GetPackageList(latestPackageset)));
}

//use temporary variables so if anything goes wrong, values wont be out of sync.
Dictionary<string, string> _Manifests = new Dictionary<string, string>();

Expand All @@ -110,7 +113,7 @@ public class Program {
if (!File.Exists((_Config + "/latestappversion.txt").AsPath())) File.WriteAllText((_Config + "/latestappversion.txt").AsPath(), "NO VERSION");
if (!File.Exists((_Config + "/validchannels.txt").AsPath())) File.WriteAllLines((_Config + "/validchannels.txt").AsPath(), new string[] { "latest", "nightly" });

foreach(string n in Directory.EnumerateDirectories(_Files )) {
foreach(string n in Directory.EnumerateDirectories(_Files)) {
string k = n.Split(Path.DirectorySeparatorChar).Last();
if (!File.Exists((_Files + "/" + k + "/manifest6.json").AsPath())) File.WriteAllText(_Files + "/" + k + "/manifest6.json", "{}");
}
Expand All @@ -131,21 +134,46 @@ public class Program {
_Manifests[k] = JsonConvert.SerializeObject(m, Formatting.Indented);
}


//this must be set before GetPackageListInLatestPackageset() is called
Files = _Files;

DownloadStats _dlStats;

if (File.Exists((_Config + "/dlstats.bin").AsPath())) {
_dlStats = DownloadStats.FromDataBinary(File.ReadAllText((_Config + "/dlstats.bin").AsPath()));
} else {
_dlStats = new DownloadStats();
}

Manifest latestManifest = JsonConvert.DeserializeObject<Manifest>(_Manifests[_latestPackageset]);
_dlStats.VerifyStatisticsIntegrity(U.GetPackageList(_latestPackageset), latestManifest);
_Manifests[_latestPackageset] = JsonConvert.SerializeObject(latestManifest);

if (dlStatsSaveTimer != null) dlStatsSaveTimer.Stop();
dlStatsSaveTimer = new Timer();
dlStatsSaveTimer.Interval = 10000; //10 minutes
dlStatsSaveTimer.AutoReset = true;
dlStatsSaveTimer.Elapsed += (sender, e) => {
Console.WriteLine("[ SAVE ] Wrote download stats to file (" + DateTime.Now.ToShortDateString() + " | " + DateTime.Now.ToShortTimeString() + ").");
File.WriteAllText((Config + "/dlstats.bin").AsPath(), dlStats.ToDataBinary(U.GetPackageList(latestPackageset)));
};
dlStatsSaveTimer.Start();

//update the real variables
Temp = _Temp;
Files = _Files;
Config = _Config;
latestPackageset = _latestPackageset;
latestAppVersion = _latestAppVersion;
validChannels = _validChannels;
Manifests = _Manifests;
dlStats = _dlStats;

dlstats.VerifyStatisticsIntegrity(U.GetPackageListInLatestPackageset());
dlStatsInitialized = true;

//} catch (Exception e) {
// return "[ERROR] Something went wrong while reloading: \n\n\nMessage:\n " + e.Message + "\n\nStack Trace:\n" + e.StackTrace + "\n\n\nThe server will continue running and no changes will be saved";
//}
} catch (Exception e) {
throw e;
return "[ERROR] Something went wrong while reloading: \n\n\nMessage:\n " + e.Message + "\n\nStack Trace:\n" + e.StackTrace + "\n\n\nThe server will continue running and no changes will be saved";
}
return "Success";
}

Expand Down
27 changes: 13 additions & 14 deletions SDSetupBackend/Properties/launchSettings.json
Original file line number Diff line number Diff line change
@@ -1,30 +1,29 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
{
"iisSettings": {
"windowsAuthentication": false,
"anonymousAuthentication": true,
"windowsAuthentication": false,
"anonymousAuthentication": true,
"iisExpress": {
"applicationUrl": "http://localhost:58109",
"sslPort": 0
}
},
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"IIS Express": {
"commandName": "IISExpress",
"launchBrowser": true,
"launchUrl": "api/v1/fetch",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"sdsetup_backend": {
"commandName": "Project",
"launchBrowser": true,
"launchUrl": "api/v1/fetch",
"applicationUrl": "http://localhost:5000",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
}
"sdsetup_backend": {
"commandName": "Project",
"launchBrowser": true,
"launchUrl": "api/v1/fetch",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
},
"applicationUrl": "http://localhost:5000"
}
}
}
2 changes: 1 addition & 1 deletion SDSetupBackend/SDSetupBackend.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

<ItemGroup>
<PackageReference Include="AspNetCoreRateLimit" Version="2.1.0" />
<PackageReference Include="Microsoft.AspNet.WebApi.Cors" Version="5.2.7" />
<PackageReference Include="Microsoft.AspNet.Cors" Version="5.2.7" />
<PackageReference Include="Microsoft.AspNetCore.App" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
<PackageReference Include="SharpZipLib" Version="1.0.0" />
Expand Down
14 changes: 13 additions & 1 deletion SDSetupBackend/Startup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,17 @@ public class Startup {
services.AddSingleton<IIpPolicyStore, MemoryCacheIpPolicyStore>();
services.AddSingleton<IRateLimitCounterStore, MemoryCacheRateLimitCounterStore>();

services.AddCors();
services.AddCors(options =>
{
options.AddPolicy("AllowAll",
builder => {
builder
.AllowAnyOrigin()
.AllowAnyMethod()
.AllowAnyHeader()
.AllowCredentials();
});
});
services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_1);
}

Expand All @@ -52,6 +62,8 @@ public class Startup {
});
app.UseIpRateLimiting();

app.UseCors("AllowAll");

app.UseMvc();
}
}
Expand Down
2 changes: 2 additions & 0 deletions SDSetupBackend/U.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
using System.IO;
using System.Threading.Tasks;

using SDSetupCommon;

namespace SDSetupBackend {
public class U {
public static List<string> GetPackageListInLatestPackageset() {
Expand Down
4 changes: 3 additions & 1 deletion SDSetupBlazor/Components/UIPackageSection.cshtml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@
<h4 style="margin:0">@p.Name</h4>
<h6 style="margin-top:0;margin-bottom:0.5rem">by @p.Authors</h6>
<p>@((MarkupString)p.Description)</p>
<br /><p>Retrieved from:<br /><a style="color:#DB2828" href="@p.Source">@p.Source</a></p>
<br />
<p>Downloads: @p.Downloads<br />
Retrieved from:<br /><a style="color:#DB2828" href="@p.Source">@p.Source</a></p>
</div>
}
</div>
Expand Down
7 changes: 7 additions & 0 deletions SDSetupBlazor/G.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@
namespace SDSetupBlazor
{
public static class G {
#if (DEBUG)
public static string hostname = "http://localhost:58109";
#else
public static string hostname = "https://files.sdsetup.com";
#endif

public static string packageset = "default";
public static string channel = "latest";

Expand All @@ -31,6 +37,7 @@ public static class G {
public static Dictionary<string, Dictionary<string, bool>> selectedPackages = new Dictionary<string, Dictionary<string, bool>>();
public static Dictionary<string, Dictionary<string, Package>> packages = new Dictionary<string, Dictionary<string, Package>>();
public static Manifest manifest;
public static DownloadStats downloadStats;

public static bool showWarning = false;
public static int scrollPosStore;
Expand Down
4 changes: 2 additions & 2 deletions SDSetupBlazor/Pages/Consoles.cshtml
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@
string uuid = Guid.NewGuid().ToString().Replace("-", "").ToLower();

saveUrl = UriHelper.GetBaseUri() + "console?" + G.consoleId + "#" + packages;
string getUrl = "https://testing.files.sdsetup.com/api/v1/fetch/zip";
string getUrl = G.hostname + "/api/v1/fetch/zip";


HttpClient client = new HttpClient();
Expand Down Expand Up @@ -290,7 +290,7 @@
}
string tresp = await resp.Content.ReadAsStringAsync();
if (tresp == "READY") {
UriHelper.NavigateTo("https://testing.files.sdsetup.com/api/v1/fetch/generatedzip/" + uuid);
UriHelper.NavigateTo(G.hostname + "/api/v1/fetch/generatedzip/" + uuid);

saveMessage = true;
gettingZip = false;
Expand Down
91 changes: 91 additions & 0 deletions SDSetupBlazor/Pages/Statistics.cshtml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
<!-- Copyright (c) 2018 noahc3
- This Source Code Form is subject to the terms of the Mozilla Public
- License, v. 2.0. If a copy of the MPL was not distributed with this
- file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
@page "/stats"
@inject HttpClient Http
<h1 style="text-align:center">Download Statistics</h1>
@if (G.initialized && localInit) {

<h3 style="text-align:center;font-style:italic;margin-top:-1rem">Download statistics tracked since @G.downloadStats.StatisticsTrackingInitDate UTC</h3>
<h3 style="text-align:center">Total bundles downloaded from SDSetup to date: <b>@G.downloadStats.AllTimeBundles</b></h3>

<div class="ui styled fluid accordion" style="margin-top:1.5em">

@foreach (Platform plat in G.manifest.Platforms.Values) {
if (!plat.Visible) { continue; }
<div class="title @(opened[plat.ID] == true ? "active" : "")" onclick="@(() => opened[plat.ID] = !opened[plat.ID])">
<i class="dropdown icon"></i>
@plat.Name
</div>
<div class="content @(opened[plat.ID] == true ? "active" : "")">
<BlazorTable Class="sortable" IsResponsive="true" IsSmall="true" IsStriped="true" IsBordered="true">
<thead>
<tr>
<th scope="col">Package Name</th>
<th scope="col">Author(s)</th>
<th scope="col">Downloads (Last 7 Days)</th>
<th scope="col">Downloads (Last 30 Days)</th>
<th scope="col">Downloads (All Time)</th>
</tr>
</thead>
<tbody>
@foreach (Package p in G.packages[plat.ID].Values) {
if (!calculatedStats.ContainsKey(p.ID)) { continue; }
<tr>
<th scope="row">@p.Name</th>
<td>@p.Authors</td>
<td>@calculatedStats[p.ID][0]</td>
<td>@calculatedStats[p.ID][1]</td>
<td>@calculatedStats[p.ID][2]</td>
</tr>
}

</tbody>
</BlazorTable>
</div>
}
</div>
} else {
<div class="ui active dimmer" style="height:100%;width:100%">
<div class="ui massive text loader">Loading</div>
</div>
}

@functions {
Dictionary<string, bool> opened = new Dictionary<string, bool>();
public static Action ForceUiUpdate;
private bool localInit = false;
private Dictionary<string, int>[] granularStats;
private Dictionary<string, long[]> calculatedStats = new Dictionary<string, long[]>();

protected override async Task OnInitAsync() {
string dlStats = await Http.GetStringAsync(G.hostname + "/api/v1/fetch/dlstats");
G.downloadStats = DownloadStats.FromDataBinary(dlStats);
granularStats = G.downloadStats.GranularStats.Values.ToArray();
foreach (Platform plat in G.manifest.Platforms.Values) {
opened[plat.ID] = false;
if (!plat.Visible) { continue; }
foreach (Package p in G.packages[plat.ID].Values) {
if (String.IsNullOrEmpty(p.Authors) || !p.Visible || !G.downloadStats.AllTimeStats.ContainsKey(p.ID)) { continue; }
calculatedStats[p.ID] = new long[3];
calculatedStats[p.ID][0] = CalcHours(p.ID, 168);
calculatedStats[p.ID][1] = CalcHours(p.ID, 720);
calculatedStats[p.ID][2] = G.downloadStats.AllTimeStats[p.ID];
}
Console.WriteLine("Done");
}

ForceUiUpdate = new Action(() => StateHasChanged());
localInit = true;
}

private long CalcHours(string package, int hours) {
if (!G.downloadStats.AllTimeStats.ContainsKey(package)) return 0;
int dl = 0;
for (int i = 0; i < hours; i++) {
dl += granularStats[i][package];
}
return dl;
}
}

0 comments on commit 3352230

Please sign in to comment.