log-network-requests

Web extension to log network requests

This project is part of my outreachy application for Mozilla's Lightbeam project.

---

Commit 1

As a first step, I have created a web-extension to log network requests. Here's an example how to run this web extension.

---

Commit 2

The web extension now logs only 3rd party network requests. Following logic has been used to filter 3rd party network requests:

• Get URL of current tab:

function handleUpdated(tabId, changeInfo, tabInfo) {
  if (changeInfo.url) {
    pattern = getHostname(changeInfo.url);
  }
}

browser.tabs.onUpdated.addListener(handleUpdated);

• Filter for network requests which do not match the tab's hostname:

function logURL(requestDetails) {
  let url = requestDetails.url;

  if (pattern &&
  	/^http.*/.test(url) &&	 
  	!pattern.match(getHostname(url))) {
  		console.log(" Loading 3rd party: " + url);
  }
}

browser.webRequest.onBeforeRequest.addListener(
  logURL,
  {urls: [ "<all_urls>"]}
);

---

Commit 3

The web extension now loads a html page in a new tab and logs third party network requests there.

---

Commit 4

Logging unique third party hostnames:

function logURL(requestDetails) {
  let url = requestDetails.url;
  if (pattern &&
  	/^http.*/.test(url) &&	 
  		!pattern.match(getHostname(url))) {
  			let uniqueUrl = getHostname(url);
  			if (thirdPartyUrls.indexOf(uniqueUrl) === -1) {
  				thirdPartyUrls.push(uniqueUrl);
  			}
  }
  document.getElementById("urls").innerHTML = thirdPartyUrls;
}

---

Commit 5

Visualising third party network requests using D3.js

Here's a video of the visualisations

---

Visualisation ideas are discussed here