Baseline formats of different Centinel versions
Arian Niaki edited this page Oct 4, 2017
·
21 revisions
Different versions of centinel save different formats of baseline JSON files. In this document, we will present these different formats.
The following sample belongs to centinel versions prior to 0.1.5.4
{
"meta_exception": "",
"meta": {
"as_number": "1",
"server_time": "time",
"ip": "1.2.3.0/24",
"schedule_name": "text",
"as_owner": "AS1111 AS Name",
"country": "Alpha2 Code of Country",
"client_time": "time",
"time_taken": 1.1
},
"baseline": [
{
"tls": {
"www.somewebsite.com:443": {
"cert": "",
"fingerprint": ""
},
"www.someotherwebsite.com:443": {
"tls_error": "",
"fingerprint_error": ""
}
},
"total_time": 1.1,
"http": {
"http://www.somewebsite.com": {
"redirects": {
"0": {
"path": "/",
"host": "somewebsite.com",
"response": {
"status": 301,
"headers": {
},
"reason": "Moved Permanently",
"body": ""
}
}
},
"request": {
"path": "/",
"host": "somewebsite.com",
"method": "GET",
"ssl": false
},
"response": {
"status": 200,
"headers": {
},
"reason": "OK",
"body": "" (or "body.b64" : "")
}
},
},
"file_name": "file name",
"file_metadata": {},
"file_comments": [
"comment"
],
"traceroute.udp": {
"www.somewebsite.com": {
"domain": "www.somewebsite.com",
"total_hops": 2,
"hops": {
"1": {
"rtt2": "205.327",
"ip": "172.20.32.1",
"rtt1": "205.517",
"domain_name": "172.20.32.1",
"rtt3": "204.827"
},
"2": {
"raw": "7 * * *"
}
},
"meaningful_hops": 2,
"unparseable_lines": {
"1": "traceroute to www.somewebsite.com (1.1.1.1), 30 hops max, 60 byte packets"
},
"forcefully_terminated": false,
"time_elapsed": 2,
"method": "udp"
}
},
"dns": {
"www. somewebsite.com": {
"domain": "www. somewebsite.com",
"response2": "",
"response1": "",
"request": "",
"response1-ips": [
"1.1.1.1",
"2.2.2.2",
],
"nameserver": "8.8.8.8"
}
},
"url_metadata": {
"http://www.somewebsite.com": {
"theme": "",
"generalStatus": "",
"urlCategories": ""
}
},
"traceroute.tcp": {
"www.somewebsite.com": {
"domain": "www.somewebsite.com",
"total_hops": 3,
"hops": {
"1": {
"raw": "1 1.1.1.1 (1.1.1.1) 152.237 ms 153.893 ms *"
},
"2": {
"raw": "2 * * *"
},
"3": {
"raw": "6 * somewebsite.net (2.2.2.2) 206.775 ms 206.131 ms"
}
},
"meaningful_hops": 0,
"unparseable_lines": {
"1": "traceroute to www.somewebsite.com (1.1.1.1), 30 hops max, 60 byte packets"
},
"forcefully_terminated": false,
"time_elapsed": 9,
"method": "tcp"
}
}
}
]
}
centinel_version has been added to meta. Furthermore, the traceroute format has also changed.
{
"meta_exception": "",
"meta": {
"as_number": "0",
"server_time": "time",
"ip": "1.2.3.0/24",
"schedule_name": "baseline",
"as_owner": "AS0 AS Name",
"centinel_version": "0.1.5.4.1",
"country": "Alpha2 Code of Country",
"client_time": "time",
"time_taken": 381.19438
},
"baseline": [
{
"tls": {
"www.somewebsite.com:443": {
"cert": "",
"fingerprint": ""
}
},
"total_time": 370.7348291873932,
"http": {
"http://somewebsite.com": {
"request": {
"headers": {
"user-Agent": "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
},
"host": "somewebsite.com",
"method": "GET",
"ssl": false,
"path": "/"
},
"response": {
"status": 200,
"headers": {
},
"reason": "OK",
"body": ""
}
},
"someotherwebsite.com": {
"redirects": {
"0": {
"path": "/",
"host": "someotherwebsite.com",
"response": {
"status": 200,
"headers": {
},
"reason": "Moved",
"body": ""
}
}
},
"request": {
"headers": {
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.1 (KHTML, like Gecko) Maxthon/3.0.8.2 Safari/533.1"
},
"path": "/",
"host": "someotherwebsite.com",
"method": "GET",
"ssl": false
},
"response": {
"failure": "(6, 'Could not resolve host: ')"
}
}
},
"file_name": "file.csv",
"file_metadata": {},
"file_comments": [],
"dns": {
"somewebsite.com": [
{
"nameserver": "8.8.8.8",
"domain": "somewebsite.com",
"response1-ips": [
"1.1.1.1"
]
}
]
},
"url_metadata": {
"http://somewebsite.com": {
"category_code": "CODE",
"notes": "",
"category_description": "Desc for CODE",
"date_added": "time",
"source": ""
}
},
"traceroute.tcp": {
"somewebsite.com": {
"dest_name": "somewebsite.com",
"hops": [
{
"index": 1,
"asn": null,
"probes": [
{
"anno": "",
"ip": null,
"name": null,
"rtt": null
},
{
"anno": "",
"ip": null,
"name": null,
"rtt": null
},
{
"anno": "",
"ip": null,
"name": null,
"rtt": null
}
]
},
],
"forcefully_terminated": false,
"time_elapsed": 2,
"method": "tcp",
"dest_ip": "1.2.3.4"
}
}
}
]
}
{
"meta_exception": "",
"meta": {
"as_number": "0",
"server_time": "time",
"ip": "1.2.3.0/24",
"schedule_name": "schedule name",
"as_owner": "AS0 AS Name",
"centinel_version": "0.1.5.4.2",
"country": "Alpha2 Code of Country",
"client_time": "time",
"time_taken": 795.380704,
"asn_error": ""
},
"baseline": [
{
"tls": {
"www.somewebsite.com:443": {
"cert": "",
"fingerprint": ""
}
},
"total_time": 795.3721878528595,
"http": {
"somewebsite.com": {
"request": {
"headers": {
"user-Agent": "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
},
"path": "/",
"host": "somewebsite.com",
"method": "GET",
"ssl": false
},
"response": {
"status": 200,
"headers": {
},
"reason": "OK",
"body": ""
}
}
},
"file_name": "file.csv",
"file_metadata": {},
"file_comments": [],
"traceroute.udp": {
"somewebsite.com": {
"error": "",
"method": "udp",
"dest_name": "somewebsite.com"
},
"someotherwebsite.com": {
"dest_name": "someotherwebsite.com",
"hops": [
{
"index": 1,
"asn": null,
"probes": [
{
"anno": "",
"ip": "1.2.3.4",
"name": "1.2.3.4",
"rtt": 0.575
},
{
"anno": "",
"ip": "1.2.3.4",
"name": "1.2.3.4",
"rtt": null
},
{
"anno": "",
"ip": "1.2.3.4",
"name": "1.2.3.4",
"rtt": null
}
]
},
{
"index": 2,
"asn": null,
"probes": [
{
"anno": "",
"ip": "2.3.4.5",
"name": "2.3.4.5",
"rtt": 6.663
},
{
"anno": "",
"ip": "2.3.4.5",
"name": "2.3.4.5",
"rtt": 6.604
},
{
"anno": "",
"ip": "2.3.4.5",
"name": "2.3.4.5",
"rtt": 6.897
}
]
}
],
"forcefully_terminated": true,
"time_elapsed": 60,
"method": "udp",
"dest_ip": "4.5.6.7"
}
},
"dns": {
"somewebsite.com": {
"domain": "somewebsite.com",
"response2": null,
"response1": "",
"request": "",
"response1-ips": [
"1.2.3.4"
],
"nameserver": "8.8.8.8"
}
},
"url_metadata": {
"somewebsite.com": {
"rank": "470"
}
}
}
]
}
These two versions are similar to 0.1.5.4.2. The only difference is the addition of vpn_provider in meta_data
{
"meta": {
"as_number": "0",
"server_time": "time",
"ip": "1.2.3.0/24",
"schedule_name": "schedule name",
"as_owner": "AS0 AS Name",
"vpn_provider": "vpn provider name",
"centinel_version": "0.1.5.5",
"country": "Alpha2 Code of Country",
"client_time": "time",
"time_taken": 1453.571022
},
The only difference between this version and 1.5.5 is the url_metadata which is as follows:
"url_metadata": {
"http://www.somewebsite.com": {
"category_code": "some code",
"notes": "",
"source": "some source",
"date_added": "time",
"category_description": "Desc for Category"
}
}
In some versions of 1.5.5.1 the url_metadata is as follows:
"url_metadata": {
"http://somewebsite.com": {
"category_code": "Category",
"secondary_category_code": "",
"notes": "",
"source": "some source",
"date_added": "time",
"subcategory_code": ""
},
{
"meta_exception": "",
"meta": {
"as_number": "0",
"server_time": "time",
"ip": "1.2.3.0/24",
"schedule_name": "schedule name",
"as_owner": "AS0 AS Name",
"vpn_provider": "provider",
"centinel_version": "0.1.5.6.3",
"country": "US",
"client_time": "time",
"time_taken": 162.69669
},
"baseline": [
{
"tls": {
"somewebsite.com:443": {
"tls_error": "timed out",
"fingerprint_error": null
},
"someotherwebsite.com:443": {
"cert": "",
"fingerprint": "d79f076110b39293e349ac89845b0380c19e2f8b"
}
},
"tcp_connect": {
"somewebsite.com:17613": {
"ip": "1.2.3.4",
"host": "somewebsite.com",
"port": 17613,
"success": "true",
"time": "97"
},
"someotherwebsite.com:26919": {
"ip": "2.3.4.5",
"host": "someotherwebsite.com",
"time": "5002",
"port": 26919,
"failure": "timed out"
}
},
"http": {
"somewebsite.com:17613": {
"request": {
"ssl": false,
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.1 (KHTML, like Gecko) Maxthon/3.0.8.2 Safari/533.1"
},
"host": "somewebsite.com",
"path": "/",
"port": "17613"
},
"redirect_count": 0,
"redirect_loop": false,
"full_url": "somewebsite.com:17613",
"response": {
"failure": "(28, 'Operation timed out after 10000 milliseconds with 0 bytes received')"
}
}
},
"file_name": "file.csv",
"file_metadata": {},
"file_comments": [],
"dns": {
"somewebsite.com": [
{
"domain": "somewebsite.com",
"response2": null,
"response1": "",
"request": "",
"response1-ips": [
"1.2.3.4"
],
"nameserver": "8.8.8.8"
}
]
},
"total_time": 154.65991115570068,
"url_metadata": {
"somewebsite.com:17613": {
"protocol": "protocol",
"nickname": "nickname"
}
},
"traceroute.tcp": {
"somewebsite.com": {
"dest_name": "somewebsite.com",
"hops": [
{
"index": 1,
"asn": null,
"probes": [
{
"anno": "",
"ip": null,
"name": null,
"rtt": null
},
{
"anno": "",
"ip": null,
"name": null,
"rtt": null
},
{
"anno": "",
"ip": null,
"name": null,
"rtt": null
}
]
}
],
"forcefully_terminated": false,
"time_elapsed": 0,
"method": "tcp",
"dest_ip": "198.245.60.50"
}
}
]
}
{
"meta": {
"as_number": "0",
"server_time": "time",
"ip": "1.2.3.0/24",
"schedule_name": "schedule name",
"as_owner": "AS0 AS Owner",
"vpn_provider": "provider",
"centinel_version": "0.1.5.7",
"country": "Alpha2 Country Code",
"client_time": "time",
"time_taken": 3355.795091
},
"baseline": [
{
"tls": {
"www.somewebsite.com:443": {
"cert": "",
"fingerprint": ""
}
},
"tcp_connect": {
"somewebsite.com:80": {
"ip": "1.2.3.4",
"host": "somwebsite.com",
"port": 80,
"success": "true",
"time": "1239"
}
},
"http": {
"http://somewebsite.com/": {
"redirects": {
"0": {
"request": {
"ssl": false,
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A"
},
"host": "ultrasurf.us",
"path": "/",
"port": 80
},
"full_url": "http://somewebsite.com/",
"response": {
"status": 200,
"headers": {
},
"reason": "Moved",
"body": ""
}
},
"1": {
"request": {
"path": "/",
"host": "somewebsite.com",
"port": 443,
"method": "GET",
"ssl": true
},
"full_url": "https://somewebsite.com/",
"response": {
"status": 200,
"headers": {
},
"reason": "OK",
"body": ""
}
}
},
"redirect_loop": false,
"redirect_count": 1,
"full_url": "https://somewebsite.com/"
}
},
"file_name": "file.csv",
"file_metadata": {},
"file_comments": [],
"dns": {
"www.somewebsite.com": [
{
"domain": "www.somewebsite.com",
"response2": null,
"response1": "",
"request": "",
"response1-ips": [
"1.2.3.4",
"1.2.3.5",
"1.2.3.6"
],
"nameserver": "8.8.8.8"
},
{
"domain": "www.somewebsite.com",
"request": "",
"nameserver": "8.8.4.4.",
"response1": null
},
{
"domain": "www.somewebsite.com",
"response2": null,
"response1": "",
"request": "",
"response1-ips": [
"1.2.3.4",
"1.2.3.5",
"1.2.3.6"
],
"nameserver": "8.8.8.8"
}
]
},
"total_time": 3269.9958488941193,
"url_metadata": {
"http://somewebsite.com/": {
"category_code": "ANON",
"secondary_category_code": "",
"notes": "",
"source": "source",
"date_added": "1111-01-11",
"subcategory_code": ""
}
},
"traceroute.tcp": {
"www.somewebsite.com": {
"dest_name": "www.somewebsite.com",
"hops": [
{
"index": 1,
"asn": null,
"probes": [
{
"anno": "",
"ip": "1.2.3.4",
"name": "name",
"rtt": 234.093
},
{
"anno": "",
"ip": "1.2.3.4",
"name": "name",
"rtt": 234.104
},
{
"anno": "",
"ip": "1.2.3.4",
"name": "name",
"rtt": 234.196
}
]
},
{
"index": 2,
"asn": null,
"probes": [
{
"anno": "",
"ip": "1.2.3.5",
"name": "name",
"rtt": 344.378
},
{
"anno": "",
"ip": "1.2.3.5",
"name": "name",
"rtt": 231.048
},
{
"anno": "",
"ip": "1.2.3.5",
"name": "name",
"rtt": 231.046
}
]
}
],
"forcefully_terminated": false,
"time_elapsed": 7,
"method": "tcp",
"dest_ip": "1.2.3.5"
}
}
}
]
}
This section contains notes on the attributes of the baseline files.
This part contains some metadata information about the experiment
- as_number: The Autonomous System Number for the ISP in which the centinel is run.
- server_time: The server time when running the centinel.
- ip: The ip address of the centinel (CIDR version).
- schedule_name: The name of the schedule that the centinel is running.
- as_owner: The Autonomous System owner name for the ISP in which the centinel is run.
- country: The Alpha2 Code for the country in which the centinel is being run.
- client_time: The client time when running the centinel.
- time_taken: The time taken to complete an experiment.
- vpn_provider: The name of the vpn provider that centinel is using.
- centinel_version: The version of the centinel.
The main experiment which consists of tls, tcp, dns, http, url_metadata and traceroute parts. We also store the filename of the csv we use for our experiment.
- cert: The .pem format of the websites certificate.
- fingerprint: The fingerprint of the certificate.
- fingerprint_error: The error that occurred when trying to get the fingerprint.
- tls_error: The error that occurred when trying to get the certificate.
- ip: ip address that we try to connect to.
- host: The hostname we try to connect to.
- port: The port we use for the connection.
- success: Indicates the success of the tcp connection.
- time: The time it takes for the connection to be set up.
- failure: If failure occurred, what was the reason.
- domain: The domain name we try to resolve.
- response1: The b64encoded of the first dns response.
- response2: The b64encode of the second dns response.
- request: The b64encode of our dns request.
- nameserver: The ip of the nameserver we used for name resolution.
- response1-ips: The list of ips returned for the first response.
- response2-ips: The list of ips returned for the second response.
- dest_name: The domain we execute traceroute for.
- method: The method used: TCP/UDP.
- time_elapsed: Time it took in seconds.
- dest_ip: The ip address of the destination.
- forcefully_terminated: true or false.
- total_hops: Total number of hops.
- meaningful_hops: The number of hops that could be parsed correctly.
- unparseable_lines: lines that could not be parsed in the old parseer.
- hops: A list of traceroute hops.
- raw: This was available in older version of the centinel. older versions of traceroute were parsed with a different parser the parser missed some lines so we kept the raw text.
- index: The index number of the hop.
- asn: The ASN associated with the hop.
- probes: A list of 3 probes:
- ip: IP address of the probe.
- name: The name of the probe.
- rtt: The Round-Trip-Time of the probe.
- anno: It shows that waiting for the ICMP "Time exceeded" (TTL expired) message timed out.
- request:
- headers: Stores the reques headers.
- host: The host we send an HTTP request to.
- method: GET ...
- ssl: Indicates if we sent it using SSL.
- path: The path part of the hostname.
- port: The port used for the connection.
- full_url: The full url containing the hostname and the path.
- response:
- status: Status code of the HTTP request.
- headers: Response header.
- reason: It’s the reason provided next to the code provided (e.g. 303 Moved Temporarily).
- body: HTML Body of the response or "body.b64".
- failure: an Error text.
- redirects:
A list of redirects.
- request:
- headers: Stores the requests headers.
- host: The host we send an HTTP request to.
- method: GET ...
- ssl: Indicates if we sent it using SSL.
- path: The path part of the hostname.
- port: The port used for the connection.
- full_url: The full url containing hostname and the path.
- response:
- status: Status code of the HTTP request.
- headers: Response header.
- reason: It’s the reason provided next to the code provided (e.g. 303 Moved Temporarily)
- body: HTML Body of the response or "body.b64".
- failure: an Error text.
- redirect_count: The number of redirects.
- redirect_loop: Indicates whether a loop has occured in the redirects.
- request:
This part is different among the different versions of centinel and it's metadata provided by CitizenLab.
- category_code: Category codes of Citizen lab.
- secondary_category_code: Category codes of Citizen lab.
- notes:
- source: The source of the list. (ex: Citizen Lab)
- date_added: The date that the url was added to the list.
- subcategory_code:
- theme:
- generalStatus:
- urlCategories:
- rank: The rank of the url in alexa.
- protocol:
- nickname: