Merge branch 'checkforipviolations-step2' into 'master'

Checkforipviolations step 2

Closes NebulousLabs#3197

See merge request NebulousLabs/Sia!3236

modules/renter.go

@@ -131,13 +131,18 @@ type HostDBEntry struct {
 	HistoricUptime   time.Duration `json:"historicuptime"`
 	ScanHistory      HostDBScans   `json:"scanhistory"`
 
+	// Measurements that are taken whenever we interact with a host.
 	HistoricFailedInteractions     float64 `json:"historicfailedinteractions"`
 	HistoricSuccessfulInteractions float64 `json:"historicsuccessfulinteractions"`
 	RecentFailedInteractions       float64 `json:"recentfailedinteractions"`
 	RecentSuccessfulInteractions   float64 `json:"recentsuccessfulinteractions"`
 
 	LastHistoricUpdate types.BlockHeight
 
+	// Measurements related to the IP subnet mask.
+	IPNets          []string
+	LastIPNetChange time.Time
+
 	// The public key of the host, stored separately to minimize risk of certain
 	// MitM based vulnerabilities.
 	PublicKey types.SiaPublicKey `json:"publickey"`

modules/renter/hostdb/hostdb.go

@@ -9,15 +9,14 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"sync"
 	"time"
 
-	"gitlab.com/NebulousLabs/Sia/build"
 	"gitlab.com/NebulousLabs/Sia/modules"
 	"gitlab.com/NebulousLabs/Sia/modules/renter/hostdb/hosttree"
 	"gitlab.com/NebulousLabs/Sia/persist"
 	"gitlab.com/NebulousLabs/Sia/types"
-	"gitlab.com/NebulousLabs/fastrand"
 	"gitlab.com/NebulousLabs/threadgroup"
 )
 
@@ -231,34 +230,38 @@ func (hdb *HostDB) AverageContractPrice() (totalPrice types.Currency) {
 // CheckForIPViolations accepts a number of host public keys and returns the
 // ones that violate the rules of the addressFilter.
 func (hdb *HostDB) CheckForIPViolations(hosts []types.SiaPublicKey) []types.SiaPublicKey {
-	// Shuffle the hosts to non-deterministically decide which host is bad. The
-	// reason being that the address which is passed to the filter first, has
-	// priority over addresses which are passed in later. So if address A and B
-	// together violate the rules, passing B first will result in A being
-	// considered a bad host and vice versa.
-	if build.Release != "testing" {
-		fastrand.Shuffle(len(hosts), func(i, j int) { hosts[i], hosts[j] = hosts[j], hosts[i] })
-	}
-
-	// Create a filter.
-	filter := hosttree.NewFilter(hdb.deps.Resolver())
-
+	var entries []modules.HostDBEntry
 	var badHosts []types.SiaPublicKey
+
+	// Get the entries which correspond to the keys.
 	for _, host := range hosts {
-		// Get the host from the db.
-		node, exists := hdb.hostTree.Select(host)
+		entry, exists := hdb.hostTree.Select(host)
 		if !exists {
 			// A host that's not in the hostdb is bad.
 			badHosts = append(badHosts, host)
 			continue
 		}
+		entries = append(entries, entry)
+	}
+
+	// Sort the entries by the amount of time they have occupied their
+	// corresponding subnets. This is the order in which they will be passed
+	// into the filter which prioritizes entries which are passed in earlier.
+	// That means 'younger' entries will be replaced in case of a violation.
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].LastIPNetChange.Before(entries[j].LastIPNetChange)
+	})
+
+	// Create a filter and apply it.
+	filter := hosttree.NewFilter(hdb.deps.Resolver())
+	for _, entry := range entries {
 		// Check if the host violates the rules.
-		if filter.Filtered(node.NetAddress) {
-			badHosts = append(badHosts, host)
+		if filter.Filtered(entry.NetAddress) {
+			badHosts = append(badHosts, entry.PublicKey)
 			continue
 		}
 		// If it didn't then we add it to the filter.
-		filter.Add(node.NetAddress)
+		filter.Add(entry.NetAddress)
 	}
 	return badHosts
 }

modules/renter/hostdb/hostdb_test.go

@@ -634,16 +634,33 @@ func TestCheckForIPViolations(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Scan the entries. entry1 should be the 'oldest' and entry3 the
+	// 'youngest'. This also inserts the entries into the hosttree.
+	hdbt.hdb.managedScanHost(entry1)
+	time.Sleep(time.Millisecond)
+	hdbt.hdb.managedScanHost(entry2)
+	time.Sleep(time.Millisecond)
+	hdbt.hdb.managedScanHost(entry3)
+	time.Sleep(time.Millisecond)
+
+	// Scan all the entries again in reversed order. This is a sanity check. If
+	// the code works as expected this shouldn't do anything since the
+	// hostnames didn't change. If it doesn't, it will update the timestamps
+	// and the following checks will fail.
+	time.Sleep(time.Millisecond)
+	hdbt.hdb.managedScanHost(entry3)
+	time.Sleep(time.Millisecond)
+	hdbt.hdb.managedScanHost(entry2)
+	time.Sleep(time.Millisecond)
+	hdbt.hdb.managedScanHost(entry1)
+
 	// Add entry1 and entry2. There should be no violation.
-	hdbt.hdb.hostTree.Insert(entry1)
-	hdbt.hdb.hostTree.Insert(entry2)
 	badHosts := hdbt.hdb.CheckForIPViolations([]types.SiaPublicKey{entry1.PublicKey, entry2.PublicKey})
 	if len(badHosts) != 0 {
 		t.Errorf("Got %v violations, should be 0", len(badHosts))
 	}
 
 	// Add entry3. It should cause a violation for entry 3.
-	hdbt.hdb.hostTree.Insert(entry3)
 	badHosts = hdbt.hdb.CheckForIPViolations([]types.SiaPublicKey{entry1.PublicKey, entry2.PublicKey, entry3.PublicKey})
 	if len(badHosts) != 1 {
 		t.Errorf("Got %v violations, should be 1", len(badHosts))
@@ -663,12 +680,13 @@ func TestCheckForIPViolations(t *testing.T) {
 	}
 
 	// Calling CheckForIPViolations with entry 3 as the first argument should
-	// result in 2 bad hosts. entry1 and entry2.
+	// result in 1 bad host, entry3. The reason being that entry3 is the
+	// 'youngest' entry.
 	badHosts = hdbt.hdb.CheckForIPViolations([]types.SiaPublicKey{entry3.PublicKey, entry1.PublicKey, entry2.PublicKey})
-	if len(badHosts) != 2 {
+	if len(badHosts) != 1 {
 		t.Errorf("Got %v violations, should be 1", len(badHosts))
 	}
-	if len(badHosts) > 1 && (!bytes.Equal(badHosts[0].Key, entry1.PublicKey.Key) || !bytes.Equal(badHosts[1].Key, entry2.PublicKey.Key)) {
+	if len(badHosts) > 1 || !bytes.Equal(badHosts[0].Key, entry3.PublicKey.Key) {
 		t.Error("Hdb returned violation for wrong host")
 	}
 }

modules/renter/hostdb/hosttree/addressfilter.go

@@ -8,8 +8,12 @@ import (
 )
 
 const (
-	ipv4FilterRange = 24
-	ipv6FilterRange = 54
+	// IPv4FilterRange is the number of bits within an IP address (starting
+	// from the left) which have to be unique for the host not to be filtered.
+	IPv4FilterRange = 24
+	// IPv6FilterRange is the number of bits within an IP address (starting
+	// from the left) which have to be unique for the host not to be filtered.
+	IPv6FilterRange = 54
 )
 
 // Filter filters host addresses which belong to the same subnet to
@@ -44,9 +48,9 @@ func (af *Filter) Add(host modules.NetAddress) {
 		// Set the filterRange according to the type of IP address.
 		var filterRange int
 		if ip.To4() != nil {
-			filterRange = ipv4FilterRange
+			filterRange = IPv4FilterRange
 		} else {
-			filterRange = ipv6FilterRange
+			filterRange = IPv6FilterRange
 		}
 		// Get the subnet.
 		_, ipnet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", ip.String(), filterRange))
@@ -83,10 +87,11 @@ func (af *Filter) Filtered(host modules.NetAddress) bool {
 		// Set the filterRange according to the type of IP address.
 		var filterRange int
 		if ip.To4() != nil {
-			filterRange = ipv4FilterRange
+			filterRange = IPv4FilterRange
 		} else {
-			filterRange = ipv6FilterRange
+			filterRange = IPv6FilterRange
 		}
+
 		// Get the subnet.
 		_, ipnet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", ip.String(), filterRange))
 		if err != nil {

modules/renter/hostdb/scan.go

@@ -14,9 +14,30 @@ import (
 	"gitlab.com/NebulousLabs/Sia/crypto"
 	"gitlab.com/NebulousLabs/Sia/encoding"
 	"gitlab.com/NebulousLabs/Sia/modules"
+	"gitlab.com/NebulousLabs/Sia/modules/renter/hostdb/hosttree"
 	"gitlab.com/NebulousLabs/fastrand"
 )
 
+// equalIPNets checks if two slices of IP subnets contain the same subnets.
+func equalIPNets(ipNetsA, ipNetsB []string) bool {
+	// Check the length first.
+	if len(ipNetsA) != len(ipNetsB) {
+		return false
+	}
+	// Create a map of all the subnets in ipNetsA.
+	mapNetsA := make(map[string]struct{})
+	for _, subnet := range ipNetsA {
+		mapNetsA[subnet] = struct{}{}
+	}
+	// Make sure that all the subnets from ipNetsB are in the map.
+	for _, subnet := range ipNetsB {
+		if _, exists := mapNetsA[subnet]; !exists {
+			return false
+		}
+	}
+	return true
+}
+
 // queueScan will add a host to the queue to be scanned. The host will be added
 // at a random position which means that the order in which queueScan is called
 // is not necessarily the order in which the hosts get scanned. That guarantees
@@ -251,6 +272,38 @@ func (hdb *HostDB) updateEntry(entry modules.HostDBEntry, netErr error) {
 	}
 }
 
+// managedLookupIPNets returns string representations of the CIDR subnets
+// used by the host.  In case of an error we return nil. We don't really care
+// about the error because we don't update host entries if we are offline
+// anyway. So if we fail to resolve a hostname, the problem is not related to
+// us.
+func (hdb *HostDB) managedLookupIPNets(address modules.NetAddress) (ipNets []string, err error) {
+	// Lookup the IP addresses of the host.
+	addresses, err := hdb.deps.Resolver().LookupIP(address.Host())
+	if err != nil {
+		return nil, err
+	}
+	// Get the subnets of the addresses.
+	for _, ip := range addresses {
+		// Set the filterRange according to the type of IP address.
+		var filterRange int
+		if ip.To4() != nil {
+			filterRange = hosttree.IPv4FilterRange
+		} else {
+			filterRange = hosttree.IPv6FilterRange
+		}
+
+		// Get the subnet.
+		_, ipnet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", ip.String(), filterRange))
+		if err != nil {
+			return nil, err
+		}
+		// Add the subnet to the host.
+		ipNets = append(ipNets, ipnet.String())
+	}
+	return
+}
+
 // managedScanHost will connect to a host and grab the settings, verifying
 // uptime and updating to the host's preferences.
 func (hdb *HostDB) managedScanHost(entry modules.HostDBEntry) {
@@ -266,14 +319,26 @@ func (hdb *HostDB) managedScanHost(entry modules.HostDBEntry) {
 		netAddr = modules.NetAddress(fmt.Sprintf("127.0.0.1:%s", port))
 	}
 
+	// Resolve the host's used subnets and update the timestamp if they
+	// changed. We only update the timestamp if resolving the ipNets was
+	// successful.
+	ipNets, err := hdb.managedLookupIPNets(entry.NetAddress)
+	if err == nil && !equalIPNets(ipNets, entry.IPNets) {
+		entry.IPNets = ipNets
+		entry.LastIPNetChange = time.Now()
+	}
+	if err != nil {
+		hdb.log.Debugln("mangedScanHost: failed to look up IP nets", err)
+	}
+
 	// Update historic interactions of entry if necessary
 	hdb.mu.RLock()
 	updateHostHistoricInteractions(&entry, hdb.blockHeight)
 	hdb.mu.RUnlock()
 
 	var settings modules.HostExternalSettings
 	var latency time.Duration
-	err := func() error {
+	err = func() error {
 		timeout := hostRequestTimeout
 		hdb.mu.RLock()
 		if len(hdb.initialScanLatencies) > minScansForSpeedup {