-
Notifications
You must be signed in to change notification settings - Fork 5
/
main.go
97 lines (77 loc) · 2.27 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
package main
import (
"bufio"
"dhtc/cache"
"dhtc/config"
"dhtc/db"
dhtcclient "dhtc/dhtc-client"
"dhtc/notifier"
"dhtc/ui"
"fmt"
"github.com/ostafen/clover/v2"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
telegram "gopkg.in/telebot.v3"
"os"
)
func ReadFileLines(filePath string) []string {
var rVal []string
file, err := os.Open(filePath)
if err != nil {
log.Error().Err(err)
return rVal
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
rVal = append(rVal, scanner.Text())
}
return rVal
}
func crawl(configuration *config.Configuration, database *clover.DB, bot *telegram.Bot) {
indexerAddrs := []string{"0.0.0.0:0"}
interruptChan := make(chan os.Signal, 1)
bootstrapNodes := ReadFileLines(configuration.BootstrapNodeFile)
if len(bootstrapNodes) == 0 {
log.Warn().Msg("No bootstrap nodes found in '" + configuration.BootstrapNodeFile + "'.")
log.Info().Msg("Using default bootstrap nodes.")
bootstrapNodes = []string{
"router.bittorrent.com:6881", "dht.transmissionbt.com:6881", "dht.libtorrent.org:25401",
}
}
trawlingManager := dhtcclient.NewManager(bootstrapNodes, indexerAddrs, 1, configuration.MaxNeighbors)
metadataSink := dhtcclient.NewSink(configuration.DrainTimeout, configuration.MaxLeeches)
for stopped := false; !stopped; {
select {
case result := <-trawlingManager.Output():
hash := result.InfoHash()
if !cache.InfoHashCache.Contains(hash) {
cache.InfoHashCache.Add(hash)
metadataSink.Sink(result)
}
case md := <-metadataSink.Drain():
if db.InsertMetadata(configuration, database, md) {
fmt.Println("\t + Added:", md.Name)
db.CheckWatches(configuration, database, md, bot)
}
case <-interruptChan:
trawlingManager.Terminate()
stopped = true
}
}
}
func main() {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
cfg := config.ParseArguments()
database := db.OpenDatabase(cfg)
cache.PopulateInfoHashCacheFromDatabase(database)
db.AddToBlacklist(database, ReadFileLines(cfg.NameBlacklist), "0")
db.AddToBlacklist(database, ReadFileLines(cfg.FileBlacklist), "1")
if !cfg.OnlyWebServer {
bot := notifier.SetupTelegramBot(cfg)
for i := 0; i < cfg.CrawlerThreads; i++ {
go crawl(cfg, database, bot)
}
}
ui.RunWebServer(cfg, database)
}