use local file from command line and get latest version without githu…

…b api which has rate limits

biobtree.go

@@ -28,9 +28,14 @@ var config *conf.Conf
 var defaultDataset = "uniprot,go,eco,efo,hgnc,chebi,taxonomy,interpro,ensembl"
 
 //var defaultDataset = "uniprot,go,eco,efo,hgnc,chebi,taxonomy,interpro,hmdb,literature_mappings,chembl,ensembl"
-
 //var defaultDataset = "efo"
 
+var allDatasets = []string{"uniprot", "go", "eco", "efo", "hgnc", "chebi", "taxonomy",
+	"interpro", "hmdb", "literature_mappings", "ensembl",
+	"uniparc", "uniref50", "uniref90", "uniref100", "my_data", "uniprot_unreviewed",
+	"ensembl_bacteria", "ensembl_fungi", "ensembl_metazoa", "ensembl_plants", "ensembl_protists",
+	"chembl_document", "chembl_assay", "chembl_activity", "chembl_molecule", "chembl_target", "chembl_target_component", "chembl_cell_line"}
+
 func main() {
 
 	app := cli.NewApp()
@@ -99,6 +104,14 @@ func main() {
 		},
 	}
 
+	// add dataset local flags
+	for _, dataset := range allDatasets {
+		app.Flags = append(app.Flags, cli.StringFlag{
+			Name:   dataset + ".file",
+			Hidden: true,
+		})
+	}
+
 	app.Commands = []cli.Command{
 		{
 			Name:  "start",
@@ -268,6 +281,14 @@ func runUpdateCommand(c *cli.Context) error {
 		runtime.GOMAXPROCS(cpu)
 	}
 
+	// check local file path settings
+	for _, dset := range allDatasets {
+		if len(c.GlobalString(dset+".file")) > 0 {
+			config.Dataconf[dset]["path"] = c.GlobalString(dset + ".file")
+			config.Dataconf[dset]["useLocalFile"] = "yes"
+		}
+	}
+
 	update.NewDataUpdate(d, ts, sp, spatterns, config, chunkIdxx).Update()
 
 	elapsed := time.Since(start)

conf/conf.go

@@ -17,7 +17,7 @@ import (
 var fileBufSize = 65536
 var channelOverflowCap = 100000
 
-const latestReleasePath = "https://api.github.com/repos/tamerh/biobtree/releases/latest"
+const latestReleasePath = "https://github.com/tamerh/biobtree/releases/latest"
 
 type Conf struct {
 	Appconf               map[string]string
@@ -31,10 +31,6 @@ type Conf struct {
 	versionTag            string
 }
 
-type gitLatestRelease struct {
-	Tag string `json:"tag_name"`
-}
-
 type gitContent struct {
 	Name   string `json:"name"`
 	RawURL string `json:"download_url"`
@@ -342,27 +338,12 @@ func (c *Conf) checkForNewVersion() {
 		log.Println("Warn: Versions data could not recieved.")
 		return
 	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		log.Println("Warn: Versions data could not recieved from github api.")
-		return
-	}
 
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		log.Println("Warn: Versions data could not recieved from github api.")
-		return
-	}
-
-	latestRelease := gitLatestRelease{}
-	if err := json.Unmarshal(data, &latestRelease); err != nil {
-		log.Println("Warn: Versions data could not parsed.")
-		return
-	}
+	finalURL := resp.Request.URL.String()
+	splitteURL := strings.Split(finalURL, "/")
 
-	if latestRelease.Tag != c.versionTag {
-		log.Println("Warning: There is a new biobtree version available to download")
+	if len(splitteURL) > 0 && splitteURL[len(splitteURL)-1] != c.versionTag {
+		log.Println("New biobtree version " + splitteURL[len(splitteURL)-1] + " is available to download")
 	}
 
 }

conf/source.dataset.json

@@ -140,10 +140,9 @@
   "my_data": {
     "id": "14",
     "name": "my_data",
-    "path": "./my_data/my_data.xml.gz",
+    "path": "",
     "url": "",
-    "useLocalFile": "yes",
-    "active": "false"
+    "useLocalFile": "yes"
   },
   "chembl_document": {
     "name": "chembl_document",

update/update.go

@@ -329,12 +329,16 @@ func (d *DataUpdate) Update() (uint64, uint64) {
 			go g.update()
 			break
 		case "my_data":
-			if config.Dataconf[data]["active"] == "true" {
+
+			if len(config.Dataconf[data]["path"]) > 0 {
 				d.wg.Add(1)
 				u := uniprot{source: data, d: d}
 				d.datasets2 = append(d.datasets2, data)
 				go u.update()
+			} else {
+				log.Fatal("Missing source path for my_data ")
 			}
+
 			break
 		case "literature_mappings":
 			d.wg.Add(1)