Adding support for GHE

README.md

@@ -28,6 +28,7 @@ You have two options. I'd recommend the first as it will give you access to the
 4. Bring up the stack: `docker-compose up`
 5. Open up http://localhost:8080/
 
+
 ### via Go get
 
 _Note_: this method does not include the shhgit web interface
@@ -40,7 +41,8 @@ _Note_: this method does not include the shhgit web interface
 
 shhgit can work in two ways: consuming the public APIs of GitHub, Gist, GitLab and BitBucket  or by processing files in a local directory.
 
-By default, shhgit will run in the former 'public mode'. For GitHub and Gist, you will need to obtain and provide an access token (see [this guide](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line); it doesn't require any scopes or permissions. And then place it under `github_access_tokens` in `config.yaml`). GitLab and BitBucket do not require any API tokens.
+By default, shhgit will run in the former 'public mode'. For GitHub and Gist, you will need to obtain and provide an access token (see [this guide](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line); it doesn't require any scopes or permissions. And then place it under `github_access_tokens` in `config.yaml`).
+For GitHub Enterprise (GHE) a token is also needed, but rate limiting is configurable and your token might need read access to the repositories in questions depending on the configuration of your GHE instance. GitLab and BitBucket do not require any API tokens.
 
 You can also forgo the signatures and use shhgit with your own custom search query, e.g. to find all AWS keys you could use `shhgit --search-query AWS_ACCESS_KEY_ID=AKIA`. And to run in local mode (and perhaps integrate in to your CI pipelines) you can pass the `--local` flag (see usage below).
 
@@ -87,6 +89,7 @@ The `config.yaml` file has 7 elements. A [default is provided](https://github.co
 github_access_tokens: # provide at least one token
   - 'token one'
   - 'token two'
+github_enterprise_url: '' # url to your github enterprise (optional)
 webhook: '' # URL to a POST webhook.
 webhook_payload: '' # Payload to POST to the webhook URL
 blacklisted_strings: [] # list of strings to ignore

config.yaml

@@ -1,5 +1,6 @@
 github_access_tokens:
   - ''
+github_enterprise_url: ''
 webhook: '' # URL to which the payload is POSTed
 
 # This default payload will work for Slack and MatterMost.

core/config.go

@@ -13,6 +13,7 @@ import (
 
 type Config struct {
 	GitHubAccessTokens           []string          `yaml:"github_access_tokens"`
+	GitHubEnterpriseUrl          string            `yaml:"github_enterprise_url"`
 	Webhook                      string            `yaml:"webhook,omitempty"`
 	WebhookPayload               string            `yaml:"webhook_payload,omitempty"`
 	BlacklistedStrings           []string          `yaml:"blacklisted_strings"`

core/git.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	"context"
+	"net/url"
 	"strings"
 	"time"
 
@@ -24,22 +25,33 @@ type GitResource struct {
 	Url  string
 }
 
-func CloneRepository(session *Session, url string, dir string) (*git.Repository, error) {
+func CloneRepository(session *Session, rawUrl string, dir string) (*git.Repository, error) {
 	timeout := time.Duration(*session.Options.CloneRepositoryTimeout) * time.Second
 	localCtx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
+
+  if len(session.Config.GitHubEnterpriseUrl) > 0 {
+		githubUrl, err := url.Parse(rawUrl)
+		if err != nil {
+			return nil, err
+		}
+
+		userInfo := url.User(session.Config.GitHubAccessTokens[0])
+		githubUrl.User = userInfo
+		rawUrl = githubUrl.String()
+	}
 
-	session.Log.Debug("[%s] Cloning in to %s", url, strings.Replace(dir, *session.Options.TempDirectory, "", -1))
+	session.Log.Debug("[%s] Cloning in to %s", rawUrl, strings.Replace(dir, *session.Options.TempDirectory, "", -1))
 	repository, err := git.PlainCloneContext(localCtx, dir, false, &git.CloneOptions{
 		Depth:             1,
 		RecurseSubmodules: git.NoRecurseSubmodules,
-		URL:               url,
+		URL:               rawUrl,
 		SingleBranch:      true,
 		Tags:              git.NoTags,
 	})
 
 	if err != nil {
-		session.Log.Debug("[%s] Cloning failed: %s", url, err.Error())
+		session.Log.Debug("[%s] Cloning failed: %s", rawUrl, err.Error())
 		return nil, err
 	}
 

core/github.go

@@ -50,7 +50,7 @@ func GetRepositories(session *Session) {
 				GetSession().Log.Warn("Error getting GitHub events... trying again", err)
 			}
 
-			if opt.Page == 0 {
+			if opt.Page == 0 && resp.Rate.Limit > 0 {
 				session.Log.Warn("Token %s[..] has %d/%d calls remaining.", client.Token[:10], resp.Rate.Remaining, resp.Rate.Limit)
 			}
 
@@ -152,10 +152,11 @@ func GetRepository(session *Session, id int64) (*github.Repository, error) {
 	repo, resp, err := client.Repositories.GetByID(session.Context, id)
 
 	if err != nil {
+		session.Log.Warn("Got error %s", err)
 		return nil, err
 	}
 
-	if resp.Rate.Remaining <= 1 {
+	if resp.Rate.Remaining <= 1 && resp.Rate.Limit > 0 {
 		session.Log.Warn("Token %s[..] rate limited. Reset at %s", client.Token[:10], resp.Rate.Reset)
 		client.RateLimitedUntil = resp.Rate.Reset.Time
 	}

core/session.go

@@ -5,8 +5,10 @@ import (
 	"encoding/csv"
 	"fmt"
 	"math/rand"
+	"net/url"
 	"os"
 	"runtime"
+	"strings"
 	"sync"
 	"time"
 
@@ -65,7 +67,23 @@ func (s *Session) InitGitHubClients() {
 			ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
 			tc := oauth2.NewClient(s.Context, ts)
 
-			client := github.NewClient(tc)
+		client := github.NewClient(tc)
+		enterpriseUrl := s.Config.GitHubEnterpriseUrl
+
+		if len(enterpriseUrl) > 0 {
+			baseEndpoint, err := url.Parse(enterpriseUrl)
+
+			if err != nil {
+				s.Log.Warn("Failed to parse GitHubEnterpriseUrl %s[..]: %s", enterpriseUrl, err)
+				return
+			}
+
+			if !strings.HasSuffix(baseEndpoint.Path, "/api/v3/") {
+				baseEndpoint.Path += "api/v3/"
+			}
+
+			client.BaseURL = baseEndpoint
+		}
 
 			client.UserAgent = fmt.Sprintf("%s v%s", Name, Version)
 			_, _, err := client.Users.Get(s.Context, "")