Fixed maintenance process and github bugs

src/github.js

@@ -1,5 +1,4 @@
 import GithubApi from 'github'
-import unionBy from 'lodash/unionBy'
 import toPairs from 'lodash/toPairs'
 import bluebird from 'bluebird'
 import {
@@ -81,7 +80,7 @@ export default class GithubHelper {
   async getTree(time, sha, blobs, recursive) {
     try {
       await this.checkValidity()
-      const { tree } = (await this.client
+      const { tree, sha: treeSha } = (await this.client
         .gitdata
         .getTree({
           ...this.config,
@@ -90,11 +89,11 @@ export default class GithubHelper {
         })).data
       if (!recursive) {
         if (time.deleteDate) {
-          return tree.filter(item => !item.path.includes(time.deleteDate)).concat(blobs)
+          return { tree: tree.filter(item => !item.path.includes(time.deleteDate)).concat(blobs) }
         }
-        return tree.concat(blobs)
+        return { tree: tree.concat(blobs) }
       }
-      return unionBy(blobs, tree, 'path')
+      return { tree: blobs, base_tree: treeSha }
     } catch (e) {
       return Promise.reject(e)
     }
@@ -103,11 +102,12 @@ export default class GithubHelper {
   async createTree(tree) {
     try {
       await this.checkValidity()
+
       return (await this.client
         .gitdata
         .createTree({
           ...this.config,
-          tree,
+          ...tree,
         })).data.sha
     } catch (e) {
       return Promise.reject(e)
@@ -163,7 +163,7 @@ export default class GithubHelper {
       await this
         .createBlobs(data, recursive)
         .then(blobs => this.getTree(data.time, headSha, blobs, recursive))
-        .then(tree => this.createTree(tree))
+        .then(tree => this.createTree(tree, recursive))
         .then(createdTree => this.createCommit(createdTree, data.time, headSha, message))
         .then(commit => this.updateReference(commit))
 

src/maintenance.js

@@ -594,7 +594,7 @@ export class Maintenance {
 
     this.redisClient = redisStore
     this.config = config
-    this.twitterClient = config && config.TWITTER_CONFIG ?
+    this.twitterClient = config && _.has(config, 'TWITTER_CONFIG.consumer_key') ?
       new TwitterHelper(config.TWITTER_CONFIG, config.LIST_ID) : null
     this.options = opts
   }

test/github.test.js

@@ -3,10 +3,10 @@ import path from 'path'
 import MockApi from './helpers/api-mock'
 import GithubHelper from '../src/github'
 import {
-    testConfig,
+  testConfig,
 } from './util/test-util'
 import {
-    nativeClone,
+  nativeClone,
 } from '../src/util'
 
 jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000
@@ -143,39 +143,38 @@ describe('Github helper methods', () => {
     describe('getTree', () => {
       test('Retrieves tree', async () => {
         const blobs = ['data/DATE.json', '_posts/DATE--tweets.md']
-                    .map(x => ({
-                      sha: 'foo',
-                      url: 'foo',
-                      path: x.replace('DATE', '2017-02-02'),
-                      type: 'blob',
-                      mode: '100644',
-                    }))
+          .map(x => ({
+            sha: 'foo',
+            url: 'foo',
+            path: x.replace('DATE', '2017-02-02'),
+            type: 'blob',
+            mode: '100644',
+          }))
         const tree = await githubClient.getTree(data.time, 'foo', blobs)
 
-        expect(tree).toEqual(expect.arrayContaining(blobs))
-        expect(tree).toHaveLength(11)
+        expect(tree).toEqual({ tree: expect.arrayContaining(blobs) })
+        expect(tree.tree).toHaveLength(11)
       })
 
       test('Retrieves tree and omits files from X date when deleteDate set', async () => {
         const blobs = ['data/DATE.json', '_posts/DATE--tweets.md']
-                    .map(x => ({
-                      sha: 'foo',
-                      url: 'foo',
-                      path: x.replace('DATE', '2017-02-02'),
-                      type: 'blob',
-                      mode: '100644',
-                    }))
+          .map(x => ({
+            sha: 'foo',
+            url: 'foo',
+            path: x.replace('DATE', '2017-02-02'),
+            type: 'blob',
+            mode: '100644',
+          }))
         const locData = nativeClone(data)
         locData.time.deleteDate = '2017-06-01'
         const tree = await githubClient.getTree(locData.time, 'foo', blobs)
 
-        expect(tree).toEqual(expect.arrayContaining(blobs))
-        expect(tree).toEqual(expect.arrayContaining(blobs))
-        expect(tree.map(item => item.path)).not.toContain('data/2017-06-01.json')
-        expect(tree).toHaveLength(9)
+        expect(tree).toEqual({ tree: expect.arrayContaining(blobs) })
+        expect(tree.tree.map(item => item.path)).not.toContain('data/2017-06-01.json')
+        expect(tree.tree).toHaveLength(9)
       })
 
-      test('Retrieves tree with overwritten files when self-updating class option set', async () => {
+      test('Retrieves tree with new files when self-updating class option set', async () => {
         mockApi.options = {
           recursive: true,
         }
@@ -185,19 +184,19 @@ describe('Github helper methods', () => {
           'historical-users',
           'historical-users-filtered',
         ]
-                    .map(x => ({
-                      sha: 'foo',
-                      url: 'foo',
-                      path: `data/${x}.json`,
-                      type: 'blob',
-                      mode: '100644',
-                    }))
+          .map(x => ({
+            sha: 'foo',
+            url: 'foo',
+            path: `data/${x}.json`,
+            type: 'blob',
+            mode: '100644',
+          }))
 
         const tree = await githubClient.getTree(data.time, 'foo', blobs, true)
 
-        expect(tree).toEqual(expect.arrayContaining(blobs))
-        expect(tree.filter(item => item.path.includes('historical-users-filtered'))).toHaveLength(1)
-        expect(tree).toHaveLength(13)
+        expect(tree).toEqual({ tree: blobs, base_tree: expect.any(String) })
+        expect(tree.tree.filter(item => item.path.includes('historical-users-filtered'))).toHaveLength(1)
+        expect(tree.tree).toHaveLength(4)
       })
     })
 
@@ -208,22 +207,46 @@ describe('Github helper methods', () => {
           'historical-users',
           'historical-users-filtered',
         ]
-                    .map(x => ({
-                      sha: 'foo',
-                      url: 'foo',
-                      path: `data/${x}.json`,
-                      type: 'blob',
-                      mode: '100644',
-                    }))
+          .map(x => ({
+            sha: 'foo',
+            url: 'foo',
+            path: `data/${x}.json`,
+            type: 'blob',
+            mode: '100644',
+          }))
 
-        const createdTree = await githubClient.createTree(blobs)
+        const createdTree = await githubClient.createTree({ tree: blobs })
 
         expect(mockFns.createTree).toBeCalledWith(expect.objectContaining({
           ...githubClient.config,
           tree: blobs,
         }))
         expect(createdTree).toEqual(expect.any(String))
       })
+
+      test('Creates tree on top of base_tree', async () => {
+        const blobs = ['users',
+          'users-filtered',
+          'historical-users',
+          'historical-users-filtered',
+        ]
+          .map(x => ({
+            sha: 'foo',
+            url: 'foo',
+            path: `data/${x}.json`,
+            type: 'blob',
+            mode: '100644',
+          }))
+
+        const createdTree = await githubClient.createTree({ tree: blobs, base_tree: 'foo' })
+
+        expect(mockFns.createTree).toBeCalledWith(expect.objectContaining({
+          ...githubClient.config,
+          tree: blobs,
+          base_tree: 'foo',
+        }))
+        expect(createdTree).toEqual(expect.any(String))
+      })
     })