Merge development into master for 0.8.3

CHANGELOG.md

@@ -2,6 +2,26 @@
 <!-- Use the tags Added, Changed, Deprecated, Removed, Fixed, Security, and
      Contributor to describe changes -->
 
+## [0.8.2]
+###### 2018-06-14
+
+### Added
+* CSV to Array action in the Utilities app
+
+
+### Changed
+* The action results SSE stream truncates the result using the
+  `MAX_STREAM_RESULTS_SIZE_KB` config option
+
+
+### Fixed
+* Bytes conversion bug in the RedisCacheAdapter
+* Bug in playbook editor using users and roles as arguments
+* Bug where some callbacks weren't getting registered
+* Column width bug in playbook editor, execution, and metrics pages
+* OpenAPI validation bug with newest version of the swagger validator
+
+
 ## [0.8.2]
 ###### 2018-05-03
 

apps/Utilities/actions.py

@@ -34,6 +34,11 @@ def echo_array(data):
     return data
 
 
+@action
+def csv_as_array(data):
+    return data.split(",")
+
+
 @action
 def json_select(json_reference, element):
     return json.loads(json_reference)[element]

apps/Utilities/api.yaml

@@ -71,6 +71,19 @@ actions:
         description: echoed list
         schema:
           type: array
+  csv as array:
+    run: actions.csv_as_array
+    description: returns a csv string as an array
+    parameters:
+      - name: data
+        description: csv to return
+        required: true
+        type: string
+    returns:
+      Success:
+        description: echoed list
+        schema:
+          type: array
   'json select':
     run: actions.json_select
     description: Gets a selected sub element of a json

docs/index.rst

@@ -7,7 +7,7 @@ Welcome to Walkoff's documentation!
 ***********************************
 Welcome to Walkoff's Python documentation. If you are looking for documentation and
 tutorials on getting started with Walkoff, please first look at
-`our Github Pages site <https://iadgov.github.io/WALKOFF>`_. Here you'll find tutorials and documentation on both UI
+`our Github Pages site <https://nsacyber.github.io/WALKOFF>`_. Here you'll find tutorials and documentation on both UI
 usage and app and interface development. This documentation is intended to help app and interface developers as well as
 provide a reference for project contributors.
 

setup_walkoff.py

@@ -1,7 +1,5 @@
 import os
 
-from scripts.compose_api import compose_api
-
 
 def main():
     print('\nInstalling Python Dependencies...')

tests/test_workflow_results_stream.py

@@ -81,6 +81,23 @@ def test_format_action_data_with_results(self):
         self.assert_and_strip_timestamp(result)
         self.assertDictEqual(result, expected)
 
+    def test_format_action_data_with_long_results(self):
+        size_limit = 128
+        self.app.config['MAX_STREAM_RESULTS_SIZE_KB'] = size_limit
+        workflow_id = str(uuid4())
+        kwargs = {'data': {'workflow': {'execution_id': workflow_id},
+                           'data': {'result': 'x'*1024*2*size_limit}}}  # should exceed limit
+        sender = self.get_sample_action_sender()
+        status = ActionStatusEnum.executing
+        result = format_action_data_with_results(sender, kwargs, status)
+        expected = sender
+        expected['action_id'] = expected.pop('id')
+        expected['workflow_execution_id'] = workflow_id
+        expected['status'] = status.name
+        expected['result'] = {'truncated': 'x'*1024*size_limit}
+        self.assert_and_strip_timestamp(result)
+        self.assertDictEqual(result, expected)
+
     def check_action_callback(self, callback, status, event, mock_publish, with_result=False):
         sender = self.get_sample_action_sender()
         kwargs = self.get_action_kwargs(with_result=with_result)

walkoff.py

@@ -23,7 +23,6 @@ def run(app, host, port):
     pids = spawn_worker_processes()
     monkey.patch_all()
 
-
     app.running_context.executor.initialize_threading(app, pids)
     # The order of these imports matter for initialization (should probably be fixed)
 

walkoff/api/objects/appapi.yaml

@@ -372,4 +372,5 @@ ParameterSchema:
       type: boolean
     enum:
       type: array
+      items: {}
       minItems: 1

walkoff/client/src/execution/execution.component.ts

@@ -53,6 +53,7 @@ export class ExecutionComponent implements OnInit, AfterViewChecked, OnDestroy {
 
 	workflowStatusEventSource: any;
 	actionStatusEventSource: any;
+	recalculateTableCallback: any;
 
 	constructor(
 		private executionService: ExecutionService, private authService: AuthService, private cdr: ChangeDetectorRef,
@@ -90,6 +91,14 @@ export class ExecutionComponent implements OnInit, AfterViewChecked, OnDestroy {
 		Observable.interval(30000).subscribe(() => {
 			this.recalculateRelativeTimes();
 		});
+
+		this.recalculateTableCallback = (e: JQuery.Event<HTMLElement, null>) => {
+			if (this.actionStatusTable && this.actionStatusTable.recalculate) {
+				this.actionStatusTable.recalculate();
+			}
+		}
+
+		$(document).on('shown.bs.modal', '.actionStatusModal', this.recalculateTableCallback)
 	}
 
 	/**
@@ -115,6 +124,9 @@ export class ExecutionComponent implements OnInit, AfterViewChecked, OnDestroy {
 		if (this.actionStatusEventSource && this.actionStatusEventSource.close) {
 			this.actionStatusEventSource.close();
 		}
+		if (this.recalculateTableCallback) {
+			$(document).off('shown.bs.modal', '.actionStatusModal', this.recalculateTableCallback)
+		}
 	}
 
 	/**

walkoff/client/src/metrics/metrics.component.ts

@@ -1,5 +1,4 @@
-import { Component, ViewEncapsulation, OnInit, ViewChild } from '@angular/core';
-import * as _ from 'lodash';
+import { Component, ViewEncapsulation, OnInit, ViewChild, ChangeDetectorRef } from '@angular/core';
 import { ToastyService, ToastyConfig } from 'ng2-toasty';
 import { Select2OptionData } from 'ng2-select2';
 import 'rxjs/add/operator/debounceTime';
@@ -8,10 +7,8 @@ import { DatatableComponent } from '@swimlane/ngx-datatable';
 import { MetricsService } from './metrics.service';
 import { UtilitiesService } from '../utilities.service';
 
-import { Playbook } from '../models/playbook/playbook';
 import { AppMetric } from '../models/metric/appMetric';
 import { WorkflowMetric } from '../models/metric/workflowMetric';
-import { ActionMetric } from '../models/metric/actionMetric';
 
 @Component({
 	selector: 'metrics-component',
@@ -28,13 +25,16 @@ export class MetricsComponent implements OnInit {
     appFilter: string = '';
     workflowMetrics: WorkflowMetric[] = [];
 	availableApps: Select2OptionData[] = [];
-    appSelectConfig: Select2Options;
+	appSelectConfig: Select2Options;
+	recalculateTableCallback: any;
 
-    @ViewChild('appMetricsTable') appMetricsTable: DatatableComponent; 
+	@ViewChild('appMetricsTable') appMetricsTable: DatatableComponent; 
+	@ViewChild('workflowMetricsTable') workflowMetricsTable: DatatableComponent; 
 
 	constructor(
         private metricsService: MetricsService, private toastyService: ToastyService, 
-        private toastyConfig: ToastyConfig, private utils: UtilitiesService,
+		private toastyConfig: ToastyConfig, private utils: UtilitiesService,
+		private cdr: ChangeDetectorRef
 	) {}
 
 	ngOnInit(): void {
@@ -45,10 +45,39 @@ export class MetricsComponent implements OnInit {
 			placeholder: 'Select an App to view its Metrics',
 		};
 
+		this.recalculateTableCallback = (e: JQuery.Event<HTMLElement, null>) => {
+			this.recalculateTable(e);
+		}
+		$(document).on('shown.bs.tab', 'a[data-toggle="tab"]', this.recalculateTableCallback)
+
 		this.getAppMetrics();
 		this.getWorkflowMetrics();
 	}
 
+	/**
+	 * Closes our SSEs on component destroy.
+	 */
+	ngOnDestroy(): void {
+		if (this.recalculateTableCallback) { $(document).off('shown.bs.tab', 'a[data-toggle="tab"]', this.recalculateTableCallback); }
+	}
+
+	/**
+	 * This angular function is used primarily to recalculate column widths for execution results table.
+	 */
+	recalculateTable(event: JQuery.Event<HTMLElement, null>) : void {
+		let table: DatatableComponent;
+		switch(event.target.getAttribute('href')) {
+			case '#apps':
+				table = this.appMetricsTable;
+				break;
+			case '#workflows':
+				table = this.workflowMetricsTable;
+		}
+		if (table && table.recalculate) {
+			this.cdr.detectChanges();
+			table.recalculate();
+		}
+	}
 	/**
 	 * Grabs case events from the server for the selected case (from the JS event supplied).
 	 * Will update the case events data table with the new case events.
@@ -62,22 +91,6 @@ export class MetricsComponent implements OnInit {
             this.appFilter = event.value;
         }
     }
-
-    test(event: any): void {
-        //this.appMetricsTable.recalculate();
-        //setTimeout(() => this.appMetricsTable.recalculate(), 100)
-        //console.log(this.appMetricsTable);
-    }
-
-    /**
-	 * This angular function is used primarily to recalculate column widths for execution results table.
-	 */
-	ngAfterViewChecked(): void {
-		// Check if the table size has changed,
-		if (this.appMetricsTable && this.appMetricsTable.recalculate) {
-			this.appMetricsTable.recalculate();
-		}
-	}
 
     getFilteredAppMetrics(): AppMetric[] {
         if (!this.appFilter || this.appFilter == 'all') return this.appMetrics;

walkoff/client/src/playbook/playbook.argument.component.ts

@@ -42,7 +42,7 @@ export class PlaybookArgumentComponent implements OnInit {
 	parameterSchema: ParameterSchema;
 	selectData: Select2OptionData[];
 	selectConfig: Select2Options;
-	selectInitialValue: number[];
+	selectInitialValue: string[];
 
 	// tslint:disable-next-line:no-empty
 	constructor() { }
@@ -56,11 +56,11 @@ export class PlaybookArgumentComponent implements OnInit {
 	 */
 	ngOnInit(): void {
 		this.initParameterSchema();
-		this.initUserSelect();
-		this.initRoleSelect();
 		this.initDeviceSelect()
 		this.initBranchCounterSelect();
 		this.initTypeSelector();
+		this.initUserSelect();
+		this.initRoleSelect();
 	}
 
 	initDeviceSelect(): void {
@@ -130,14 +130,16 @@ export class PlaybookArgumentComponent implements OnInit {
 			placeholder: 'Select user',
 		};
 
+		this.selectInitialValue = JSON.parse(JSON.stringify(this.argument.value));
+
 		if (this.parameterSchema.type === 'array') {
 			this.selectConfig.placeholder += '(s)';
 			this.selectConfig.multiple = true;
 			this.selectConfig.allowClear = true;
 			this.selectConfig.closeOnSelect = false;
+			if (Array.isArray(this.argument.value)) 
+				this.selectInitialValue = this.argument.value.map((val: number) => val.toString());
 		}
-
-		this.selectInitialValue = JSON.parse(JSON.stringify(this.argument.value));
 	}
 
 	/**
@@ -155,14 +157,16 @@ export class PlaybookArgumentComponent implements OnInit {
 			placeholder: 'Select role',
 		};
 
+		this.selectInitialValue = JSON.parse(JSON.stringify(this.argument.value));
+
 		if (this.parameterSchema.type === 'array') {
 			this.selectConfig.placeholder += '(s)';
 			this.selectConfig.multiple = true;
 			this.selectConfig.allowClear = true;
 			this.selectConfig.closeOnSelect = false;
+			if (Array.isArray(this.argument.value)) 
+				this.selectInitialValue = this.argument.value.map((val: number) => val.toString());
 		}
-
-		this.selectInitialValue = JSON.parse(JSON.stringify(this.argument.value));
 	}
 
 	initBranchCounterSelect(): void {

walkoff/client/src/playbook/playbook.argument.html

@@ -35,11 +35,11 @@
 			<input *ngIf="isBooleanSelect" type="checkbox" class="form-control" name="{{argument.name}}" [(ngModel)]="argument.value" (change)="clearReference()">
 
 			<!-- User select -->
-			<select2 *ngIf="isUserSelect" [data]="selectData" [value]="selectInitialValue" [options]="selectConfig"
+			<select2 *ngIf="isStatic && isUserSelect" [data]="selectData" [value]="selectInitialValue" [options]="selectConfig"
 				(valueChanged)="selectChange($event)"></select2>
 
 			<!-- Role select -->
-			<select2 *ngIf="isRoleSelect" [data]="selectData" [value]="selectInitialValue" [options]="selectConfig"
+			<select2 *ngIf="isStatic && isRoleSelect" [data]="selectData" [value]="selectInitialValue" [options]="selectConfig"
 				(valueChanged)="selectChange($event)"></select2>
 
 			<!-- Device select -->

walkoff/client/src/playbook/playbook.component.ts

@@ -56,7 +56,9 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy {
 	@ViewChild('workflowResultsContainer') workflowResultsContainer: ElementRef;
 	@ViewChild('workflowResultsTable') workflowResultsTable: DatatableComponent;
 	@ViewChild('consoleContainer') consoleContainer: ElementRef;
-    @ViewChild('consoleTable') consoleTable: DatatableComponent;
+	@ViewChild('consoleTable') consoleTable: DatatableComponent;
+	@ViewChild('errorLogContainer') errorLogContainer: ElementRef;
+    @ViewChild('errorLogTable') errorLogTable: DatatableComponent;
 
 	devices: Device[] = [];
 	relevantDevices: Device[] = [];
@@ -88,6 +90,7 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy {
 	eventSource: any;
 	consoleEventSource: any;
 	playbookToImport: File;
+	recalculateConsoleTableCallback: any;
 
 	// Simple bootstrap modal params
 	modalParams: {
@@ -143,6 +146,9 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy {
 		Observable.interval(30000).subscribe(() => {
 			this.recalculateRelativeTimes();
 		});
+
+		this.recalculateConsoleTableCallback = (e: JQuery.Event<HTMLElement, null>) => this.recalculateConsoleTable(e);
+		$(document).on('shown.bs.tab', 'a[data-toggle="tab"]', this.recalculateConsoleTableCallback)
 	}
 
 	/**
@@ -164,6 +170,7 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy {
 	ngOnDestroy(): void {
 		if (this.eventSource && this.eventSource.close) { this.eventSource.close(); }
 		if (this.consoleEventSource && this.consoleEventSource.close) { this.consoleEventSource.close(); }
+		if (this.recalculateConsoleTableCallback) { $(document).off('shown.bs.tab', 'a[data-toggle="tab"]', this.recalculateConsoleTableCallback); }
 	}
 
     ///------------------------------------------------------------------------------------------------------
@@ -1693,4 +1700,25 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy {
 		if (!this.loadedWorkflow) return [];
 		return this.loadedWorkflow.all_errors.map(error => ({ error }));
 	}
+
+	/**
+	 * This function is used primarily to recalculate column widths for execution results table.
+	 */
+	recalculateConsoleTable(e: JQuery.Event<HTMLElement, null>) {
+		let table: DatatableComponent;
+		switch(e.target.getAttribute('href')) {
+			case '#console':
+				table = this.consoleTable;
+				break;
+			case '#executionLog':
+				table = this.workflowResultsTable;
+				break;
+			case '#errorLog':
+				table = this.errorLogTable;
+		}
+		if (table && table.recalculate) { 
+			this.cdr.detectChanges();
+			table.recalculate(); 
+		}
+	}
 }

walkoff/config.py

@@ -136,6 +136,7 @@ class Config(object):
     JWT_TOKEN_LOCATION = 'headers'
 
     JWT_BLACKLIST_PRUNE_FREQUENCY = 1000
+    MAX_STREAM_RESULTS_SIZE_KB = 156
 
     @classmethod
     def load_config(cls, config_path=None):