Merge branch 'master' into release-1.2

Signed-off-by: Wei Tie <wtie@cisco.com>

Makefile

@@ -8,14 +8,15 @@
 DEFAULT_DOCKER_VERSION := 1.12.6
 V2PLUGIN_DOCKER_VERSION := 1.13.1
 SHELL := /bin/bash
-EXCLUDE_DIRS := bin docs Godeps scripts vagrant vendor install
+# TODO: contivmodel should be removed once its code passes golint and misspell
+EXCLUDE_DIRS := bin docs Godeps scripts vagrant vendor install contivmodel
 PKG_DIRS := $(filter-out $(EXCLUDE_DIRS),$(subst /,,$(sort $(dir $(wildcard */)))))
 TO_BUILD := ./netplugin/ ./netmaster/ ./netctl/netctl/ ./mgmtfn/k8splugin/contivk8s/ ./mgmtfn/mesosplugin/netcontiv/
 HOST_GOBIN := `if [ -n "$$(go env GOBIN)" ]; then go env GOBIN; else dirname $$(which go); fi`
 HOST_GOROOT := `go env GOROOT`
 NAME := netplugin
 VERSION := $(shell scripts/getGitVersion.sh)
-TAR := $(shell command -v gtar || echo command -v tar || echo "Could not find tar")
+TAR := $(shell command -v gtar || command -v tar || echo "Could not find tar")
 TAR_EXT := tar.bz2
 export NETPLUGIN_CONTAINER_TAG := $(shell ./scripts/getGitVersion.sh)
 TAR_FILENAME := $(NAME)-$(VERSION).$(TAR_EXT)
@@ -25,7 +26,7 @@ export V2PLUGIN_TAR_FILENAME := v2plugin-$(VERSION).tar.gz
 GO_MIN_VERSION := 1.7
 GO_MAX_VERSION := 1.8
 GO_VERSION := $(shell go version | cut -d' ' -f3 | sed 's/go//')
-CI_HOST_TARGETS ?= "host-unit-test host-integ-test host-build-docker-image"
+CI_HOST_TARGETS ?= "host-unit-test host-integ-test host-build-docker-image tar host-pluginfs-create clean-tar"
 SYSTEM_TESTS_TO_RUN ?= "00SSH|Basic|Network|Policy|TestTrigger|ACIM|Netprofile"
 K8S_SYSTEM_TESTS_TO_RUN ?= "00SSH|Basic|Network|Policy"
 ACI_GW_IMAGE ?= "contiv/aci-gw:04-12-2017.2.2_1n"
@@ -130,19 +131,8 @@ update:
 start:
 	CONTIV_DOCKER_VERSION="$${CONTIV_DOCKER_VERSION:-$(DEFAULT_DOCKER_VERSION)}" CONTIV_NODE_OS=${CONTIV_NODE_OS} vagrant up
 
-# ===================================================================
-#kubernetes demo targets
-k8s-demo:
-	cd vagrant/k8s/ && ./copy_demo.sh
-
-k8s-demo-start:
-	cd vagrant/k8s/ && ./restart_cluster.sh && vagrant ssh k8master
-
 # ===================================================================
 # kubernetes cluster bringup/cleanup targets
-k8s-legacy-cluster:
-	cd vagrant/k8s/ && ./setup_cluster.sh
-
 k8s-cluster:
 	cd vagrant/k8s/ && CONTIV_K8S_USE_KUBEADM=1 ./setup_cluster.sh
 
@@ -157,15 +147,6 @@ k8s-l3-destroy:
 
 # ===================================================================
 # kubernetes test targets
-k8s-legacy-test:
-	export CONTIV_K8S_LEGACY=1 && \
-	make k8s-sanity-cluster && \
-	cd vagrant/k8s/ && vagrant ssh k8master -c 'bash -lc "cd /opt/gopath/src/github.com/contiv/netplugin && make run-build"' && \
-	./start_sanity_service.sh
-	cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python && PYTHONIOENCODING=utf-8 ./createcfg.py -scheduler 'k8s'
-	CONTIV_K8S_LEGACY=1 CONTIV_NODES=3 go test -v -timeout 540m ./test/systemtests -check.v -check.abort -check.f "00SSH|TestBasic|TestNetwork|ACID|TestPolicy|TestTrigger"
-	cd vagrant/k8s && vagrant destroy -f
-
 k8s-test: k8s-cluster
 	cd vagrant/k8s/ && vagrant ssh k8master -c 'bash -lc "cd /opt/gopath/src/github.com/contiv/netplugin && make run-build"'
 	cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python && PYTHONIOENCODING=utf-8 ./createcfg.py -scheduler 'k8s' -binpath contiv/bin -install_mode 'kubeadm'
@@ -272,8 +253,8 @@ start-aci-gw:
 	docker pull $(ACI_GW_IMAGE)
 	docker run --net=host -itd -e "APIC_URL=SANITY" -e "APIC_USERNAME=IGNORE" -e "APIC_PASSWORD=IGNORE" --name=contiv-aci-gw $(ACI_GW_IMAGE)
 
-host-build-docker-image:
-	./scripts/netContain/build_image.sh
+host-build-docker-image: compile-with-docker binaries-from-container
+	@./scripts/netContain/build_image.sh
 
 host-cleanup:
 	@echo dev: cleaning up services...
@@ -381,7 +362,7 @@ binaries-from-container:
 	c_id=$$(docker create --name netplugin-build \
 		 netplugin-build:$(NETPLUGIN_CONTAINER_TAG)) && \
 	for f in netplugin netmaster netctl contivk8s netcontiv; do \
-		docker cp -a $${c_id}:/go/bin/$$f bin/$$f; done && \
+		docker cp $${c_id}:/go/bin/$$f bin/$$f; done && \
 	docker rm $${c_id}
 
 ##########################

RELEASE.md

@@ -70,6 +70,6 @@ release isn't made from the HEAD of master.
 ## Build and upload container image (manual only)
 
 	1. cd scripts/netContain
-	2. ./ReleaseContainer.sh -v <version> -u contiv -i contiv/netplugin
+	2. ./release_image.sh -v <version> \[-i <image_name>] \[-t <image_tag>]
 
 Here <version> is a version that has been released using steps above and exists on github.

Vagrantfile

@@ -18,15 +18,17 @@ BEGIN {
 go_version = ENV['GO_VERSION'] || '1.7.6'
 docker_version = ENV['CONTIV_DOCKER_VERSION'] || '1.12.6'
 docker_swarm = ENV['CONTIV_DOCKER_SWARM'] || 'classic_mode'
+docker_ee_url = ENV['DOCKERURL']
 gopath_folder = '/opt/gopath'
 http_proxy = ENV['HTTP_PROXY'] || ENV['http_proxy'] || ''
 https_proxy = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || ''
 build_version = ENV['BUILD_VERSION'] || ''
 cluster_ip_nodes = ''
 v2plugin_name = ENV['CONTIV_V2PLUGIN_NAME'] || 'contiv/v2netplugin:0.1'
-cluster_store = ENV['CONTIV_CLUSTER_STORE'] || 'etcd://localhost:2379'
+cluster_store_driver = ENV['CONTIV_CLUSTER_STORE_DRIVER'] || 'etcd'
+cluster_store_url = ENV['CONTIV_CLUSTER_STORE_URL'] || 'http://localhost:2379'
 nightly_release = ENV['NIGHTLY_RELEASE'] || ''
-node_os = ENV['CONTIV_NODE_OS'] || 'centos'
+node_os = ENV['CONTIV_NODE_OS'] != '' ? ENV['CONTIV_NODE_OS'] : 'centos'
 base_ip = ENV['CONTIV_IP_PREFIX'] || '192.168.2.'
 num_nodes = ENV['CONTIV_NODES'].to_i == 0 ? 3 : ENV['CONTIV_NODES'].to_i
 num_vm_cpus = (ENV['CONTIV_CPUS'] || 4).to_i
@@ -46,7 +48,8 @@ export https_proxy='#{https_proxy}'
 export NIGHTLY_RELEASE=#{nightly_release}
 export no_proxy=%{cluster_ip_nodes},127.0.0.1,localhost,netmaster
 export CLUSTER_NODE_IPS=%{cluster_ip_nodes}
-export CONTIV_CLUSTER_STORE=#{cluster_store}
+export CONTIV_CLUSTER_STORE_DRIVER=#{cluster_store_driver}
+export CONTIV_CLUSTER_STORE_URL=#{cluster_store_url}
 export CONTIV_V2PLUGIN_NAME=#{v2plugin_name}
 export CONTIV_DOCKER_SWARM=#{docker_swarm}
 export BUILD_VERSION=#{build_version}
@@ -93,6 +96,16 @@ rm -rf /var/lib/docker
 if [[ "#{node_os}" == "ubuntu" ]] && [[ "$reinstall" -eq 1 ]]; then
     sudo apt-get purge docker-engine -y || :
     curl https://get.docker.com | sed s/docker-engine/docker-engine=#{docker_version}-0~xenial/g | bash
+elif [[ "#{node_os}" == "centos" ]] && [[ -n "#{docker_ee_url}" ]]; then
+    echo "Preparing for Docker EE installation"
+    sudo yum remove -y docker docker-common docker-selinux docker-engine-selinux docker-engine docker-ce || :
+    sudo rm /etc/yum.repos.d/*docker*
+    export DOCKERURL='#{docker_ee_url}'
+    sudo -E sh -c 'echo "$DOCKERURL/centos" > /etc/yum/vars/dockerurl'
+    sudo -E yum-config-manager --add-repo "$DOCKERURL/centos/docker-ee.repo"
+    echo "Installing Docker EE #{docker_version}"
+    sudo yum -y install docker-ee-#{docker_version}
+    sudo yum install -y yum-utils device-mapper-persistent-data lvm2
 elif [[ "$reinstall" -eq 1 ]] && [[ "#{legacy_docker}" -eq 1 ]]; then
     # cleanup openstack-kilo repo if required
     yum remove docker-engine -y || :
@@ -105,6 +118,7 @@ elif [[ "$reinstall" -eq 1 ]] && [[ "#{legacy_docker}" -eq 1 ]]; then
         curl https://get.docker.com | sed s/docker-engine/docker-engine-#{docker_version}/ | bash
     fi
 elif [[ "$reinstall" -eq 1 ]]; then
+    echo "Installing Docker CE #{docker_version}"
     yum remove docker-engine -y || :
     yum remove docker-ce || :
     yum-config-manager --disable openstack-kilo
@@ -119,7 +133,7 @@ fi
 if [[ #{docker_swarm} == "swarm_mode" ]]; then
     perl -i -lpe 's!^ExecStart(.+)$!ExecStart$1 !' /lib/systemd/system/docker.service
 else
-    if [[ "$CONTIV_CLUSTER_STORE" == *"consul:"* ]]
+    if [[ "$CONTIV_CLUSTER_STORE_DRIVER" == "consul" ]]
     then
         perl -i -lpe 's!^ExecStart(.+)$!ExecStart$1 -H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --cluster-store=consul://localhost:8500!' /lib/systemd/system/docker.service
     else
@@ -128,7 +142,7 @@ else
 fi
 
 # setup docker remote api
-mkdir /etc/systemd/system/docker.service.d
+mkdir -p /etc/systemd/system/docker.service.d
 echo "[Service]" | sudo tee -a /etc/systemd/system/docker.service.d/http-proxy.conf
 echo "Environment=\\\"no_proxy=$CLUSTER_NODE_IPS,127.0.0.1,localhost,netmaster\\\" \\\"http_proxy=$http_proxy\\\" \\\"https_proxy=$https_proxy\\\"" | sudo tee -a /etc/systemd/system/docker.service.d/http-proxy.conf
 sudo systemctl daemon-reload
@@ -225,7 +239,7 @@ module VagrantPlugins
 
       def self.up_hook(arg)
         unless File.exist?(STATEFILE) # prevent it from writing more than once.
-          f = File.open(STATEFILE, "w") 
+          f = File.open(STATEFILE, "w")
           ENV.each do |x,y|
             f.puts "%s=%s" % [x,y]
           end
@@ -256,7 +270,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.box_check_update = false
     if Vagrant.has_plugin?("vagrant-vbguest")
         config.vbguest.auto_update = false
-    end 
+    end
     if node_os == "ubuntu" then
         config.vm.box = "contiv/ubuntu1604-netplugin"
         config.vm.box_version = "0.7.0"

contivmodel/scripts/build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# build and install the code
+bash generate.sh
+go install ./ ./client/
+
+# update the docs based on the latest code
+
+# NOTE: disabled on 2017/06/27 due to a breakage in the RAML Dockerfile.  it's complaining
+#       about a xhr2 package being missing or something.  since we're not even using the
+#       generated documentation, disabling it is the best option for now.
+
+# pushd spec
+# make docs
+# popd

contivmodel/spec/Dockerfile

@@ -0,0 +1,9 @@
+FROM node:alpine
+
+RUN npm install -g raml2html
+
+RUN mkdir /contiv
+
+WORKDIR /contiv
+
+ENTRYPOINT ["raml2html"]

contivmodel/spec/Dockerfile.cleanup

@@ -0,0 +1,5 @@
+FROM ruby:2.4.0-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get -y install build-essential && gem install nokogiri

contivmodel/spec/Makefile

@@ -0,0 +1,6 @@
+all: docs
+
+docs:
+	@bash ./build.sh
+
+.PHONY: docs

contivmodel/spec/auth_proxy.raml

@@ -0,0 +1,99 @@
+#%RAML 1.0
+title: Contiv
+description: Contiv API Specification
+version: v1
+baseUri:
+  value: https://{serverfqdn}:10000/api/{version}
+  (rediractable): true
+baseUriParameters:
+  serverfqdn:
+    type: string
+protocols: [  HTTPS ]
+mediaType: [ application/json ]
+
+resourceTypes:
+  collection: !include auth_proxy/schemas/collection.raml
+  non-upd-collection-item: !include auth_proxy/schemas/non-upd-collection-item.raml
+  collection-item: !include auth_proxy/schemas/collection-item.raml
+  ro-collection-item: !include auth_proxy/schemas/ro-collection-item.raml
+
+annotationTypes:
+  info:
+    properties:
+      license:
+        type: string
+        enum: [ "Apache 2.0" ]
+    allowedTargets: API
+  rediractable: boolean
+
+securitySchemes:
+  custom_scheme: !include auth_proxy/schemas/custom-scheme.raml
+
+# Resource templates
+uses:
+  auth_proxy: auth_proxy/libraries/auth_proxy.raml
+
+securedBy: custom_scheme
+
+# auth_proxy endpoints
+/auth_proxy:
+  displayName: Auth API
+  description: Authentication/Authorization related API
+
+  /health:
+    get:
+      description: Returns the health status of Contiv API server and its netmaster
+      securedBy: [ null ]
+      responses:
+        200:
+          body:
+            application/json:
+              type: auth_proxy.health
+
+  /login:
+    post:
+      description: Login to Contiv API server
+      securedBy: [ null ]
+      body:
+        application/json:
+          type: auth_proxy.login
+      responses:
+        200:
+          body:
+            application/json:
+              type: auth_proxy.login_response
+        400:
+        401:
+
+  /version:
+    get:
+      description: Returns the Contiv API server version
+      securedBy: [ null ]
+      responses:
+        200:
+          body:
+            application/json: |
+              { "version": "1.0.0" }
+
+  /authorizations:
+    type: {collection: {provider: auth_proxy}}
+    displayName: Authorizations
+
+    /{authzUUID}:
+      type: {non-upd-collection-item: {provider: auth_proxy}}
+      displayName: Authorization
+
+  /local_users:
+    type: {collection: {provider: auth_proxy}}
+    displayName: Local Users
+
+    /{username}:
+      type: {collection-item: {provider: auth_proxy}}
+      displayName: Local User
+      patch:
+
+  /ldap_configuration:
+    type: {collection-item: {provider: auth_proxy}}
+    displayName: LDAP Configuration
+    put:
+    patch: