Merge pull request #625 from intel-go/develop

New release 0.8.1

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:cosmic
+FROM ubuntu:disco
 
 ARG MAKEFLAGS=-j2
 
@@ -11,7 +11,6 @@ RUN apt-get -q update && apt-get -q -y install \
     git \
     curl \
     wget \
-    golang-${GO_VERSION} \
     libpcap-dev \
     libelf-dev \
     hugepages  \
@@ -21,7 +20,7 @@ RUN apt-get -q update && apt-get -q -y install \
     libmnl-dev \
     libibverbs-dev
 
-RUN cd /opt && curl -L -s https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz | tar zx
+RUN cd /opt && curl -L -s https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz | tar zx
 
 RUN mkdir -p ${NFF_GO}
 COPY . ${NFF_GO}

Makefile

@@ -4,8 +4,7 @@
 
 PATH_TO_MK = mk
 SUBDIRS = nff-go-base dpdk test examples
-DOC_TARGETS = flow packet
-CI_TESTING_TARGETS = packet low common
+CI_TESTING_TARGETS = packet internal/low common
 TESTING_TARGETS = $(CI_TESTING_TARGETS) test/stability
 
 all: $(SUBDIRS)
@@ -33,9 +32,4 @@ citesting: $(CI_TESTING_TARGETS)
 $(TESTING_TARGETS):
 	$(MAKE) -C $@ testing
 
-.PHONY: doc
-doc: $(DOC_TARGETS)
-	mkdir doc
-	$(foreach package,$(DOC_TARGETS),godoc -analysis=type -analysis=pointer -html github.com/intel-go/nff-go/$(package) > doc/$(package).html;)
-
 include $(PATH_TO_MK)/intermediate.mk

README.md

@@ -1,5 +1,5 @@
-[![Go Report Card](https://goreportcard.com/badge/github.com/intel-go/yanff)](https://goreportcard.com/report/github.com/intel-go/yanff) 
-[![GoDoc](https://godoc.org/github.com/intel-go/yanff?status.svg)](https://godoc.org/github.com/intel-go/yanff)
+[![Go Report Card](https://goreportcard.com/badge/github.com/intel-go/nff-go)](https://goreportcard.com/report/github.com/intel-go/nff-go) 
+[![GoDoc](https://godoc.org/github.com/intel-go/nff-go?status.svg)](https://godoc.org/github.com/intel-go/nff-go)
 [![Dev chat at https://gitter.im/intel-yanff/Lobby](https://img.shields.io/badge/gitter-developer_chat-46bc99.svg)](https://gitter.im/intel-yanff/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Build Status](https://travis-ci.org/intel-go/nff-go.svg?branch=develop)](https://travis-ci.org/intel-go/nff-go)
 # Network Function Framework for Go (former YANFF)
@@ -84,9 +84,15 @@ Requirements in the DPDK Getting Started Guide for
 Linux](http://dpdk.org/doc/guides/linux_gsg/sys_reqs.html) for more
 information.
 
-Since NFF-Go is build with Mellanox cards support out of the box you
+By default NFF-Go is build with Mellanox cards support out of the box you
 need to install additional dependencies required for MLX network
-drivers. On Ubuntu they are called `libmnl-dev` and `libibverbs-dev`.
+drivers. On Ubuntu they are called `libmnl-dev` and
+`libibverbs-dev`. For more details see MLX drivers respective pages
+for [MLX4](https://doc.dpdk.org/guides/nics/mlx4.html) and
+[MLX5](https://doc.dpdk.org/guides/nics/mlx5.html). If these
+dependencies cannot be satisfied, and Mellanox drivers are not needed,
+you can set variable `NFF_GO_NO_MLX_DRIVERS` to some unempty value to
+disable MLX drivers compilation.
 
 Additional dependencies are required for pktgen, especially if you are
 running RedHat or CentOS Linux distributions. See [this
@@ -133,18 +139,9 @@ when it is run for the first time, but may be quite slow.
 
 ## Documentation 
 
-Use:
-
-        make doc
-
-to generate full documentation. Alternatively, you can do:
-
-        godoc -http=:6060
-
-and browse the following URLs:
-
-* http://localhost:6060/pkg/nff-go/flow/
-* http://localhost:6060/pkg/nff-go/packet/
+Online API documentation is available on [godoc.org
+site](https://godoc.org/github.com/intel-go/nff-go). API usage is
+explained on our [Wiki pages](https://github.com/intel-go/nff-go/wiki).
 
 ## Tests
 
@@ -187,7 +184,7 @@ deployed images listed in NFF_GO_HOSTS, use the **make cleanall** command.
 ## Contributing
 
 If you want to contribute to NFF-Go, check our [Contributing
-guide](https://github.com/intel-go/yanff/blob/master/CONTRIBUTING.md). We also
+guide](https://github.com/intel-go/nff-go/blob/master/CONTRIBUTING.md). We also
 recommend checking the bugs with 'help-wanted' or 'easyfix' in our list of open issues; these bugs
 can be solved without an extensive knowledge of NFF-Go. We would love to help
 you start contributing.

devices/consts.go

@@ -75,7 +75,7 @@ var (
 
 var (
 	pathSysClassNetDeviceDriver stringBuilder = PathSysClassNet + "/%s/device/driver"
-	pathSysClassNetDevice       stringBuilder = PathSysClassNet + "/%s/device"
+	pathSysClassNetDevice       stringBuilder = PathSysClassNet + "/%s"
 )
 
 var (
@@ -107,6 +107,7 @@ func init() {
 
 	rVmbusDriver = regexp.MustCompile("/sys/bus/vmbus/drivers/(\\S+)/")
 
-	IsPciID = regexp.MustCompile("^\\d{4}:\\d{2}:\\d{2}.\\d$")
+	// domains are numbered from 0 to ffff), bus (0 to ff), slot (0 to 1f) and function (0 to 7)
+	IsPciID = regexp.MustCompile("^[[:xdigit:]]{4}:[[:xdigit:]]{2}:[0-1][[:xdigit:]].[0-7]$")
 	IsUUID = regexp.MustCompile("^[[:xdigit:]]{8}-[[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}$")
 }

devices/misc.go

@@ -34,8 +34,18 @@ func FindDefaultDpdkDriver(nicName string) string {
 
 // GetDeviceID returns the device ID of given NIC name.
 func GetDeviceID(nicName string) (string, error) {
-	// DEV_ID=$(basename $(readlink /sys/class/net/<nicName>/device))
-	return readlinkBaseCmd(pathSysClassNetDevice.With(nicName))
+	// DEV_ID=$(basename $(readlink /sys/class/net/<nicName>))
+	raw, err := readlinkCmd(pathSysClassNetDevice.With(nicName))
+	if err != nil {
+		return "", err
+	}
+	// raw should be like /sys/devices/pci0002:00/0000:00:08.0/virtio2/net/ens8
+	raws := strings.Split(raw, "/")
+	if len(raws) < 5 {
+		return "", fmt.Errorf("path not correct")
+	}
+	return raws[4], nil
+
 }
 
 // IsModuleLoaded checks if the kernel has already loaded the driver or not.
@@ -69,12 +79,20 @@ func writeToTargetWithData(sysfs string, flag int, mode os.FileMode, data string
 	return nil
 }
 
-func readlinkBaseCmd(path string) (string, error) {
-	output, err := cmdOutputWithTimeout(defaultTimeoutLimitation, "readlink", path)
+func readlinkCmd(path string) (string, error) {
+	output, err := cmdOutputWithTimeout(defaultTimeoutLimitation, "readlink", "-f", path)
 	if err != nil {
 		return "", fmt.Errorf("Cmd Execute readlink failed: %s", err.Error())
 	}
 	outputStr := strings.Trim(string(output), "\n")
+	return outputStr, nil
+}
+
+func readlinkBaseCmd(path string) (string, error) {
+	outputStr, err := readlinkCmd(path)
+	if err != nil {
+		return "", fmt.Errorf("Cmd Execute readlink failed: %s", err.Error())
+	}
 	result := filepath.Base(outputStr)
 	return result, nil
 }

dpdk/Makefile

@@ -23,7 +23,12 @@ all: pktgen
 
 $(DPDK_DIR)/$(DPDK_INSTALL_DIR):
 	$(MAKE) -C $(DPDK_DIR) config T=$(RTE_TARGET)
-	sed -ri 's,(MLX._PMD=)n,\1y,' $(DPDK_DIR)/build/.config
+	@if [ -z '${NFF_GO_NO_MLX_DRIVERS}' ]; then				\
+		echo BUILDING DPDK WITH MLX DRIVERS;				\
+		sed -ri 's,(MLX._PMD=)n,\1y,' $(DPDK_DIR)/build/.config;	\
+	else									\
+		echo BUILDING DPDK WITHOUT MLX DRIVERS;				\
+	fi
 	$(MAKE) -C $(DPDK_DIR)
 	$(MAKE) -C $(DPDK_DIR) install DESTDIR=$(DPDK_INSTALL_DIR)
 

examples/.gitignore

@@ -13,3 +13,5 @@ netlink
 devbind
 OSforwarding
 generate
+jumbo
+decrementTTL

examples/Dockerfile

@@ -21,3 +21,5 @@ COPY timer .
 COPY netlink .
 COPY generate .
 COPY OSforwarding .
+COPY jumbo .
+COPY decrementTTL .

examples/Makefile

@@ -7,7 +7,8 @@ IMAGENAME = nff-go-examples
 EXECUTABLES = dump clonablePcapDumper kni copy errorHandling timer \
 		createPacket sendFixedPktsNumber gtpu pingReplay \
 		netlink gopacketParserExample devbind generate \
-		OSforwarding
+		OSforwarding jumbo decrementTTL
+
 SUBDIRS = tutorial antiddos demo fileReadWrite firewall forwarding ipsec lb
 
 .PHONY: dpi nffPktgen

examples/decrementTTL.go

@@ -0,0 +1,40 @@
+// Copyright 2017 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+
+	"github.com/intel-go/nff-go/flow"
+	"github.com/intel-go/nff-go/packet"
+)
+
+// Main function for constructing packet processing graph.
+func main() {
+	inPort := flag.Uint("inPort", 0, "port for receiver")
+	outPort := flag.Uint("outPort", 1, "port for sender")
+	flag.Parse()
+
+	flow.SystemInit(nil)
+	inputFlow, _ := flow.SetReceiver(uint16(*inPort))
+	flow.SetHandlerDrop(inputFlow, decrementTTL, nil)
+	flow.SetSender(inputFlow, uint16(*outPort))
+	flow.SystemStart()
+}
+
+func decrementTTL(current *packet.Packet, c flow.UserContext) bool {
+	current.ParseL3() // must parse before header can be read
+	header := current.GetIPv4()
+	if header == nil { // not IPv4
+		return false
+	}
+
+	header.TimeToLive--
+	if header.TimeToLive == 0 { // TTL exceeded, drop
+		return false
+	} else {
+		return true
+	}
+}

examples/dpi/Makefile

@@ -5,6 +5,6 @@
 PATH_TO_MK = ../../mk
 SUBDIRS = main pattern
 
-export GO_COMPILE_FLAGS += -tags hyperscan_v4
+export GO_BUILD_TAGS += hyperscan_v4
 
 include $(PATH_TO_MK)/intermediate.mk

examples/dpi/main/Makefile

@@ -9,7 +9,7 @@ EXECUTABLES = dpi
 dpi: ../pattern gohs
 
 gohs:
-	go get $(GO_COMPILE_FLAGS) -v github.com/flier/gohs/hyperscan
+	go get -tags hyperscan_v4 -v github.com/flier/gohs/hyperscan
 
 COMMON_FILES = handlers.go
 

examples/dpi/pattern/Makefile

@@ -6,4 +6,4 @@ PATH_TO_MK = ../../../mk
 IMAGENAME = dpi-pattern
 
 pattern:
-	go install $(GO_COMPILE_FLAGS)
+	go install -tags hyperscan_v4

examples/jumbo.go

@@ -0,0 +1,100 @@
+// Copyright 2019 Intel Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/intel-go/nff-go/flow"
+	"github.com/intel-go/nff-go/packet"
+	"time"
+)
+
+// You need to enable memory jumbo for maxPacketSegment more than 2014.
+// Explanation: internal packet buffers are 2048 bytes, first packet will have
+// ethernet and ipv4 headers = 14 + 2 = 34 bytes.
+// Buffer will overflows if maxPacketSegment > 2048 - 34 = 2014 bytes
+
+// Note: usage of 1484 < maxPacketSegment < 2014 without memory jumbo can work
+// on some NICs, but is treated as underfined behaviour
+// Explanation: in this situation packet will be fit in internal buffers
+// however packet will be more than 1518 bytes - Ethernet max non-jumbo frame
+const maxPacketSegment = 1484
+
+// You need to enale chained jumbo for requiredPacketLength > maxPacketSegment
+// Explanation: if required length is more than segment length packet will consist of
+// several chained segments. You should enable chained jumbo to correctly handle them
+
+// Note: usage of maxPacketSegment > 2014 and also requiredPacketLength > maxPacketSegment
+// is forbidden. If you decicde to chain packets you shouldn't chain non-standart packets
+// Explanation: in this situation you will need to enable both
+// memory and chained jumbo and this is forbidden
+const requiredPacketLength = 7000
+
+func main() {
+	chained := flag.Bool("chained", false, "enable chained jumbo")
+	memory := flag.Bool("memory", false, "enalbe memory jumbo")
+	flag.Parse()
+
+	// Initialize NFF-GO library at 10 available cores
+	config := flow.Config{
+		ChainedJumbo: *chained,
+		MemoryJumbo:  *memory,
+	}
+
+	flow.CheckFatal(flow.SystemInit(&config))
+
+	generated := flow.SetGenerator(generate, nil)
+	flow.SetHandler(generated, dump, nil)
+	flow.SetSender(generated, 0)
+
+	received, _ := flow.SetReceiver(0)
+	flow.SetHandler(received, dump, nil)
+	flow.SetStopper(received)
+	flow.SystemStart()
+}
+
+func generate(pkt *packet.Packet, context flow.UserContext) {
+	remain := requiredPacketLength
+	c := 0
+	diff, remain := m(maxPacketSegment, remain)
+	if packet.InitEmptyIPv4Packet(pkt, diff) == false {
+		fmt.Printf("maxPacketSegment %d is more than 2014 bytes without enabling memory jumbo\n", maxPacketSegment)
+		time.Sleep(5 * time.Second)
+		return
+	}
+	pkt.Ether.DAddr = [6]uint8{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}
+	for i := 0; i < maxPacketSegment; i++ {
+		(*(*[maxPacketSegment]byte)(pkt.Data))[i] = byte(c % 10)
+		c++
+	}
+	for remain != 0 {
+		diff, remain = m(maxPacketSegment, remain)
+		pkt = packet.InitNextPacket(diff, pkt)
+		for i := 0; i < maxPacketSegment; i++ {
+			(*(*[maxPacketSegment]byte)(pkt.Data))[i] = byte(c % 10)
+			c++
+		}
+	}
+	time.Sleep(1 * time.Second)
+}
+
+func m(max int, remain int) (uint, int) {
+	if remain > max {
+		return uint(max), remain - max
+	} else {
+		return uint(remain), 0
+	}
+}
+
+func dump(currentPacket *packet.Packet, context flow.UserContext) {
+	fmt.Println("NEW PACKET")
+	fmt.Printf("BLOCK\n%x\nEND_BLOCK\n", currentPacket.GetRawPacketBytes())
+	for currentPacket.Next != nil {
+		currentPacket = currentPacket.Next
+		fmt.Printf("BLOCK\n%x\nEND_BLOCK\n", currentPacket.GetRawPacketBytes())
+	}
+	fmt.Println("END PACKET")
+}

examples/lb/balancer.go

@@ -51,7 +51,7 @@ func balancer(pkt *packet.Packet, ctx flow.UserContext) bool {
 	workerMAC, found := LBConfig.TunnelPort.neighCache.LookupMACForIPv4(workerIP)
 	if !found {
 		fmt.Println("Not found MAC address for IP", workerIP.String())
-		LBConfig.TunnelPort.neighCache.SendARPRequestForIPv4(workerIP, 0)
+		LBConfig.TunnelPort.neighCache.SendARPRequestForIPv4(workerIP, LBConfig.TunnelPort.Subnet.IPv4.Addr, 0)
 		return false
 	}
 

examples/lb/config.go

@@ -64,5 +64,11 @@ func InitFlows() {
 
 func (port *IpPort) initPort() {
 	port.macAddress = flow.GetPortMACAddress(port.Index)
-	port.neighCache = packet.NewNeighbourTable(port.Index, port.macAddress, port.Subnet.IPv4.Addr, port.Subnet.IPv6.Addr)
+	port.neighCache = packet.NewNeighbourTable(port.Index, port.macAddress,
+		func(ipv4 types.IPv4Address) bool {
+			return ipv4 == port.Subnet.IPv4.Addr
+		},
+		func(ipv6 types.IPv6Address) bool {
+			return ipv6 == port.Subnet.IPv6.Addr
+		})
 }

examples/nffPktgen/Makefile

@@ -4,6 +4,6 @@
 
 PATH_TO_MK = ../../mk
 IMAGENAME = nff-pktgen
-SUBDIRS = testing
+SUBDIRS = testing gtp-u
 
 include $(PATH_TO_MK)/intermediate.mk