Merge branch 'master' into gh

sys/dev/netmap/netmap.c

@@ -1145,16 +1145,24 @@ nm_may_forward_up(struct netmap_kring *kring)
 }
 
 static inline int
-nm_may_forward_down(struct netmap_kring *kring)
+nm_may_forward_down(struct netmap_kring *kring, int sync_flags)
 {
 	return	_nm_may_forward(kring) &&
+		 (sync_flags & NAF_CAN_FORWARD_DOWN) &&
 		 kring->ring_id == kring->na->num_rx_rings;
 }
 
 /*
  * Send to the NIC rings packets marked NS_FORWARD between
- * kring->nr_hwcur and kring->rhead
- * Called under kring->rx_queue.lock on the sw rx ring,
+ * kring->nr_hwcur and kring->rhead.
+ * Called under kring->rx_queue.lock on the sw rx ring.
+ *
+ * It can only be called if the user opened all the TX hw rings,
+ * see NAF_CAN_FORWARD_DOWN flag.
+ * We can touch the TX netmap rings (slots, head and cur) since
+ * we are in poll/ioctl system call context, and the application
+ * is not supposed to touch the ring (using a different thread)
+ * during the execution of the system call.
  */
 static u_int
 netmap_sw_to_nic(struct netmap_adapter *na)
@@ -1168,8 +1176,6 @@ netmap_sw_to_nic(struct netmap_adapter *na)
 
 	/* scan rings to find space, then fill as much as possible */
 	for (i = 0; i < na->num_tx_rings; i++) {
-		/* XXX some krings may not be in netmap mode,
-		 * buffers may not be there */
 		struct netmap_kring *kdst = &na->tx_rings[i];
 		struct netmap_ring *rdst = kdst->ring;
 		u_int const dst_lim = kdst->nkr_num_slots - 1;
@@ -1197,11 +1203,9 @@ netmap_sw_to_nic(struct netmap_adapter *na)
 			dst->len = tmp.len;
 			dst->flags = NS_BUF_CHANGED;
 
-			/* XXX is it safe to write head/cur concurrently to
-			 * the userspace application? */
 			rdst->head = rdst->cur = nm_next(dst_head, dst_lim);
 		}
-		/* if (sent) XXX txsync ? */
+		/* if (sent) XXX txsync ? it would be just an optimization */
 	}
 	return sent;
 }
@@ -1291,7 +1295,7 @@ netmap_rxsync_from_host(struct netmap_kring *kring, int flags)
 	 */
 	nm_i = kring->nr_hwcur;
 	if (nm_i != head) { /* something was released */
-		if (nm_may_forward_down(kring)) {
+		if (nm_may_forward_down(kring, flags)) {
 			ret = netmap_sw_to_nic(na);
 			if (ret > 0) {
 				kring->nr_kflags |= NR_FORWARD;
@@ -1806,6 +1810,13 @@ netmap_interp_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags
 	}
 	priv->np_flags = (flags & ~NR_REG_MASK) | reg;
 
+	/* Allow transparent forwarding mode in the host --> nic
+	 * direction only if all the TX hw rings have been opened. */
+	if (priv->np_qfirst[NR_TX] == 0 &&
+			priv->np_qlast[NR_TX] >= na->num_tx_rings) {
+		priv->np_sync_flags |= NAF_CAN_FORWARD_DOWN;
+	}
+
 	if (netmap_verbose) {
 		D("%s: tx [%d,%d) rx [%d,%d) id %d",
 			na->name,
@@ -2069,7 +2080,7 @@ netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
 		goto err_rel_excl;
 
 	/* in all cases, create a new netmap if */
-	nifp = netmap_mem_if_new(na);
+	nifp = netmap_mem_if_new(na, priv);
 	if (nifp == NULL) {
 		error = ENOMEM;
 		goto err_del_rings;
@@ -2177,6 +2188,7 @@ netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data, struct thread
 	u_int i, qfirst, qlast;
 	struct netmap_if *nifp;
 	struct netmap_kring *krings;
+	int sync_flags;
 	enum txrx t;
 
 	if (cmd == NIOCGINFO || cmd == NIOCREGIF) {
@@ -2387,6 +2399,7 @@ netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data, struct thread
 		krings = NMR(na, t);
 		qfirst = priv->np_qfirst[t];
 		qlast = priv->np_qlast[t];
+		sync_flags = priv->np_sync_flags;
 
 		for (i = qfirst; i < qlast; i++) {
 			struct netmap_kring *kring = krings + i;
@@ -2404,7 +2417,7 @@ netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data, struct thread
 					    kring->nr_hwcur);
 				if (nm_txsync_prologue(kring, ring) >= kring->nkr_num_slots) {
 					netmap_ring_reinit(kring);
-				} else if (kring->nm_sync(kring, NAF_FORCE_RECLAIM) == 0) {
+				} else if (kring->nm_sync(kring, sync_flags | NAF_FORCE_RECLAIM) == 0) {
 					nm_sync_finalize(kring);
 				}
 				if (netmap_verbose & NM_VERB_TXSYNC)
@@ -2419,7 +2432,7 @@ netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data, struct thread
 					/* transparent forwarding, see netmap_poll() */
 					netmap_grab_packets(kring, &q, netmap_fwd);
 				}
-				if (kring->nm_sync(kring, NAF_FORCE_READ) == 0) {
+				if (kring->nm_sync(kring, sync_flags | NAF_FORCE_READ) == 0) {
 					nm_sync_finalize(kring);
 				}
 				ring_timestamp_set(ring);
@@ -2518,6 +2531,7 @@ netmap_poll(struct netmap_priv_d *priv, int events, NM_SELRECORD_T *sr)
 	 * file descriptor.
 	 */
 	int send_down = 0;
+	int sync_flags = priv->np_sync_flags;
 
 	mbq_init(&q);
 
@@ -2627,7 +2641,7 @@ netmap_poll(struct netmap_priv_d *priv, int events, NM_SELRECORD_T *sr)
 				netmap_ring_reinit(kring);
 				revents |= POLLERR;
 			} else {
-				if (kring->nm_sync(kring, 0))
+				if (kring->nm_sync(kring, sync_flags))
 					revents |= POLLERR;
 				else
 					nm_sync_finalize(kring);
@@ -2691,7 +2705,7 @@ netmap_poll(struct netmap_priv_d *priv, int events, NM_SELRECORD_T *sr)
 			 * the nm_sync() below only on for the host RX ring (see
 			 * netmap_rxsync_from_host()). */
 			kring->nr_kflags &= ~NR_FORWARD;
-			if (kring->nm_sync(kring, 0))
+			if (kring->nm_sync(kring, sync_flags))
 				revents |= POLLERR;
 			else
 				nm_sync_finalize(kring);

sys/dev/netmap/netmap_generic.c

@@ -165,12 +165,12 @@ nm_os_get_mbuf(struct ifnet *ifp, int len)
  * has a KASSERT(), checking that the mbuf dtor function is not NULL.
  */
 
+static void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { }
+
 #define SET_MBUF_DESTRUCTOR(m, fn)	do {		\
-	(m)->m_ext.ext_free = (void *)fn;	\
+	(m)->m_ext.ext_free = fn ? (void *)fn : (void *)void_mbuf_dtor;	\
 } while (0)
 
-static void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { }
-
 static inline struct mbuf *
 nm_os_get_mbuf(struct ifnet *ifp, int len)
 {

sys/dev/netmap/netmap_kern.h

@@ -749,8 +749,9 @@ struct netmap_adapter {
 	int (*nm_txsync)(struct netmap_kring *kring, int flags);
 	int (*nm_rxsync)(struct netmap_kring *kring, int flags);
 	int (*nm_notify)(struct netmap_kring *kring, int flags);
-#define NAF_FORCE_READ    1
-#define NAF_FORCE_RECLAIM 2
+#define NAF_FORCE_READ      1
+#define NAF_FORCE_RECLAIM   2
+#define NAF_CAN_FORWARD_DOWN 4
 	/* return configuration information */
 	int (*nm_config)(struct netmap_adapter *,
 		u_int *txr, u_int *txd, u_int *rxr, u_int *rxd);
@@ -1405,7 +1406,7 @@ u_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg);
 int netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na,
 		  struct ifnet **ifp, struct netmap_mem_d *nmd, int create);
 void netmap_unget_na(struct netmap_adapter *na, struct ifnet *ifp);
-int netmap_get_hw_na(struct ifnet *ifp, 
+int netmap_get_hw_na(struct ifnet *ifp,
 		struct netmap_mem_d *nmd, struct netmap_adapter **na);
 
 
@@ -1808,6 +1809,7 @@ struct netmap_priv_d {
 	u_int		np_qfirst[NR_TXRX],
 			np_qlast[NR_TXRX]; /* range of tx/rx rings to scan */
 	uint16_t	np_txpoll;	/* XXX and also np_rxpoll ? */
+	int             np_sync_flags; /* to be passed to nm_sync */
 
 	int		np_refs;	/* use with NMG_LOCK held */
 
@@ -2069,7 +2071,7 @@ struct netmap_pt_host_adapter {
 	void *ptns;
 };
 /* ptnetmap HOST routines */
-int netmap_get_pt_host_na(struct nmreq *nmr, struct netmap_adapter **na, 
+int netmap_get_pt_host_na(struct nmreq *nmr, struct netmap_adapter **na,
 		struct netmap_mem_d * nmd, int create);
 int ptnetmap_ctl(struct nmreq *nmr, struct netmap_adapter *na);
 static inline int

sys/dev/netmap/netmap_mem2.c

@@ -142,7 +142,8 @@ struct netmap_mem_ops {
 	ssize_t  (*nmd_if_offset)(struct netmap_mem_d *, const void *vaddr);
 	void (*nmd_delete)(struct netmap_mem_d *);
 
-	struct netmap_if * (*nmd_if_new)(struct netmap_adapter *);
+	struct netmap_if * (*nmd_if_new)(struct netmap_adapter *,
+					 struct netmap_priv_d *);
 	void (*nmd_if_delete)(struct netmap_adapter *, struct netmap_if *);
 	int  (*nmd_rings_create)(struct netmap_adapter *);
 	void (*nmd_rings_delete)(struct netmap_adapter *);
@@ -221,7 +222,7 @@ NMD_DEFCB(int, config);
 NMD_DEFCB1(ssize_t, if_offset, const void *);
 NMD_DEFCB(void, delete);
 
-NMD_DEFNACB(struct netmap_if *, if_new);
+NMD_DEFNACB1(struct netmap_if *, if_new, struct netmap_priv_d *);
 NMD_DEFNACB1(void, if_delete, struct netmap_if *);
 NMD_DEFNACB(int, rings_create);
 NMD_DEFNACB(void, rings_delete);
@@ -1756,7 +1757,7 @@ netmap_mem2_rings_delete(struct netmap_adapter *na)
  * the interface is in netmap mode.
  */
 static struct netmap_if *
-netmap_mem2_if_new(struct netmap_adapter *na)
+netmap_mem2_if_new(struct netmap_adapter *na, struct netmap_priv_d *priv)
 {
 	struct netmap_if *nifp;
 	ssize_t base; /* handy for relative offsets between rings and nifp */
@@ -1795,24 +1796,28 @@ netmap_mem2_if_new(struct netmap_adapter *na)
 	 */
 	base = netmap_if_offset(na->nm_mem, nifp);
 	for (i = 0; i < n[NR_TX]; i++) {
-		if (na->tx_rings[i].ring == NULL) {
-			// XXX maybe use the offset of an error ring,
-			// like we do for buffers?
-			*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] = 0;
-			continue;
+		/* XXX instead of ofs == 0 maybe use the offset of an error
+		 * ring, like we do for buffers? */
+		ssize_t ofs = 0;
+
+		if (na->tx_rings[i].ring != NULL && i >= priv->np_qfirst[NR_TX]
+				&& i < priv->np_qlast[NR_TX]) {
+			ofs = netmap_ring_offset(na->nm_mem,
+						 na->tx_rings[i].ring) - base;
 		}
-		*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] =
-			netmap_ring_offset(na->nm_mem, na->tx_rings[i].ring) - base;
+		*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] = ofs;
 	}
 	for (i = 0; i < n[NR_RX]; i++) {
-		if (na->rx_rings[i].ring == NULL) {
-			// XXX maybe use the offset of an error ring,
-			// like we do for buffers?
-			*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+n[NR_TX]] = 0;
-			continue;
+		/* XXX instead of ofs == 0 maybe use the offset of an error
+		 * ring, like we do for buffers? */
+		ssize_t ofs = 0;
+
+		if (na->rx_rings[i].ring != NULL && i >= priv->np_qfirst[NR_RX]
+				&& i < priv->np_qlast[NR_RX]) {
+			ofs = netmap_ring_offset(na->nm_mem,
+						 na->rx_rings[i].ring) - base;
 		}
-		*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+n[NR_TX]] =
-			netmap_ring_offset(na->nm_mem, na->rx_rings[i].ring) - base;
+		*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+n[NR_TX]] = ofs;
 	}
 
 	NMA_UNLOCK(na->nm_mem);
@@ -2171,7 +2176,7 @@ netmap_mem_pt_guest_delete(struct netmap_mem_d *nmd)
 }
 
 static struct netmap_if *
-netmap_mem_pt_guest_if_new(struct netmap_adapter *na)
+netmap_mem_pt_guest_if_new(struct netmap_adapter *na, struct netmap_priv_d *priv)
 {
 	struct netmap_mem_ptg *ptnmd = (struct netmap_mem_ptg *)na->nm_mem;
 	struct mem_pt_if *ptif;

sys/dev/netmap/netmap_mem2.h

@@ -130,7 +130,7 @@ PMDL win32_build_user_vm_map(struct netmap_mem_d* nmd);
 int	   netmap_mem_finalize(struct netmap_mem_d *, struct netmap_adapter *);
 int 	   netmap_mem_init(void);
 void 	   netmap_mem_fini(void);
-struct netmap_if * netmap_mem_if_new(struct netmap_adapter *);
+struct netmap_if * netmap_mem_if_new(struct netmap_adapter *, struct netmap_priv_d *);
 void 	   netmap_mem_if_delete(struct netmap_adapter *, struct netmap_if *);
 int	   netmap_mem_rings_create(struct netmap_adapter *);
 void	   netmap_mem_rings_delete(struct netmap_adapter *);

sys/dev/netmap/netmap_monitor.c

@@ -198,7 +198,7 @@ nm_monitor_alloc(struct netmap_kring *kring, u_int n)
 	if (n <= kring->max_monitors)
 		/* we already have more entries that requested */
 		return 0;
-	
+
 	old_len = sizeof(struct netmap_kring *)*kring->max_monitors;
         len = sizeof(struct netmap_kring *) * n;
 	nm = nm_os_realloc(kring->monitors, len, old_len);

sys/dev/netmap/netmap_pipe.c

@@ -525,7 +525,7 @@ netmap_pipe_dtor(struct netmap_adapter *na)
 }
 
 int
-netmap_get_pipe_na(struct nmreq *nmr, struct netmap_adapter **na, 
+netmap_get_pipe_na(struct nmreq *nmr, struct netmap_adapter **na,
 		struct netmap_mem_d *nmd, int create)
 {
 	struct nmreq pnmr;

sys/dev/netmap/netmap_pt.c

@@ -1146,7 +1146,7 @@ nm_pt_host_dtor(struct netmap_adapter *na)
 
 /* check if nmr is a request for a ptnetmap adapter that we can satisfy */
 int
-netmap_get_pt_host_na(struct nmreq *nmr, struct netmap_adapter **na, 
+netmap_get_pt_host_na(struct nmreq *nmr, struct netmap_adapter **na,
 		struct netmap_mem_d *nmd, int create)
 {
     struct nmreq parent_nmr;

sys/dev/netmap/netmap_vale.c

@@ -161,7 +161,7 @@ SYSCTL_DECL(_dev_netmap);
 SYSCTL_INT(_dev_netmap, OID_AUTO, bridge_batch, CTLFLAG_RW, &bridge_batch, 0 , "");
 SYSEND;
 
-static int netmap_vp_create(struct nmreq *, struct ifnet *, 
+static int netmap_vp_create(struct nmreq *, struct ifnet *,
 		struct netmap_mem_d *nmd, struct netmap_vp_adapter **);
 static int netmap_vp_reg(struct netmap_adapter *na, int onoff);
 static int netmap_bwrap_reg(struct netmap_adapter *, int onoff);

sys/net/netmap_user.h

@@ -309,7 +309,7 @@ typedef void (*nm_cb_t)(u_char *, const struct nm_pkthdr *, const u_char *d);
  * ifname	(netmap:foo or vale:foo) is the port name
  *		a suffix can indicate the follwing:
  *		^		bind the host (sw) ring pair
- *		*		bind host and NIC ring pairs (transparent)
+ *		*		bind host and NIC ring pairs
  *		-NN		bind individual NIC ring pair
  *		{NN		bind master side of pipe NN
  *		}NN		bind slave side of pipe NN
@@ -870,7 +870,7 @@ nm_open(const char *ifname, const struct nmreq *req,
 
 	nr_reg = d->req.nr_flags & NR_REG_MASK;
 
-	if (nr_reg ==  NR_REG_SW) { /* host stack */
+	if (nr_reg == NR_REG_SW) { /* host stack */
 		d->first_tx_ring = d->last_tx_ring = d->req.nr_tx_rings;
 		d->first_rx_ring = d->last_rx_ring = d->req.nr_rx_rings;
 	} else if (nr_reg ==  NR_REG_ALL_NIC) { /* only nic */