[Open-FCoE] [PATCH] [FCoE] fixup fcoe_crc_eof handling and page ref counts

Chris Leech christopher.leech at intel.com
Tue Apr 8 22:52:16 UTC 2008


Move the fcoe_crc_eof structure to a real page fragment, and call
get_page() on everything we add from the scsi buffer.
This should let us get rid of the destructor, which will
fix the problems with packet sockets (the "bad address" errors
in wireshark).

Signed-off-by: Chris Leech <christopher.leech at intel.com>
---

 drivers/scsi/ofc/fcoe/fcoe_def.h      |    2 +
 drivers/scsi/ofc/fcoe/fcoe_dev.c      |   72 ++++++++++++++++++++++++---------
 drivers/scsi/ofc/fcoe/fcoeinit.c      |    2 +
 drivers/scsi/ofc/openfc/openfc_scsi.c |    6 ---
 4 files changed, 57 insertions(+), 25 deletions(-)


diff --git a/drivers/scsi/ofc/fcoe/fcoe_def.h b/drivers/scsi/ofc/fcoe/fcoe_def.h
index cc4c97b..346f8f9 100644
--- a/drivers/scsi/ofc/fcoe/fcoe_def.h
+++ b/drivers/scsi/ofc/fcoe/fcoe_def.h
@@ -46,6 +46,8 @@ struct fcoe_percpu_s {
 	struct task_struct *thread;
 	struct sk_buff_head fcoe_rx_list;
 	struct fc_frame	frame;
+	struct page *crc_eof_page;
+	int crc_eof_offset;
 };
 
 struct fcoe_info {
diff --git a/drivers/scsi/ofc/fcoe/fcoe_dev.c b/drivers/scsi/ofc/fcoe/fcoe_dev.c
index 8119024..4e404b9 100644
--- a/drivers/scsi/ofc/fcoe/fcoe_dev.c
+++ b/drivers/scsi/ofc/fcoe/fcoe_dev.c
@@ -185,12 +185,48 @@ static void fcoe_skb_destroy(struct sk_buff *skb)
 	if (unlikely(fp->fr_destructor))
 		(fp->fr_destructor) (fp->fr_arg);
 
-	skb_shinfo(skb)->nr_frags = 0;
 	atomic_dec(&fci->cleanup_needed);
 	if (fc && fc->fcoe_pending_queue.qlen)
 		fcoe_check_wait_queue(fdev);
 }
 
+static int fcoe_get_paged_crc_eof(struct sk_buff *skb, int tlen)
+{
+	struct fcoe_info *fci = &fcoei;
+	struct fcoe_percpu_s *fps;
+	struct page *page;
+	int cpu_idx;
+
+	cpu_idx = get_cpu();
+	fps = fci->fcoe_percpu[cpu_idx];
+	page = fps->crc_eof_page;
+	if (!page) {
+		page = alloc_page(GFP_ATOMIC);
+		if (!page) {
+			put_cpu();
+			return -ENOMEM;
+		}
+		fps->crc_eof_page = page;
+		WARN_ON(fps->crc_eof_offset != 0);
+	}
+
+	get_page(page);
+	skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, page,
+			   fps->crc_eof_offset, tlen);
+	skb->len += tlen;
+	skb->data_len += tlen;
+	skb->truesize += tlen;
+	fps->crc_eof_offset += sizeof(struct fcoe_crc_eof);
+
+	if (fps->crc_eof_offset >= PAGE_SIZE) {
+		fps->crc_eof_page = NULL;
+		fps->crc_eof_offset = 0;
+		put_page(page);
+	}
+	put_cpu();
+	return 0;
+}
+
 /*
  * Allocate a frame intended to be sent via fcoe_xmit.
  * Get an sk_buff for the frame and set the length.
@@ -239,7 +275,6 @@ int fcoe_xmit(struct fcdev *fc_dev, struct fc_frame *fp)
 	struct scatterlist *sg;
 	void *data;
 	uint8_t sof, eof;
-	int flg = 0;
 
 	if (unlikely(debug_fcoe))
 		fc_print_frame_hdr("fcoe_xmit", fp);
@@ -287,6 +322,7 @@ int fcoe_xmit(struct fcdev *fc_dev, struct fc_frame *fp)
 		skb_fill_page_desc(skb, indx, sg_page(sg), off, len);
 		skb->len += len;
 		skb->data_len += len;
+		skb->truesize += len;
 		while (len > 0) {
 			u_long clen;
 
@@ -321,22 +357,14 @@ int fcoe_xmit(struct fcdev *fc_dev, struct fc_frame *fp)
 	 * setup the destructor to remove this frag.
 	 */
 	if (skb_is_nonlinear(skb)) {
-		struct page *p;
-		cp = (struct fcoe_crc_eof *)fp->fr_util;
-		p = virt_to_page((char *)cp);
-		/*
-		 * set the destructor and inc the cleanup_needed
-		 * this is done because before unloading the module we should
-		 * wait for all the calls to fcoe_skb_destroy()
-		 */
-		skb->destructor = fcoe_skb_destroy;
-		atomic_inc(&fci->cleanup_needed);
-		flg = 1;
-		skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
-				   p, offset_in_page((char *)cp), tlen);
-		skb->len += tlen;
-		skb->data_len += tlen;
-		skb->truesize += tlen;
+		skb_frag_t *frag;
+		if (fcoe_get_paged_crc_eof(skb, tlen)) {
+			kfree(skb);
+			return -ENOMEM;
+		}
+		frag = &skb_shinfo(skb)->frags[skb_shinfo(skb)->nr_frags - 1];
+		cp = kmap_atomic(frag->page, KM_SKB_DATA_SOFTIRQ) +
+			frag->page_offset;
 	} else {
 		cp = (struct fcoe_crc_eof *)skb_put(skb, tlen);
 	}
@@ -347,8 +375,7 @@ int fcoe_xmit(struct fcdev *fc_dev, struct fc_frame *fp)
 	 */
 	if (unlikely(fp->fr_destructor)) {
 		skb->destructor = fcoe_skb_destroy;
-		if (flg == 0)
-			atomic_inc(&fci->cleanup_needed);
+		atomic_inc(&fci->cleanup_needed);
 	}
 
 	cp->fcoe_eof = eof;
@@ -357,6 +384,11 @@ int fcoe_xmit(struct fcdev *fc_dev, struct fc_frame *fp)
 		memset(cp->fcoe_resvd, 0, sizeof(cp->fcoe_resvd));
 	wlen = (skb->len - tlen + sizeof(crc)) / FCOE_WORD_TO_BYTE;
 
+	if (skb_is_nonlinear(skb)) {
+		kunmap_atomic(cp, KM_SKB_DATA_SOFTIRQ);
+		cp = NULL;
+	}
+
 	/*
 	 *      Fill in the control structures
 	 */
diff --git a/drivers/scsi/ofc/fcoe/fcoeinit.c b/drivers/scsi/ofc/fcoe/fcoeinit.c
index 4b03382..191c4f8 100644
--- a/drivers/scsi/ofc/fcoe/fcoeinit.c
+++ b/drivers/scsi/ofc/fcoe/fcoeinit.c
@@ -371,6 +371,8 @@ static void __exit fcoe_exit(void)
 			while ((skb = __skb_dequeue(&p->fcoe_rx_list)) != NULL)
 				kfree_skb(skb);
 			spin_unlock_bh(&p->fcoe_rx_list.lock);
+			if (fci->fcoe_percpu[idx]->crc_eof_page)
+				put_page(fci->fcoe_percpu[idx]->crc_eof_page);
 			kfree(fci->fcoe_percpu[idx]);
 		}
 	}
diff --git a/drivers/scsi/ofc/openfc/openfc_scsi.c b/drivers/scsi/ofc/openfc/openfc_scsi.c
index bfd68d9..806f315 100644
--- a/drivers/scsi/ofc/openfc/openfc_scsi.c
+++ b/drivers/scsi/ofc/openfc/openfc_scsi.c
@@ -306,13 +306,9 @@ static void openfc_scsi_send_data(struct fc_scsi_pkt *fsp, struct fc_seq *sp,
 		}
 		sg_bytes = min(tlen, sg->length - offset);
 		if (using_sg) {
-			/*
-			 * do not need to do get page
-			 * because we can unpin the memory when the
-			 * status of this I/O is received
-			 */
 			WARN_ON(fp->fr_sg_len > FC_FRAME_SG_LEN);
 			fsg = &fp->fr_sg[fp->fr_sg_len++];
+			get_page(sg_page(sg));
 			sg_set_page(fsg, sg_page(sg),
 				    sg_bytes, sg->offset + offset);
 		} else {




More information about the devel mailing list