/*
  * Filename: dma.c
  *
  *
  * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
  *	Philip Kelleher <pjk1939@linux.vnet.ibm.com>
  *
  * (C) Copyright 2013 IBM Corporation
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
  * License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  */
  
  #include <linux/slab.h>
  #include "rsxx_priv.h"
  
  struct rsxx_dma {
  	struct list_head	 list;
  	u8			 cmd;
  	unsigned int		 laddr;     /* Logical address */
  	struct {
  		u32		 off;
  		u32		 cnt;
  	} sub_page;
  	dma_addr_t		 dma_addr;
  	struct page		 *page;
  	unsigned int		 pg_off;    /* Page Offset */
  	rsxx_dma_cb		 cb;
  	void			 *cb_data;
  };
  
  /* This timeout is used to detect a stalled DMA channel */
  #define DMA_ACTIVITY_TIMEOUT	msecs_to_jiffies(10000)
  
  struct hw_status {
  	u8	status;
  	u8	tag;
  	__le16	count;
  	__le32	_rsvd2;
  	__le64	_rsvd3;
  } __packed;
  
  enum rsxx_dma_status {
  	DMA_SW_ERR    = 0x1,
  	DMA_HW_FAULT  = 0x2,
  	DMA_CANCELLED = 0x4,
  };
  
  struct hw_cmd {
  	u8	command;
  	u8	tag;
  	u8	_rsvd;
  	u8	sub_page; /* Bit[0:2]: 512byte offset */
  			  /* Bit[4:6]: 512byte count */
  	__le32	device_addr;
  	__le64	host_addr;
  } __packed;
  
  enum rsxx_hw_cmd {
  	HW_CMD_BLK_DISCARD	= 0x70,
  	HW_CMD_BLK_WRITE	= 0x80,
  	HW_CMD_BLK_READ		= 0xC0,
  	HW_CMD_BLK_RECON_READ	= 0xE0,
  };
  
  enum rsxx_hw_status {
  	HW_STATUS_CRC		= 0x01,
  	HW_STATUS_HARD_ERR	= 0x02,
  	HW_STATUS_SOFT_ERR	= 0x04,
  	HW_STATUS_FAULT		= 0x08,
  };
  
  static struct kmem_cache *rsxx_dma_pool;
  
  struct dma_tracker {
  	int			next_tag;
  	struct rsxx_dma	*dma;
  };
  
  #define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \
  		(sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS))
  
  struct dma_tracker_list {
  	spinlock_t		lock;
  	int			head;
  	struct dma_tracker	list[0];
  };
  
  
  /*----------------- Misc Utility Functions -------------------*/
  static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
  {
  	unsigned long long tgt_addr8;
  
  	tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
  		      card->_stripe.upper_mask) |
  		    ((addr8) & card->_stripe.lower_mask);
  	do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
  	return tgt_addr8;
  }
  
  static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
  {
  	unsigned int tgt;
  
  	tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
  
  	return tgt;
  }
  
  void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
  {
  	/* Reset all DMA Command/Status Queues */
  	iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
  }
  
  static unsigned int get_dma_size(struct rsxx_dma *dma)
  {
  	if (dma->sub_page.cnt)
  		return dma->sub_page.cnt << 9;
  	else
  		return RSXX_HW_BLK_SIZE;
  }
  
  
  /*----------------- DMA Tracker -------------------*/
  static void set_tracker_dma(struct dma_tracker_list *trackers,
  			    int tag,
  			    struct rsxx_dma *dma)
  {
  	trackers->list[tag].dma = dma;
  }
  
  static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
  					    int tag)
  {
  	return trackers->list[tag].dma;
  }
  
  static int pop_tracker(struct dma_tracker_list *trackers)
  {
  	int tag;
  
  	spin_lock(&trackers->lock);
  	tag = trackers->head;
  	if (tag != -1) {
  		trackers->head = trackers->list[tag].next_tag;
  		trackers->list[tag].next_tag = -1;
  	}
  	spin_unlock(&trackers->lock);
  
  	return tag;
  }
  
  static void push_tracker(struct dma_tracker_list *trackers, int tag)
  {
  	spin_lock(&trackers->lock);
  	trackers->list[tag].next_tag = trackers->head;
  	trackers->head = tag;
  	trackers->list[tag].dma = NULL;
  	spin_unlock(&trackers->lock);
  }
  
  
  /*----------------- Interrupt Coalescing -------------*/
  /*
   * Interrupt Coalescing Register Format:
   * Interrupt Timer (64ns units) [15:0]
   * Interrupt Count [24:16]
   * Reserved [31:25]
  */
  #define INTR_COAL_LATENCY_MASK       (0x0000ffff)
  
  #define INTR_COAL_COUNT_SHIFT        16
  #define INTR_COAL_COUNT_BITS         9
  #define INTR_COAL_COUNT_MASK         (((1 << INTR_COAL_COUNT_BITS) - 1) << \
  					INTR_COAL_COUNT_SHIFT)
  #define INTR_COAL_LATENCY_UNITS_NS   64
  
  
  static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
  {
  	u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
  
  	if (mode == RSXX_INTR_COAL_DISABLED)
  		return 0;
  
  	return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
  			(latency_units & INTR_COAL_LATENCY_MASK);
  
  }
  
  static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
  {
  	int i;
  	u32 q_depth = 0;
  	u32 intr_coal;
  
  	if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
  	    unlikely(card->eeh_state))
  		return;
  
  	for (i = 0; i < card->n_targets; i++)
  		q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
  
  	intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
  				      q_depth / 2,
  				      card->config.data.intr_coal.latency);
  	iowrite32(intr_coal, card->regmap + INTR_COAL);
  }
  
  /*----------------- RSXX DMA Handling -------------------*/
  static void rsxx_free_dma(struct rsxx_dma_ctrl *ctrl, struct rsxx_dma *dma)
  {
  	if (dma->cmd != HW_CMD_BLK_DISCARD) {
  		if (!pci_dma_mapping_error(ctrl->card->dev, dma->dma_addr)) {
  			pci_unmap_page(ctrl->card->dev, dma->dma_addr,
  				       get_dma_size(dma),
  				       dma->cmd == HW_CMD_BLK_WRITE ?
  						   PCI_DMA_TODEVICE :
  						   PCI_DMA_FROMDEVICE);
  		}
  	}
  
  	kmem_cache_free(rsxx_dma_pool, dma);
  }
  
  static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
  				  struct rsxx_dma *dma,
  				  unsigned int status)
  {
  	if (status & DMA_SW_ERR)
  		ctrl->stats.dma_sw_err++;
  	if (status & DMA_HW_FAULT)
  		ctrl->stats.dma_hw_fault++;
  	if (status & DMA_CANCELLED)
  		ctrl->stats.dma_cancelled++;
  
  	if (dma->cb)
  		dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
  
  	rsxx_free_dma(ctrl, dma);
  }
  
  int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
  			   struct list_head *q, unsigned int done)
  {
  	struct rsxx_dma *dma;
  	struct rsxx_dma *tmp;
  	int cnt = 0;
  
  	list_for_each_entry_safe(dma, tmp, q, list) {
  		list_del(&dma->list);
  		if (done & COMPLETE_DMA)
  			rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
  		else
  			rsxx_free_dma(ctrl, dma);
  		cnt++;
  	}
  
  	return cnt;
  }
  
  static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
  				 struct rsxx_dma *dma)
  {
  	/*
  	 * Requeued DMAs go to the front of the queue so they are issued
  	 * first.
  	 */
  	spin_lock_bh(&ctrl->queue_lock);
  	ctrl->stats.sw_q_depth++;
  	list_add(&dma->list, &ctrl->queue);
  	spin_unlock_bh(&ctrl->queue_lock);
  }
  
  static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
  				      struct rsxx_dma *dma,
  				      u8 hw_st)
  {
  	unsigned int status = 0;
  	int requeue_cmd = 0;
  
  	dev_dbg(CARD_TO_DEV(ctrl->card),
  		"Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)
  ",
  		dma->cmd, dma->laddr, hw_st);
  
  	if (hw_st & HW_STATUS_CRC)
  		ctrl->stats.crc_errors++;
  	if (hw_st & HW_STATUS_HARD_ERR)
  		ctrl->stats.hard_errors++;
  	if (hw_st & HW_STATUS_SOFT_ERR)
  		ctrl->stats.soft_errors++;
  
  	switch (dma->cmd) {
  	case HW_CMD_BLK_READ:
  		if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
  			if (ctrl->card->scrub_hard) {
  				dma->cmd = HW_CMD_BLK_RECON_READ;
  				requeue_cmd = 1;
  				ctrl->stats.reads_retried++;
  			} else {
  				status |= DMA_HW_FAULT;
  				ctrl->stats.reads_failed++;
  			}
  		} else if (hw_st & HW_STATUS_FAULT) {
  			status |= DMA_HW_FAULT;
  			ctrl->stats.reads_failed++;
  		}
  
  		break;
  	case HW_CMD_BLK_RECON_READ:
  		if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
  			/* Data could not be reconstructed. */
  			status |= DMA_HW_FAULT;
  			ctrl->stats.reads_failed++;
  		}
  
  		break;
  	case HW_CMD_BLK_WRITE:
  		status |= DMA_HW_FAULT;
  		ctrl->stats.writes_failed++;
  
  		break;
  	case HW_CMD_BLK_DISCARD:
  		status |= DMA_HW_FAULT;
  		ctrl->stats.discards_failed++;
  
  		break;
  	default:
  		dev_err(CARD_TO_DEV(ctrl->card),
  			"Unknown command in DMA!(cmd: x%02x "
  			   "laddr x%08x st: x%02x
  ",
  			   dma->cmd, dma->laddr, hw_st);
  		status |= DMA_SW_ERR;
  
  		break;
  	}
  
  	if (requeue_cmd)
  		rsxx_requeue_dma(ctrl, dma);
  	else
  		rsxx_complete_dma(ctrl, dma, status);
  }
  
  static void dma_engine_stalled(unsigned long data)
  {
  	struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data;
  	int cnt;
  
  	if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
  	    unlikely(ctrl->card->eeh_state))
  		return;
  
  	if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
  		/*
  		 * The dma engine was stalled because the SW_CMD_IDX write
  		 * was lost. Issue it again to recover.
  		 */
  		dev_warn(CARD_TO_DEV(ctrl->card),
  			"SW_CMD_IDX write was lost, re-writing...
  ");
  		iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
  		mod_timer(&ctrl->activity_timer,
  			  jiffies + DMA_ACTIVITY_TIMEOUT);
  	} else {
  		dev_warn(CARD_TO_DEV(ctrl->card),
  			"DMA channel %d has stalled, faulting interface.
  ",
  			ctrl->id);
  		ctrl->card->dma_fault = 1;
  
  		/* Clean up the DMA queue */
  		spin_lock(&ctrl->queue_lock);
  		cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
  		spin_unlock(&ctrl->queue_lock);
  
  		cnt += rsxx_dma_cancel(ctrl);
  
  		if (cnt)
  			dev_info(CARD_TO_DEV(ctrl->card),
  				"Freed %d queued DMAs on channel %d
  ",
  				cnt, ctrl->id);
  	}
  }
  
  static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
  {
  	struct rsxx_dma *dma;
  	int tag;
  	int cmds_pending = 0;
  	struct hw_cmd *hw_cmd_buf;
  	int dir;
  
  	hw_cmd_buf = ctrl->cmd.buf;
  
  	if (unlikely(ctrl->card->halt) ||
  	    unlikely(ctrl->card->eeh_state))
  		return;
  
  	while (1) {
  		spin_lock_bh(&ctrl->queue_lock);
  		if (list_empty(&ctrl->queue)) {
  			spin_unlock_bh(&ctrl->queue_lock);
  			break;
  		}
  		spin_unlock_bh(&ctrl->queue_lock);
  
  		tag = pop_tracker(ctrl->trackers);
  		if (tag == -1)
  			break;
  
  		spin_lock_bh(&ctrl->queue_lock);
  		dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
  		list_del(&dma->list);
  		ctrl->stats.sw_q_depth--;
  		spin_unlock_bh(&ctrl->queue_lock);
  
  		/*
  		 * This will catch any DMAs that slipped in right before the
  		 * fault, but was queued after all the other DMAs were
  		 * cancelled.
  		 */
  		if (unlikely(ctrl->card->dma_fault)) {
  			push_tracker(ctrl->trackers, tag);
  			rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
  			continue;
  		}
  
  		if (dma->cmd != HW_CMD_BLK_DISCARD) {
  			if (dma->cmd == HW_CMD_BLK_WRITE)
  				dir = PCI_DMA_TODEVICE;
  			else
  				dir = PCI_DMA_FROMDEVICE;
  
  			/*
  			 * The function pci_map_page is placed here because we
  			 * can only, by design, issue up to 255 commands to the
  			 * hardware at one time per DMA channel. So the maximum
  			 * amount of mapped memory would be 255 * 4 channels *
  			 * 4096 Bytes which is less than 2GB, the limit of a x8
  			 * Non-HWWD PCIe slot. This way the pci_map_page
  			 * function should never fail because of a lack of
  			 * mappable memory.
  			 */
  			dma->dma_addr = pci_map_page(ctrl->card->dev, dma->page,
  					dma->pg_off, dma->sub_page.cnt << 9, dir);
  			if (pci_dma_mapping_error(ctrl->card->dev, dma->dma_addr)) {
  				push_tracker(ctrl->trackers, tag);
  				rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
  				continue;
  			}
  		}
  
  		set_tracker_dma(ctrl->trackers, tag, dma);
  		hw_cmd_buf[ctrl->cmd.idx].command  = dma->cmd;
  		hw_cmd_buf[ctrl->cmd.idx].tag      = tag;
  		hw_cmd_buf[ctrl->cmd.idx]._rsvd    = 0;
  		hw_cmd_buf[ctrl->cmd.idx].sub_page =
  					((dma->sub_page.cnt & 0x7) << 4) |
  					 (dma->sub_page.off & 0x7);
  
  		hw_cmd_buf[ctrl->cmd.idx].device_addr =
  					cpu_to_le32(dma->laddr);
  
  		hw_cmd_buf[ctrl->cmd.idx].host_addr =
  					cpu_to_le64(dma->dma_addr);
  
  		dev_dbg(CARD_TO_DEV(ctrl->card),
  			"Issue DMA%d(laddr %d tag %d) to idx %d
  ",
  			ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
  
  		ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
  		cmds_pending++;
  
  		if (dma->cmd == HW_CMD_BLK_WRITE)
  			ctrl->stats.writes_issued++;
  		else if (dma->cmd == HW_CMD_BLK_DISCARD)
  			ctrl->stats.discards_issued++;
  		else
  			ctrl->stats.reads_issued++;
  	}
  
  	/* Let HW know we've queued commands. */
  	if (cmds_pending) {
  		atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
  		mod_timer(&ctrl->activity_timer,
  			  jiffies + DMA_ACTIVITY_TIMEOUT);
  
  		if (unlikely(ctrl->card->eeh_state)) {
  			del_timer_sync(&ctrl->activity_timer);
  			return;
  		}
  
  		iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
  	}
  }
  
  static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl)
  {
  	struct rsxx_dma *dma;
  	unsigned long flags;
  	u16 count;
  	u8 status;
  	u8 tag;
  	struct hw_status *hw_st_buf;
  
  	hw_st_buf = ctrl->status.buf;
  
  	if (unlikely(ctrl->card->halt) ||
  	    unlikely(ctrl->card->dma_fault) ||
  	    unlikely(ctrl->card->eeh_state))
  		return;
  
  	count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
  
  	while (count == ctrl->e_cnt) {
  		/*
  		 * The read memory-barrier is necessary to keep aggressive
  		 * processors/optimizers (such as the PPC Apple G5) from
  		 * reordering the following status-buffer tag & status read
  		 * *before* the count read on subsequent iterations of the
  		 * loop!
  		 */
  		rmb();
  
  		status = hw_st_buf[ctrl->status.idx].status;
  		tag    = hw_st_buf[ctrl->status.idx].tag;
  
  		dma = get_tracker_dma(ctrl->trackers, tag);
  		if (dma == NULL) {
  			spin_lock_irqsave(&ctrl->card->irq_lock, flags);
  			rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
  			spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
  
  			dev_err(CARD_TO_DEV(ctrl->card),
  				"No tracker for tag %d "
  				"(idx %d id %d)
  ",
  				tag, ctrl->status.idx, ctrl->id);
  			return;
  		}
  
  		dev_dbg(CARD_TO_DEV(ctrl->card),
  			"Completing DMA%d"
  			"(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.
  ",
  			ctrl->id, dma->laddr, tag, status, count,
  			ctrl->status.idx);
  
  		atomic_dec(&ctrl->stats.hw_q_depth);
  
  		mod_timer(&ctrl->activity_timer,
  			  jiffies + DMA_ACTIVITY_TIMEOUT);
  
  		if (status)
  			rsxx_handle_dma_error(ctrl, dma, status);
  		else
  			rsxx_complete_dma(ctrl, dma, 0);
  
  		push_tracker(ctrl->trackers, tag);
  
  		ctrl->status.idx = (ctrl->status.idx + 1) &
  				   RSXX_CS_IDX_MASK;
  		ctrl->e_cnt++;
  
  		count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
  	}
  
  	dma_intr_coal_auto_tune(ctrl->card);
  
  	if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
  		del_timer_sync(&ctrl->activity_timer);
  
  	spin_lock_irqsave(&ctrl->card->irq_lock, flags);
  	rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
  	spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
  
  	spin_lock_bh(&ctrl->queue_lock);
  	if (ctrl->stats.sw_q_depth)
  		queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
  	spin_unlock_bh(&ctrl->queue_lock);
  }
  
  static void rsxx_schedule_issue(struct work_struct *work)
  {
  	struct rsxx_dma_ctrl *ctrl;
  
  	ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
  
  	mutex_lock(&ctrl->work_lock);
  	rsxx_issue_dmas(ctrl);
  	mutex_unlock(&ctrl->work_lock);
  }
  
  static void rsxx_schedule_done(struct work_struct *work)
  {
  	struct rsxx_dma_ctrl *ctrl;
  
  	ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
  
  	mutex_lock(&ctrl->work_lock);
  	rsxx_dma_done(ctrl);
  	mutex_unlock(&ctrl->work_lock);
  }
  
  static int rsxx_queue_discard(struct rsxx_cardinfo *card,
  				  struct list_head *q,
  				  unsigned int laddr,
  				  rsxx_dma_cb cb,
  				  void *cb_data)
  {
  	struct rsxx_dma *dma;
  
  	dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
  	if (!dma)
  		return -ENOMEM;
  
  	dma->cmd          = HW_CMD_BLK_DISCARD;
  	dma->laddr        = laddr;
  	dma->dma_addr     = 0;
  	dma->sub_page.off = 0;
  	dma->sub_page.cnt = 0;
  	dma->page         = NULL;
  	dma->pg_off       = 0;
  	dma->cb	          = cb;
  	dma->cb_data      = cb_data;
  
  	dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x
  ", dma->laddr);
  
  	list_add_tail(&dma->list, q);
  
  	return 0;
  }
  
  static int rsxx_queue_dma(struct rsxx_cardinfo *card,
  			      struct list_head *q,
  			      int dir,
  			      unsigned int dma_off,
  			      unsigned int dma_len,
  			      unsigned int laddr,
  			      struct page *page,
  			      unsigned int pg_off,
  			      rsxx_dma_cb cb,
  			      void *cb_data)
  {
  	struct rsxx_dma *dma;
  
  	dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
  	if (!dma)
  		return -ENOMEM;
  
  	dma->cmd          = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
  	dma->laddr        = laddr;
  	dma->sub_page.off = (dma_off >> 9);
  	dma->sub_page.cnt = (dma_len >> 9);
  	dma->page         = page;
  	dma->pg_off       = pg_off;
  	dma->cb	          = cb;
  	dma->cb_data      = cb_data;
  
  	dev_dbg(CARD_TO_DEV(card),
  		"Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d
  ",
  		dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
  		dma->sub_page.cnt, dma->page, dma->pg_off);
  
  	/* Queue the DMA */
  	list_add_tail(&dma->list, q);
  
  	return 0;
  }
  
  int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
  			   struct bio *bio,
  			   atomic_t *n_dmas,
  			   rsxx_dma_cb cb,
  			   void *cb_data)
  {
  	struct list_head dma_list[RSXX_MAX_TARGETS];
  	struct bio_vec bvec;
  	struct bvec_iter iter;
  	unsigned long long addr8;
  	unsigned int laddr;
  	unsigned int bv_len;
  	unsigned int bv_off;
  	unsigned int dma_off;
  	unsigned int dma_len;
  	int dma_cnt[RSXX_MAX_TARGETS];
  	int tgt;
  	int st;
  	int i;
  
  	addr8 = bio->bi_iter.bi_sector << 9; /* sectors are 512 bytes */
  	atomic_set(n_dmas, 0);
  
  	for (i = 0; i < card->n_targets; i++) {
  		INIT_LIST_HEAD(&dma_list[i]);
  		dma_cnt[i] = 0;
  	}
  
  	if (bio->bi_rw & REQ_DISCARD) {
  		bv_len = bio->bi_iter.bi_size;
  
  		while (bv_len > 0) {
  			tgt   = rsxx_get_dma_tgt(card, addr8);
  			laddr = rsxx_addr8_to_laddr(addr8, card);
  
  			st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
  						    cb, cb_data);
  			if (st)
  				goto bvec_err;
  
  			dma_cnt[tgt]++;
  			atomic_inc(n_dmas);
  			addr8  += RSXX_HW_BLK_SIZE;
  			bv_len -= RSXX_HW_BLK_SIZE;
  		}
  	} else {
  		bio_for_each_segment(bvec, bio, iter) {
  			bv_len = bvec.bv_len;
  			bv_off = bvec.bv_offset;
  
  			while (bv_len > 0) {
  				tgt   = rsxx_get_dma_tgt(card, addr8);
  				laddr = rsxx_addr8_to_laddr(addr8, card);
  				dma_off = addr8 & RSXX_HW_BLK_MASK;
  				dma_len = min(bv_len,
  					      RSXX_HW_BLK_SIZE - dma_off);
  
  				st = rsxx_queue_dma(card, &dma_list[tgt],
  							bio_data_dir(bio),
  							dma_off, dma_len,
  							laddr, bvec.bv_page,
  							bv_off, cb, cb_data);
  				if (st)
  					goto bvec_err;
  
  				dma_cnt[tgt]++;
  				atomic_inc(n_dmas);
  				addr8  += dma_len;
  				bv_off += dma_len;
  				bv_len -= dma_len;
  			}
  		}
  	}
  
  	for (i = 0; i < card->n_targets; i++) {
  		if (!list_empty(&dma_list[i])) {
  			spin_lock_bh(&card->ctrl[i].queue_lock);
  			card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
  			list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
  			spin_unlock_bh(&card->ctrl[i].queue_lock);
  
  			queue_work(card->ctrl[i].issue_wq,
  				   &card->ctrl[i].issue_dma_work);
  		}
  	}
  
  	return 0;
  
  bvec_err:
  	for (i = 0; i < card->n_targets; i++)
  		rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i],
  					FREE_DMA);
  
  	return st;
  }
  
  
  /*----------------- DMA Engine Initialization & Setup -------------------*/
  int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
  {
  	ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8,
  				&ctrl->status.dma_addr);
  	ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8,
  				&ctrl->cmd.dma_addr);
  	if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
  		return -ENOMEM;
  
  	memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
  	iowrite32(lower_32_bits(ctrl->status.dma_addr),
  		ctrl->regmap + SB_ADD_LO);
  	iowrite32(upper_32_bits(ctrl->status.dma_addr),
  		ctrl->regmap + SB_ADD_HI);
  
  	memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
  	iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
  	iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
  
  	ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
  	if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
  		dev_crit(&dev->dev, "Failed reading status cnt x%x
  ",
  			ctrl->status.idx);
  		return -EINVAL;
  	}
  	iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
  	iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
  
  	ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
  	if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
  		dev_crit(&dev->dev, "Failed reading cmd cnt x%x
  ",
  			ctrl->status.idx);
  		return -EINVAL;
  	}
  	iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
  	iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
  
  	return 0;
  }
  
  static int rsxx_dma_ctrl_init(struct pci_dev *dev,
  				  struct rsxx_dma_ctrl *ctrl)
  {
  	int i;
  	int st;
  
  	memset(&ctrl->stats, 0, sizeof(ctrl->stats));
  
  	ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8);
  	if (!ctrl->trackers)
  		return -ENOMEM;
  
  	ctrl->trackers->head = 0;
  	for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
  		ctrl->trackers->list[i].next_tag = i + 1;
  		ctrl->trackers->list[i].dma = NULL;
  	}
  	ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
  	spin_lock_init(&ctrl->trackers->lock);
  
  	spin_lock_init(&ctrl->queue_lock);
  	mutex_init(&ctrl->work_lock);
  	INIT_LIST_HEAD(&ctrl->queue);
  
  	setup_timer(&ctrl->activity_timer, dma_engine_stalled,
  					(unsigned long)ctrl);
  
  	ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
  	if (!ctrl->issue_wq)
  		return -ENOMEM;
  
  	ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
  	if (!ctrl->done_wq)
  		return -ENOMEM;
  
  	INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue);
  	INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done);
  
  	st = rsxx_hw_buffers_init(dev, ctrl);
  	if (st)
  		return st;
  
  	return 0;
  }
  
  static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
  			      unsigned int stripe_size8)
  {
  	if (!is_power_of_2(stripe_size8)) {
  		dev_err(CARD_TO_DEV(card),
  			"stripe_size is NOT a power of 2!
  ");
  		return -EINVAL;
  	}
  
  	card->_stripe.lower_mask = stripe_size8 - 1;
  
  	card->_stripe.upper_mask  = ~(card->_stripe.lower_mask);
  	card->_stripe.upper_shift = ffs(card->n_targets) - 1;
  
  	card->_stripe.target_mask = card->n_targets - 1;
  	card->_stripe.target_shift = ffs(stripe_size8) - 1;
  
  	dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask   = x%016llx
  ",
  		card->_stripe.lower_mask);
  	dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift  = x%016llx
  ",
  		card->_stripe.upper_shift);
  	dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask   = x%016llx
  ",
  		card->_stripe.upper_mask);
  	dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask  = x%016llx
  ",
  		card->_stripe.target_mask);
  	dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx
  ",
  		card->_stripe.target_shift);
  
  	return 0;
  }
  
  int rsxx_dma_configure(struct rsxx_cardinfo *card)
  {
  	u32 intr_coal;
  
  	intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
  				      card->config.data.intr_coal.count,
  				      card->config.data.intr_coal.latency);
  	iowrite32(intr_coal, card->regmap + INTR_COAL);
  
  	return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
  }
  
  int rsxx_dma_setup(struct rsxx_cardinfo *card)
  {
  	unsigned long flags;
  	int st;
  	int i;
  
  	dev_info(CARD_TO_DEV(card),
  		"Initializing %d DMA targets
  ",
  		card->n_targets);
  
  	/* Regmap is divided up into 4K chunks. One for each DMA channel */
  	for (i = 0; i < card->n_targets; i++)
  		card->ctrl[i].regmap = card->regmap + (i * 4096);
  
  	card->dma_fault = 0;
  
  	/* Reset the DMA queues */
  	rsxx_dma_queue_reset(card);
  
  	/************* Setup DMA Control *************/
  	for (i = 0; i < card->n_targets; i++) {
  		st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
  		if (st)
  			goto failed_dma_setup;
  
  		card->ctrl[i].card = card;
  		card->ctrl[i].id = i;
  	}
  
  	card->scrub_hard = 1;
  
  	if (card->config_valid)
  		rsxx_dma_configure(card);
  
  	/* Enable the interrupts after all setup has completed. */
  	for (i = 0; i < card->n_targets; i++) {
  		spin_lock_irqsave(&card->irq_lock, flags);
  		rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
  		spin_unlock_irqrestore(&card->irq_lock, flags);
  	}
  
  	return 0;
  
  failed_dma_setup:
  	for (i = 0; i < card->n_targets; i++) {
  		struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
  
  		if (ctrl->issue_wq) {
  			destroy_workqueue(ctrl->issue_wq);
  			ctrl->issue_wq = NULL;
  		}
  
  		if (ctrl->done_wq) {
  			destroy_workqueue(ctrl->done_wq);
  			ctrl->done_wq = NULL;
  		}
  
  		if (ctrl->trackers)
  			vfree(ctrl->trackers);
  
  		if (ctrl->status.buf)
  			pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
  					    ctrl->status.buf,
  					    ctrl->status.dma_addr);
  		if (ctrl->cmd.buf)
  			pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
  					    ctrl->cmd.buf, ctrl->cmd.dma_addr);
  	}
  
  	return st;
  }
  
  int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
  {
  	struct rsxx_dma *dma;
  	int i;
  	int cnt = 0;
  
  	/* Clean up issued DMAs */
  	for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
  		dma = get_tracker_dma(ctrl->trackers, i);
  		if (dma) {
  			atomic_dec(&ctrl->stats.hw_q_depth);
  			rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
  			push_tracker(ctrl->trackers, i);
  			cnt++;
  		}
  	}
  
  	return cnt;
  }
  
  void rsxx_dma_destroy(struct rsxx_cardinfo *card)
  {
  	struct rsxx_dma_ctrl *ctrl;
  	int i;
  
  	for (i = 0; i < card->n_targets; i++) {
  		ctrl = &card->ctrl[i];
  
  		if (ctrl->issue_wq) {
  			destroy_workqueue(ctrl->issue_wq);
  			ctrl->issue_wq = NULL;
  		}
  
  		if (ctrl->done_wq) {
  			destroy_workqueue(ctrl->done_wq);
  			ctrl->done_wq = NULL;
  		}
  
  		if (timer_pending(&ctrl->activity_timer))
  			del_timer_sync(&ctrl->activity_timer);
  
  		/* Clean up the DMA queue */
  		spin_lock_bh(&ctrl->queue_lock);
  		rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
  		spin_unlock_bh(&ctrl->queue_lock);
  
  		rsxx_dma_cancel(ctrl);
  
  		vfree(ctrl->trackers);
  
  		pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
  				    ctrl->status.buf, ctrl->status.dma_addr);
  		pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
  				    ctrl->cmd.buf, ctrl->cmd.dma_addr);
  	}
  }
  
  int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
  {
  	int i;
  	int j;
  	int cnt;
  	struct rsxx_dma *dma;
  	struct list_head *issued_dmas;
  
  	issued_dmas = kzalloc(sizeof(*issued_dmas) * card->n_targets,
  			      GFP_KERNEL);
  	if (!issued_dmas)
  		return -ENOMEM;
  
  	for (i = 0; i < card->n_targets; i++) {
  		INIT_LIST_HEAD(&issued_dmas[i]);
  		cnt = 0;
  		for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
  			dma = get_tracker_dma(card->ctrl[i].trackers, j);
  			if (dma == NULL)
  				continue;
  
  			if (dma->cmd == HW_CMD_BLK_WRITE)
  				card->ctrl[i].stats.writes_issued--;
  			else if (dma->cmd == HW_CMD_BLK_DISCARD)
  				card->ctrl[i].stats.discards_issued--;
  			else
  				card->ctrl[i].stats.reads_issued--;
  
  			if (dma->cmd != HW_CMD_BLK_DISCARD) {
  				pci_unmap_page(card->dev, dma->dma_addr,
  					       get_dma_size(dma),
  					       dma->cmd == HW_CMD_BLK_WRITE ?
  					       PCI_DMA_TODEVICE :
  					       PCI_DMA_FROMDEVICE);
  			}
  
  			list_add_tail(&dma->list, &issued_dmas[i]);
  			push_tracker(card->ctrl[i].trackers, j);
  			cnt++;
  		}
  
  		spin_lock_bh(&card->ctrl[i].queue_lock);
  		list_splice(&issued_dmas[i], &card->ctrl[i].queue);
  
  		atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
  		card->ctrl[i].stats.sw_q_depth += cnt;
  		card->ctrl[i].e_cnt = 0;
  		spin_unlock_bh(&card->ctrl[i].queue_lock);
  	}
  
  	kfree(issued_dmas);
  
  	return 0;
  }
  
  int rsxx_dma_init(void)
  {
  	rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
  	if (!rsxx_dma_pool)
  		return -ENOMEM;
  
  	return 0;
  }
  
  
  void rsxx_dma_cleanup(void)
  {
  	kmem_cache_destroy(rsxx_dma_pool);
  }