/*
   * This file is part of the Chelsio FCoE driver for Linux.
   *
   * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
   *
   * This software is available to you under a choice of one of two
   * licenses.  You may choose to be licensed under the terms of the GNU
   * General Public License (GPL) Version 2, available from the file
   * COPYING in the main directory of this source tree, or the
   * OpenIB.org BSD license below:
   *
   *     Redistribution and use in source and binary forms, with or
   *     without modification, are permitted provided that the following
   *     conditions are met:
   *
   *      - Redistributions of source code must retain the above
   *        copyright notice, this list of conditions and the following
   *        disclaimer.
   *
   *      - Redistributions in binary form must reproduce the above
   *        copyright notice, this list of conditions and the following
   *        disclaimer in the documentation and/or other materials
   *        provided with the distribution.
   *
   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
   * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
   * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
   * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
   * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
   * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
   * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   * SOFTWARE.
   */
  
  #include <linux/kernel.h>
  #include <linux/pci.h>
  #include <linux/interrupt.h>
  #include <linux/cpumask.h>
  #include <linux/string.h>
  
  #include "csio_init.h"
  #include "csio_hw.h"
  
  static irqreturn_t
  csio_nondata_isr(int irq, void *dev_id)
  {
  	struct csio_hw *hw = (struct csio_hw *) dev_id;
  	int rv;
  	unsigned long flags;
  
  	if (unlikely(!hw))
  		return IRQ_NONE;
  
  	if (unlikely(pci_channel_offline(hw->pdev))) {
  		CSIO_INC_STATS(hw, n_pcich_offline);
  		return IRQ_NONE;
  	}
  
  	spin_lock_irqsave(&hw->lock, flags);
  	csio_hw_slow_intr_handler(hw);
  	rv = csio_mb_isr_handler(hw);
  
  	if (rv == 0 && !(hw->flags & CSIO_HWF_FWEVT_PENDING)) {
  		hw->flags |= CSIO_HWF_FWEVT_PENDING;
  		spin_unlock_irqrestore(&hw->lock, flags);
  		schedule_work(&hw->evtq_work);
  		return IRQ_HANDLED;
  	}
  	spin_unlock_irqrestore(&hw->lock, flags);
  	return IRQ_HANDLED;
  }
  
  /*
   * csio_fwevt_handler - Common FW event handler routine.
   * @hw: HW module.
   *
   * This is the ISR for FW events. It is shared b/w MSIX
   * and INTx handlers.
   */
  static void
  csio_fwevt_handler(struct csio_hw *hw)
  {
  	int rv;
  	unsigned long flags;
  
  	rv = csio_fwevtq_handler(hw);
  
  	spin_lock_irqsave(&hw->lock, flags);
  	if (rv == 0 && !(hw->flags & CSIO_HWF_FWEVT_PENDING)) {
  		hw->flags |= CSIO_HWF_FWEVT_PENDING;
  		spin_unlock_irqrestore(&hw->lock, flags);
  		schedule_work(&hw->evtq_work);
  		return;
  	}
  	spin_unlock_irqrestore(&hw->lock, flags);
  
  } /* csio_fwevt_handler */
  
  /*
   * csio_fwevt_isr() - FW events MSIX ISR
   * @irq:
   * @dev_id:
   *
   * Process WRs on the FW event queue.
   *
   */
  static irqreturn_t
  csio_fwevt_isr(int irq, void *dev_id)
  {
  	struct csio_hw *hw = (struct csio_hw *) dev_id;
  
  	if (unlikely(!hw))
  		return IRQ_NONE;
  
  	if (unlikely(pci_channel_offline(hw->pdev))) {
  		CSIO_INC_STATS(hw, n_pcich_offline);
  		return IRQ_NONE;
  	}
  
  	csio_fwevt_handler(hw);
  
  	return IRQ_HANDLED;
  }
  
  /*
   * csio_fwevt_isr() - INTx wrapper for handling FW events.
   * @irq:
   * @dev_id:
   */
  void
  csio_fwevt_intx_handler(struct csio_hw *hw, void *wr, uint32_t len,
  			   struct csio_fl_dma_buf *flb, void *priv)
  {
  	csio_fwevt_handler(hw);
  } /* csio_fwevt_intx_handler */
  
  /*
   * csio_process_scsi_cmpl - Process a SCSI WR completion.
   * @hw: HW module.
   * @wr: The completed WR from the ingress queue.
   * @len: Length of the WR.
   * @flb: Freelist buffer array.
   *
   */
  static void
  csio_process_scsi_cmpl(struct csio_hw *hw, void *wr, uint32_t len,
  			struct csio_fl_dma_buf *flb, void *cbfn_q)
  {
  	struct csio_ioreq *ioreq;
  	uint8_t *scsiwr;
  	uint8_t subop;
  	void *cmnd;
  	unsigned long flags;
  
  	ioreq = csio_scsi_cmpl_handler(hw, wr, len, flb, NULL, &scsiwr);
  	if (likely(ioreq)) {
  		if (unlikely(*scsiwr == FW_SCSI_ABRT_CLS_WR)) {
  			subop = FW_SCSI_ABRT_CLS_WR_SUB_OPCODE_GET(
  					((struct fw_scsi_abrt_cls_wr *)
  					    scsiwr)->sub_opcode_to_chk_all_io);
  
  			csio_dbg(hw, "%s cmpl recvd ioreq:%p status:%d
  ",
  				    subop ? "Close" : "Abort",
  				    ioreq, ioreq->wr_status);
  
  			spin_lock_irqsave(&hw->lock, flags);
  			if (subop)
  				csio_scsi_closed(ioreq,
  						 (struct list_head *)cbfn_q);
  			else
  				csio_scsi_aborted(ioreq,
  						  (struct list_head *)cbfn_q);
  			/*
  			 * We call scsi_done for I/Os that driver thinks aborts
  			 * have timed out. If there is a race caused by FW
  			 * completing abort at the exact same time that the
  			 * driver has deteced the abort timeout, the following
  			 * check prevents calling of scsi_done twice for the
  			 * same command: once from the eh_abort_handler, another
  			 * from csio_scsi_isr_handler(). This also avoids the
  			 * need to check if csio_scsi_cmnd(req) is NULL in the
  			 * fast path.
  			 */
  			cmnd = csio_scsi_cmnd(ioreq);
  			if (unlikely(cmnd == NULL))
  				list_del_init(&ioreq->sm.sm_list);
  
  			spin_unlock_irqrestore(&hw->lock, flags);
  
  			if (unlikely(cmnd == NULL))
  				csio_put_scsi_ioreq_lock(hw,
  						csio_hw_to_scsim(hw), ioreq);
  		} else {
  			spin_lock_irqsave(&hw->lock, flags);
  			csio_scsi_completed(ioreq, (struct list_head *)cbfn_q);
  			spin_unlock_irqrestore(&hw->lock, flags);
  		}
  	}
  }
  
  /*
   * csio_scsi_isr_handler() - Common SCSI ISR handler.
   * @iq: Ingress queue pointer.
   *
   * Processes SCSI completions on the SCSI IQ indicated by scm->iq_idx
   * by calling csio_wr_process_iq_idx. If there are completions on the
   * isr_cbfn_q, yank them out into a local queue and call their io_cbfns.
   * Once done, add these completions onto the freelist.
   * This routine is shared b/w MSIX and INTx.
   */
  static inline irqreturn_t
  csio_scsi_isr_handler(struct csio_q *iq)
  {
  	struct csio_hw *hw = (struct csio_hw *)iq->owner;
  	LIST_HEAD(cbfn_q);
  	struct list_head *tmp;
  	struct csio_scsim *scm;
  	struct csio_ioreq *ioreq;
  	int isr_completions = 0;
  
  	scm = csio_hw_to_scsim(hw);
  
  	if (unlikely(csio_wr_process_iq(hw, iq, csio_process_scsi_cmpl,
  					&cbfn_q) != 0))
  		return IRQ_NONE;
  
  	/* Call back the completion routines */
  	list_for_each(tmp, &cbfn_q) {
  		ioreq = (struct csio_ioreq *)tmp;
  		isr_completions++;
  		ioreq->io_cbfn(hw, ioreq);
  		/* Release ddp buffer if used for this req */
  		if (unlikely(ioreq->dcopy))
  			csio_put_scsi_ddp_list_lock(hw, scm, &ioreq->gen_list,
  						    ioreq->nsge);
  	}
  
  	if (isr_completions) {
  		/* Return the ioreqs back to ioreq->freelist */
  		csio_put_scsi_ioreq_list_lock(hw, scm, &cbfn_q,
  					      isr_completions);
  	}
  
  	return IRQ_HANDLED;
  }
  
  /*
   * csio_scsi_isr() - SCSI MSIX handler
   * @irq:
   * @dev_id:
   *
   * This is the top level SCSI MSIX handler. Calls csio_scsi_isr_handler()
   * for handling SCSI completions.
   */
  static irqreturn_t
  csio_scsi_isr(int irq, void *dev_id)
  {
  	struct csio_q *iq = (struct csio_q *) dev_id;
  	struct csio_hw *hw;
  
  	if (unlikely(!iq))
  		return IRQ_NONE;
  
  	hw = (struct csio_hw *)iq->owner;
  
  	if (unlikely(pci_channel_offline(hw->pdev))) {
  		CSIO_INC_STATS(hw, n_pcich_offline);
  		return IRQ_NONE;
  	}
  
  	csio_scsi_isr_handler(iq);
  
  	return IRQ_HANDLED;
  }
  
  /*
   * csio_scsi_intx_handler() - SCSI INTx handler
   * @irq:
   * @dev_id:
   *
   * This is the top level SCSI INTx handler. Calls csio_scsi_isr_handler()
   * for handling SCSI completions.
   */
  void
  csio_scsi_intx_handler(struct csio_hw *hw, void *wr, uint32_t len,
  			struct csio_fl_dma_buf *flb, void *priv)
  {
  	struct csio_q *iq = priv;
  
  	csio_scsi_isr_handler(iq);
  
  } /* csio_scsi_intx_handler */
  
  /*
   * csio_fcoe_isr() - INTx/MSI interrupt service routine for FCoE.
   * @irq:
   * @dev_id:
   *
   *
   */
  static irqreturn_t
  csio_fcoe_isr(int irq, void *dev_id)
  {
  	struct csio_hw *hw = (struct csio_hw *) dev_id;
  	struct csio_q *intx_q = NULL;
  	int rv;
  	irqreturn_t ret = IRQ_NONE;
  	unsigned long flags;
  
  	if (unlikely(!hw))
  		return IRQ_NONE;
  
  	if (unlikely(pci_channel_offline(hw->pdev))) {
  		CSIO_INC_STATS(hw, n_pcich_offline);
  		return IRQ_NONE;
  	}
  
  	/* Disable the interrupt for this PCI function. */
  	if (hw->intr_mode == CSIO_IM_INTX)
  		csio_wr_reg32(hw, 0, MYPF_REG(PCIE_PF_CLI_A));
  
  	/*
  	 * The read in the following function will flush the
  	 * above write.
  	 */
  	if (csio_hw_slow_intr_handler(hw))
  		ret = IRQ_HANDLED;
  
  	/* Get the INTx Forward interrupt IQ. */
  	intx_q = csio_get_q(hw, hw->intr_iq_idx);
  
  	CSIO_DB_ASSERT(intx_q);
  
  	/* IQ handler is not possible for intx_q, hence pass in NULL */
  	if (likely(csio_wr_process_iq(hw, intx_q, NULL, NULL) == 0))
  		ret = IRQ_HANDLED;
  
  	spin_lock_irqsave(&hw->lock, flags);
  	rv = csio_mb_isr_handler(hw);
  	if (rv == 0 && !(hw->flags & CSIO_HWF_FWEVT_PENDING)) {
  		hw->flags |= CSIO_HWF_FWEVT_PENDING;
  		spin_unlock_irqrestore(&hw->lock, flags);
  		schedule_work(&hw->evtq_work);
  		return IRQ_HANDLED;
  	}
  	spin_unlock_irqrestore(&hw->lock, flags);
  
  	return ret;
  }
  
  static void
  csio_add_msix_desc(struct csio_hw *hw)
  {
  	int i;
  	struct csio_msix_entries *entryp = &hw->msix_entries[0];
  	int k = CSIO_EXTRA_VECS;
  	int len = sizeof(entryp->desc) - 1;
  	int cnt = hw->num_sqsets + k;
  
  	/* Non-data vector */
  	memset(entryp->desc, 0, len + 1);
  	snprintf(entryp->desc, len, "csio-%02x:%02x:%x-nondata",
  		 CSIO_PCI_BUS(hw), CSIO_PCI_DEV(hw), CSIO_PCI_FUNC(hw));
  
  	entryp++;
  	memset(entryp->desc, 0, len + 1);
  	snprintf(entryp->desc, len, "csio-%02x:%02x:%x-fwevt",
  		 CSIO_PCI_BUS(hw), CSIO_PCI_DEV(hw), CSIO_PCI_FUNC(hw));
  	entryp++;
  
  	/* Name SCSI vecs */
  	for (i = k; i < cnt; i++, entryp++) {
  		memset(entryp->desc, 0, len + 1);
  		snprintf(entryp->desc, len, "csio-%02x:%02x:%x-scsi%d",
  			 CSIO_PCI_BUS(hw), CSIO_PCI_DEV(hw),
  			 CSIO_PCI_FUNC(hw), i - CSIO_EXTRA_VECS);
  	}
  }
  
  int
  csio_request_irqs(struct csio_hw *hw)
  {
  	int rv, i, j, k = 0;
  	struct csio_msix_entries *entryp = &hw->msix_entries[0];
  	struct csio_scsi_cpu_info *info;
  
  	if (hw->intr_mode != CSIO_IM_MSIX) {
  		rv = request_irq(hw->pdev->irq, csio_fcoe_isr,
  					(hw->intr_mode == CSIO_IM_MSI) ?
  							0 : IRQF_SHARED,
  					KBUILD_MODNAME, hw);
  		if (rv) {
  			if (hw->intr_mode == CSIO_IM_MSI)
  				pci_disable_msi(hw->pdev);
  			csio_err(hw, "Failed to allocate interrupt line.
  ");
  			return -EINVAL;
  		}
  
  		goto out;
  	}
  
  	/* Add the MSIX vector descriptions */
  	csio_add_msix_desc(hw);
  
  	rv = request_irq(entryp[k].vector, csio_nondata_isr, 0,
  			 entryp[k].desc, hw);
  	if (rv) {
  		csio_err(hw, "IRQ request failed for vec %d err:%d
  ",
  			 entryp[k].vector, rv);
  		goto err;
  	}
  
  	entryp[k++].dev_id = (void *)hw;
  
  	rv = request_irq(entryp[k].vector, csio_fwevt_isr, 0,
  			 entryp[k].desc, hw);
  	if (rv) {
  		csio_err(hw, "IRQ request failed for vec %d err:%d
  ",
  			 entryp[k].vector, rv);
  		goto err;
  	}
  
  	entryp[k++].dev_id = (void *)hw;
  
  	/* Allocate IRQs for SCSI */
  	for (i = 0; i < hw->num_pports; i++) {
  		info = &hw->scsi_cpu_info[i];
  		for (j = 0; j < info->max_cpus; j++, k++) {
  			struct csio_scsi_qset *sqset = &hw->sqset[i][j];
  			struct csio_q *q = hw->wrm.q_arr[sqset->iq_idx];
  
  			rv = request_irq(entryp[k].vector, csio_scsi_isr, 0,
  					 entryp[k].desc, q);
  			if (rv) {
  				csio_err(hw,
  				       "IRQ request failed for vec %d err:%d
  ",
  				       entryp[k].vector, rv);
  				goto err;
  			}
  
  			entryp[k].dev_id = (void *)q;
  
  		} /* for all scsi cpus */
  	} /* for all ports */
  
  out:
  	hw->flags |= CSIO_HWF_HOST_INTR_ENABLED;
  
  	return 0;
  
  err:
  	for (i = 0; i < k; i++) {
  		entryp = &hw->msix_entries[i];
  		free_irq(entryp->vector, entryp->dev_id);
  	}
  	pci_disable_msix(hw->pdev);
  
  	return -EINVAL;
  }
  
  static void
  csio_disable_msix(struct csio_hw *hw, bool free)
  {
  	int i;
  	struct csio_msix_entries *entryp;
  	int cnt = hw->num_sqsets + CSIO_EXTRA_VECS;
  
  	if (free) {
  		for (i = 0; i < cnt; i++) {
  			entryp = &hw->msix_entries[i];
  			free_irq(entryp->vector, entryp->dev_id);
  		}
  	}
  	pci_disable_msix(hw->pdev);
  }
  
  /* Reduce per-port max possible CPUs */
  static void
  csio_reduce_sqsets(struct csio_hw *hw, int cnt)
  {
  	int i;
  	struct csio_scsi_cpu_info *info;
  
  	while (cnt < hw->num_sqsets) {
  		for (i = 0; i < hw->num_pports; i++) {
  			info = &hw->scsi_cpu_info[i];
  			if (info->max_cpus > 1) {
  				info->max_cpus--;
  				hw->num_sqsets--;
  				if (hw->num_sqsets <= cnt)
  					break;
  			}
  		}
  	}
  
  	csio_dbg(hw, "Reduced sqsets to %d
  ", hw->num_sqsets);
  }
  
  static int
  csio_enable_msix(struct csio_hw *hw)
  {
  	int i, j, k, n, min, cnt;
  	struct csio_msix_entries *entryp;
  	struct msix_entry *entries;
  	int extra = CSIO_EXTRA_VECS;
  	struct csio_scsi_cpu_info *info;
  
  	min = hw->num_pports + extra;
  	cnt = hw->num_sqsets + extra;
  
  	/* Max vectors required based on #niqs configured in fw */
  	if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS || !csio_is_hw_master(hw))
  		cnt = min_t(uint8_t, hw->cfg_niq, cnt);
  
  	entries = kzalloc(sizeof(struct msix_entry) * cnt, GFP_KERNEL);
  	if (!entries)
  		return -ENOMEM;
  
  	for (i = 0; i < cnt; i++)
  		entries[i].entry = (uint16_t)i;
  
  	csio_dbg(hw, "FW supp #niq:%d, trying %d msix's
  ", hw->cfg_niq, cnt);
  
  	cnt = pci_enable_msix_range(hw->pdev, entries, min, cnt);
  	if (cnt < 0) {
  		kfree(entries);
  		return cnt;
  	}
  
  	if (cnt < (hw->num_sqsets + extra)) {
  		csio_dbg(hw, "Reducing sqsets to %d
  ", cnt - extra);
  		csio_reduce_sqsets(hw, cnt - extra);
  	}
  
  	/* Save off vectors */
  	for (i = 0; i < cnt; i++) {
  		entryp = &hw->msix_entries[i];
  		entryp->vector = entries[i].vector;
  	}
  
  	/* Distribute vectors */
  	k = 0;
  	csio_set_nondata_intr_idx(hw, entries[k].entry);
  	csio_set_mb_intr_idx(csio_hw_to_mbm(hw), entries[k++].entry);
  	csio_set_fwevt_intr_idx(hw, entries[k++].entry);
  
  	for (i = 0; i < hw->num_pports; i++) {
  		info = &hw->scsi_cpu_info[i];
  
  		for (j = 0; j < hw->num_scsi_msix_cpus; j++) {
  			n = (j % info->max_cpus) +  k;
  			hw->sqset[i][j].intr_idx = entries[n].entry;
  		}
  
  		k += info->max_cpus;
  	}
  
  	kfree(entries);
  	return 0;
  }
  
  void
  csio_intr_enable(struct csio_hw *hw)
  {
  	hw->intr_mode = CSIO_IM_NONE;
  	hw->flags &= ~CSIO_HWF_HOST_INTR_ENABLED;
  
  	/* Try MSIX, then MSI or fall back to INTx */
  	if ((csio_msi == 2) && !csio_enable_msix(hw))
  		hw->intr_mode = CSIO_IM_MSIX;
  	else {
  		/* Max iqs required based on #niqs configured in fw */
  		if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS ||
  			!csio_is_hw_master(hw)) {
  			int extra = CSIO_EXTRA_MSI_IQS;
  
  			if (hw->cfg_niq < (hw->num_sqsets + extra)) {
  				csio_dbg(hw, "Reducing sqsets to %d
  ",
  					 hw->cfg_niq - extra);
  				csio_reduce_sqsets(hw, hw->cfg_niq - extra);
  			}
  		}
  
  		if ((csio_msi == 1) && !pci_enable_msi(hw->pdev))
  			hw->intr_mode = CSIO_IM_MSI;
  		else
  			hw->intr_mode = CSIO_IM_INTX;
  	}
  
  	csio_dbg(hw, "Using %s interrupt mode.
  ",
  		(hw->intr_mode == CSIO_IM_MSIX) ? "MSIX" :
  		((hw->intr_mode == CSIO_IM_MSI) ? "MSI" : "INTx"));
  }
  
  void
  csio_intr_disable(struct csio_hw *hw, bool free)
  {
  	csio_hw_intr_disable(hw);
  
  	switch (hw->intr_mode) {
  	case CSIO_IM_MSIX:
  		csio_disable_msix(hw, free);
  		break;
  	case CSIO_IM_MSI:
  		if (free)
  			free_irq(hw->pdev->irq, hw);
  		pci_disable_msi(hw->pdev);
  		break;
  	case CSIO_IM_INTX:
  		if (free)
  			free_irq(hw->pdev->irq, hw);
  		break;
  	default:
  		break;
  	}
  	hw->intr_mode = CSIO_IM_NONE;
  	hw->flags &= ~CSIO_HWF_HOST_INTR_ENABLED;
  }