login
Search
[PATCH 4/4 2.6.28] cxgb3i - cxgb3i iscsi driver From: Karen Xie <kxie@chelsio.com> cxgb3i iSCSI driver. Signed-off-by: Karen Xie <kxie@chelsio.com> --- drivers/scsi/Kconfig | 2 drivers/scsi/Makefile | 1 drivers/scsi/cxgb3i/Kconfig | 7 drivers/scsi/cxgb3i/Makefile | 5 drivers/scsi/cxgb3i/cxgb3i.h | 177 +++ drivers/scsi/cxgb3i/cxgb3i_init.c | 109 ++ drivers/scsi/cxgb3i/cxgb3i_iscsi.c | 800 ++++++++++++++ drivers/scsi/cxgb3i/cxgb3i_offload.c | 1929 ++++++++++++++++++++++++++++++++++ drivers/scsi/cxgb3i/cxgb3i_offload.h | 243 ++++ drivers/scsi/cxgb3i/cxgb3i_ulp2.c | 696 ++++++++++++ drivers/scsi/cxgb3i/cxgb3i_ulp2.h | 106 ++ 11 files changed, 4075 insertions(+), 0 deletions(-) create mode 100644 drivers/scsi/cxgb3i/Kconfig create mode 100644 drivers/scsi/cxgb3i/Makefile create mode 100644 drivers/scsi/cxgb3i/cxgb3i.h create mode 100644 drivers/scsi/cxgb3i/cxgb3i_init.c create mode 100644 drivers/scsi/cxgb3i/cxgb3i_iscsi.c create mode 100644 drivers/scsi/cxgb3i/cxgb3i_offload.c create mode 100644 drivers/scsi/cxgb3i/cxgb3i_offload.h create mode 100644 drivers/scsi/cxgb3i/cxgb3i_ulp2.c create mode 100644 drivers/scsi/cxgb3i/cxgb3i_ulp2.h diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index c7f0629..4c354e5 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -352,6 +352,8 @@ config ISCSI_TCP http://open-iscsi.org +source "drivers/scsi/cxgb3i/Kconfig" + config SGIWD93_SCSI tristate "SGI WD93C93 SCSI Driver" depends on SGI_HAS_WD93 && SCSI diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile index 72fd504..cf0bab2 100644 --- a/drivers/scsi/Makefile +++ b/drivers/scsi/Makefile @@ -124,6 +124,7 @@ obj-$(CONFIG_SCSI_HPTIOP) += hptiop.o obj-$(CONFIG_SCSI_STEX) += stex.o obj-$(CONFIG_SCSI_MVSAS) += mvsas.o obj-$(CONFIG_PS3_ROM) += ps3rom.o +obj-$(CONFIG_SCSI_CXGB3_ISCSI) += iscsi_tcp.o cxgb3i/ obj-$(CONFIG_ARM) += arm/ diff --git a/drivers/scsi/cxgb3i/Kconfig b/drivers/scsi/cxgb3i/Kconfig new file mode 100644 index 0000000..f2a83ac --- /dev/null +++ b/drivers/scsi/cxgb3i/Kconfig @@ -0,0 +1,7 @@ +config SCSI_CXGB3_ISCSI + tristate "Chelsio S3xx iSCSI support" + select CHELSIO_T3 + select SCSI_ISCSI_ATTRS + select ISCSI_TCP + ---help--- + This driver supports iSCSI offload for the Chelsio S3 series devices. diff --git a/drivers/scsi/cxgb3i/Makefile b/drivers/scsi/cxgb3i/Makefile new file mode 100644 index 0000000..8c8a894 --- /dev/null +++ b/drivers/scsi/cxgb3i/Makefile @@ -0,0 +1,5 @@ +EXTRA_CFLAGS += -I$(TOPDIR)/drivers/net/cxgb3 + +cxgb3i-y := cxgb3i_init.o cxgb3i_iscsi.o cxgb3i_ulp2.o cxgb3i_offload.o + +obj-$(CONFIG_SCSI_CXGB3_ISCSI) += cxgb3i.o diff --git a/drivers/scsi/cxgb3i/cxgb3i.h b/drivers/scsi/cxgb3i/cxgb3i.h new file mode 100644 index 0000000..a65995d --- /dev/null +++ b/drivers/scsi/cxgb3i/cxgb3i.h @@ -0,0 +1,177 @@ +/* + * cxgb3i.h: Chelsio S3xx iSCSI driver. + * + * Copyright (c) 2008 Chelsio Communications, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * Written by: Karen Xie (kxie@chelsio.com) + */ + +#ifndef __CXGB3I_H__ +#define __CXGB3I_H__ + +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/errno.h> +#include <linux/types.h> +#include <linux/list.h> +#include <linux/netdevice.h> +#include <linux/scatterlist.h> + +/* from cxgb3 LLD */ +#include "common.h" +#include "t3_cpl.h" +#include "t3cdev.h" +#include "cxgb3_ctl_defs.h" +#include "cxgb3_offload.h" +#include "firmware_exports.h" +#include "cxgb3i_offload.h" +/* from iscsi */ +#include "../iscsi_tcp.h" + +#define CXGB3I_SCSI_QDEPTH_DFLT 128 +#define ISCSI_PDU_HEADER_MAX (56 + 256) /* bhs + digests + ahs */ + +struct cxgb3i_adapter; +struct cxgb3i_hba; +struct cxgb3i_endpoint; + +/** + * struct cxgb3i_tag_format - cxgb3i ulp tag for steering pdu payload + * + * @idx_bits: # of bits used to store itt (from iscsi laryer) + * @age_bits: # of bits used to store age (from iscsi laryer) + * @rsvd_bits: # of bits used by h/w + * @rsvd_shift: shift left + * @rsvd_mask: bit mask + * + */ +struct cxgb3i_tag_format { + unsigned char idx_bits; + unsigned char age_bits; + unsigned char rsvd_bits; + unsigned char rsvd_shift; + u32 rsvd_mask; +}; + +/** + * struct cxgb3i_ddp_info - cxgb3i direct data placement for pdu payload + * + * @llimit: lower bound of the page pod memory + * @ulimit: upper bound of the page pod memory + * @nppods: # of page pod entries + * @idx_last: page pod entry last used + * @map_lock: lock to synchonize access to the page pod map + * @map: page pod map + */ +struct cxgb3i_ddp_info { + unsigned int llimit; + unsigned int ulimit; + unsigned int nppods; + unsigned int idx_last; + spinlock_t map_lock; + u8 *map; +}; + +/** + * struct cxgb3i_hba - cxgb3i iscsi structure (per port) + * + * @snic: cxgb3i adapter containing this port + * @ndev: pointer to netdev structure + * @shost: pointer to scsi host structure + */ +struct cxgb3i_hba { + struct cxgb3i_adapter *snic; + struct net_device *ndev; + struct Scsi_Host *shost; +}; + +/** + * struct cxgb3i_adapter - cxgb3i adapter structure (per pci) + * + * @listhead: list head to link elements + * @lock: lock for this structure + * @tdev: pointer to t3cdev used by cxgb3 driver + * @pdev: pointer to pci dev + * @hba_cnt: # of hbas (the same as # of ports) + * @hba: all the hbas on this adapter + * @tx_max_size: max. tx packet size supported + * @rx_max_size: max. rx packet size supported + * @tag_format: ulp tag format settings + * @ddp: ulp ddp state + */ +struct cxgb3i_adapter { + struct list_head list_head; + spinlock_t lock; + struct t3cdev *tdev; + struct pci_dev *pdev; + unsigned char hba_cnt; + struct cxgb3i_hba *hba[MAX_NPORTS]; + + unsigned int tx_max_size; + unsigned int rx_max_size; + + struct cxgb3i_tag_format tag_format; + struct cxgb3i_ddp_info ddp; +}; + +/** + * struct cxgb3i_conn - cxgb3i iscsi connection + * + * @tcp_conn: pointer to iscsi_tcp_conn structure + * @listhead: list head to link elements + * @conn: pointer to iscsi_conn structure + * @hba: pointer to the hba this conn. is going through + */ +struct cxgb3i_conn { + struct iscsi_tcp_conn tcp_conn; + struct list_head list_head; + struct cxgb3i_endpoint *cep; + struct iscsi_conn *conn; + struct cxgb3i_hba *hba; +}; + +/** + * struct cxgb3i_endpoint - iscsi tcp endpoint + * + * @c3cn: the h/w tcp connection representation + * @hba: pointer to the hba this conn. is going through + * @cconn: pointer to the associated cxgb3i iscsi connection + */ +struct cxgb3i_endpoint { + struct s3_conn *c3cn; + struct cxgb3i_hba *hba; + struct cxgb3i_conn *cconn; +}; + +/* + * Function Prototypes + */ +int cxgb3i_iscsi_init(void); +void cxgb3i_iscsi_cleanup(void); + +struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *); +void cxgb3i_adapter_remove(struct t3cdev *); +int cxgb3i_adapter_ulp_init(struct cxgb3i_adapter *); +void cxgb3i_adapter_ulp_cleanup(struct cxgb3i_adapter *); + +struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *); +struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *, + struct net_device *); +void cxgb3i_hba_host_remove(struct cxgb3i_hba *); + +int cxgb3i_ulp2_init(void); +void cxgb3i_ulp2_cleanup(void); +int cxgb3i_conn_ulp_setup(struct cxgb3i_conn *, int, int); +void cxgb3i_ddp_tag_release(struct cxgb3i_adapter *, u32, + struct scatterlist *, unsigned int); +u32 cxgb3i_ddp_tag_reserve(struct cxgb3i_adapter *, unsigned int, + u32, unsigned int, struct scatterlist *, + unsigned int); +int cxgb3i_conn_ulp2_xmit(struct iscsi_conn *); + +void cxgb3i_display_byte_string(char *, unsigned char *, int, int); +#endif diff --git a/drivers/scsi/cxgb3i/cxgb3i_init.c b/drivers/scsi/cxgb3i/cxgb3i_init.c new file mode 100644 index 0000000..6485e0a --- /dev/null +++ b/drivers/scsi/cxgb3i/cxgb3i_init.c @@ -0,0 +1,109 @@ +/* cxgb3i_init.c: Chelsio S3xx iSCSI driver. + * + * Copyright (c) 2008 Chelsio Communications, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * Written by: Karen Xie (kxie@chelsio.com) + */ + +#include "cxgb3i.h" + +#define DRV_MODULE_NAME "cxgb3i" +#define DRV_MODULE_VERSION "1.0.0" +#define DRV_MODULE_RELDATE "Jun. 1, 2008" + +static char version[] = + "Chelsio S3xx iSCSI Driver " DRV_MODULE_NAME + " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; + +MODULE_AUTHOR("Karen Xie <kxie@chelsio.com>"); +MODULE_DESCRIPTION("Chelsio S3xx iSCSI Driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_MODULE_VERSION); + +static void open_s3_dev(struct t3cdev *); +static void close_s3_dev(struct t3cdev *); + +static cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS]; +static struct cxgb3_client t3c_client = { + .name = "iscsi_cxgb3", + .handlers = cxgb3i_cpl_handlers, + .add = open_s3_dev, + .remove = close_s3_dev, +}; + +/** + * open_s3_dev - register with cxgb3 LLD + * @t3dev cxgb3 adapter instance + */ +static void open_s3_dev(struct t3cdev *t3dev) +{ + static int vers_printed; + + if (!vers_printed) { + printk(KERN_INFO "%s", version); + vers_printed = 1; + } + + cxgb3i_log_debug("open cxgb3 %s.\n", t3dev->name); + cxgb3i_sdev_add(t3dev, &t3c_client); + cxgb3i_adapter_add(t3dev); +} + +/** + * close_s3_dev - de-register with cxgb3 LLD + * @t3dev cxgb3 adapter instance + */ +static void close_s3_dev(struct t3cdev *t3dev) +{ + cxgb3i_log_debug("close cxgb3 %s.\n", t3dev->name); + cxgb3i_adapter_remove(t3dev); + cxgb3i_sdev_remove(t3dev); +} + +/** + * cxgb3i_init_module - module init entry point + * + * initialize any driver wide global data structures and register itself + * with the cxgb3 module + */ +static int __init cxgb3i_init_module(void) +{ + int err; + + err = cxgb3i_sdev_init(cxgb3i_cpl_handlers); + if (err < 0) + return err; + + err = cxgb3i_iscsi_init(); + if (err < 0) + return err; + + err = cxgb3i_ulp2_init(); + if (err < 0) + return err; + + cxgb3_register_client(&t3c_client); + + return 0; +} + +/** + * cxgb3i_exit_module - module cleanup/exit entry point + * + * go through the driver hba list and for each hba, release any resource held. + * and unregisters iscsi transport and the cxgb3 module + */ +static void __exit cxgb3i_exit_module(void) +{ + cxgb3_unregister_client(&t3c_client); + cxgb3i_ulp2_cleanup(); + cxgb3i_iscsi_cleanup(); + cxgb3i_sdev_cleanup(); +} + +module_init(cxgb3i_init_module); +module_exit(cxgb3i_exit_module); diff --git a/drivers/scsi/cxgb3i/cxgb3i_iscsi.c b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c new file mode 100644 index 0000000..df97e15 --- /dev/null +++ b/drivers/scsi/cxgb3i/cxgb3i_iscsi.c @@ -0,0 +1,800 @@ +/* cxgb3i_iscsi.c: Chelsio S3xx iSCSI driver. + * + * Copyright (c) 2008 Chelsio Communications, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * Written by: Karen Xie (kxie@chelsio.com) + */ + +#include <net/tcp.h> +#include <scsi/scsi_cmnd.h> +#include <scsi/scsi_device.h> +#include <scsi/scsi_eh.h> +#include <scsi/scsi_host.h> +#include <scsi/scsi.h> +#include <scsi/iscsi_proto.h> +#include <scsi/libiscsi.h> +#include <scsi/scsi_transport_iscsi.h> +#include <linux/crypto.h> + +#include "cxgb3i.h" + +static struct scsi_transport_template *cxgb3i_scsi_transport; +static struct scsi_host_template cxgb3i_host_template; +static struct iscsi_transport cxgb3i_iscsi_transport; + +static LIST_HEAD(cxgb3i_snic_list); +static DEFINE_RWLOCK(cxgb3i_snic_rwlock); + +/** + * cxgb3i_adapter_add - initialize a s3 adapter structure and any h/w settings + * necessary + * @snic: pointer to adapter instance + */ +struct cxgb3i_adapter *cxgb3i_adapter_add(struct t3cdev *t3dev) +{ + struct cxgb3i_adapter *snic; + struct adapter *adapter = tdev2adap(t3dev); + int i; + + snic = kzalloc(sizeof(*snic), GFP_KERNEL); + if (!snic) { + cxgb3i_log_debug("cxgb3 %s, OOM.\n", t3dev->name); + return NULL; + } + + spin_lock_init(&snic->lock); + snic->tdev = t3dev; + snic->pdev = adapter->pdev; + + if (cxgb3i_adapter_ulp_init(snic)) + goto free_snic; + + for_each_port(adapter, i) { + snic->hba[i] = cxgb3i_hba_host_add(snic, adapter->port[i]); + if (!snic->hba[i]) + goto ulp_cleanup; + } + snic->hba_cnt = adapter->params.nports; + + /* add to the list */ + write_lock(&cxgb3i_snic_rwlock); + list_add_tail(&snic->list_head, &cxgb3i_snic_list); + write_unlock(&cxgb3i_snic_rwlock); + + return snic; + +ulp_cleanup: + cxgb3i_adapter_ulp_cleanup(snic); +free_snic: + kfree(snic); + return NULL; +} + +/** + * cxgb3i_snic_cleanup - release all the resources held and cleanup any h/w + * settings necessary + * @snic: pointer to adapter instance + */ +void cxgb3i_adapter_remove(struct t3cdev *t3dev) +{ + int i; + struct cxgb3i_adapter *snic; + + /* remove from the list */ + read_lock(&cxgb3i_snic_rwlock); + list_for_each_entry(snic, &cxgb3i_snic_list, list_head) { + if (snic->tdev == t3dev) { + list_del(&snic->list_head); + break; + } + } + write_unlock(&cxgb3i_snic_rwlock); + + if (snic) { + for (i = 0; i < snic->hba_cnt; i++) { + if (snic->hba[i]) { + cxgb3i_hba_host_remove(snic->hba[i]); + snic->hba[i] = NULL; + } + } + + /* release ddp resources */ + cxgb3i_adapter_ulp_cleanup(snic); + kfree(snic); + } +} + +struct cxgb3i_hba *cxgb3i_hba_find_by_netdev(struct net_device *ndev) +{ + struct cxgb3i_adapter *snic; + int i; + + read_lock(&cxgb3i_snic_rwlock); + list_for_each_entry(snic, &cxgb3i_snic_list, list_head) { + for (i = 0; i < snic->hba_cnt; i++) { + if (snic->hba[i]->ndev == ndev) { + read_unlock(&cxgb3i_snic_rwlock); + return snic->hba[i]; + } + } + } + read_unlock(&cxgb3i_snic_rwlock); + return NULL; +} + +struct cxgb3i_hba *cxgb3i_hba_host_add(struct cxgb3i_adapter *snic, + struct net_device *ndev) +{ + struct cxgb3i_hba *hba; + struct Scsi_Host *shost; + int err; + + shost = iscsi_host_alloc(&cxgb3i_host_template, + sizeof(struct cxgb3i_hba), + CXGB3I_SCSI_QDEPTH_DFLT); + if (!shost) { + cxgb3i_log_info("iscsi_host_alloc failed.\n"); + return NULL; + } + + shost->transportt = cxgb3i_scsi_transport; + shost->max_lun = 512; + shost->max_id = 0; + shost->max_channel = 0; + shost->max_cmd_len = 16; + + hba = iscsi_host_priv(shost); + hba->snic = snic; + hba->ndev = ndev; + hba->shost = shost; + + pci_dev_get(snic->pdev); + err = iscsi_host_add(shost, &snic->pdev->dev); + if (err) { + cxgb3i_log_info("iscsi_host_add failed.\n"); + goto pci_dev_put; + } + + cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n", + shost, hba, shost->host_no); + + return hba; + +pci_dev_put: + pci_dev_put(snic->pdev); + scsi_host_put(shost); + return NULL; +} + +void cxgb3i_hba_host_remove(struct cxgb3i_hba *hba) +{ + cxgb3i_log_debug("shost 0x%p, hba 0x%p, no %u.\n", + hba->shost, hba, hba->shost->host_no); + iscsi_host_remove(hba->shost); + pci_dev_put(hba->snic->pdev); + /* cleanup connections ? */ + iscsi_host_free(hba->shost); +} + +/** + * cxgb3i_ep_connect - establish TCP connection to target portal + * @dst_addr: target IP address + * @non_blocking: blocking or non-blocking call + * + * Initiates a TCP/IP connection to the dst_addr + */ +static struct iscsi_endpoint *cxgb3i_ep_connect(struct sockaddr *dst_addr, + int non_blocking) +{ + struct iscsi_endpoint *ep; + struct cxgb3i_endpoint *cep; + struct cxgb3i_hba *hba; + struct s3_conn *c3cn = NULL; + int err = 0; + + c3cn = cxgb3i_c3cn_create(); + if (!c3cn) { + cxgb3i_log_info("ep connect OOM.\n"); + err = -ENOMEM; + goto release_conn; + } + + err = cxgb3i_c3cn_connect(c3cn, (struct sockaddr_in *)dst_addr); + if (err < 0) { + cxgb3i_log_info("ep connect failed.\n"); + goto release_conn; + } + hba = cxgb3i_hba_find_by_netdev(c3cn->dst_cache->dev); + if (!hba) { + err = -ENOSPC; + cxgb3i_log_info("NOT going through cxgbi device.\n"); + goto release_conn; + } + if (c3cn_in_state(c3cn, C3CN_STATE_CLOSE)) { + err = -ENOSPC; + cxgb3i_log_info("ep connect unable to connect.\n"); + goto release_conn; + } + + ep = iscsi_create_endpoint(sizeof(*cep)); + if (!ep) { + err = -ENOMEM; + cxgb3i_log_info("iscsi alloc ep, OOM.\n"); + goto release_conn; + } + cep = ep->dd_data; + cep->c3cn = c3cn; + cep->hba = hba; + + cxgb3i_log_debug("ep 0x%p, 0x%p, c3cn 0x%p, hba 0x%p.\n", + ep, cep, c3cn, hba); + return ep; + +release_conn: + cxgb3i_log_debug("conn failed release.\n"); + if (c3cn) + c3cn_release(c3cn); + return ERR_PTR(err); +} + +/** + * cxgb3i_ep_poll - polls for TCP connection establishement + * @ep: TCP connection (endpoint) handle + * @timeout_ms: timeout value in milli secs + * + * polls for TCP connect request to complete + */ +static int cxgb3i_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) +{ + struct cxgb3i_endpoint *cep = ep->dd_data; + struct s3_conn *c3cn = cep->c3cn; + + cxgb3i_log_debug("ep 0x%p, timeout %d, c3cn 0x%p, state 0x%x.\n", + ep, timeout_ms, c3cn, c3cn->state); + + if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED)) { + cxgb3i_log_info("not in established state.\n"); + return 0; + } + return 1; +} + +/** + * cxgb3i_ep_disconnect - teardown TCP connection + * @ep: TCP connection (endpoint) handle + * + * teardown TCP connection + */ +static void cxgb3i_ep_disconnect(struct iscsi_endpoint *ep) +{ + struct cxgb3i_endpoint *cep = ep->dd_data; + struct cxgb3i_conn *cconn = cep->cconn; + + cxgb3i_log_debug("ep 0x%p, cep 0x%p.\n", ep, cep); + + if (cconn && cconn->conn) { + struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn; + + write_lock_bh(&cep->c3cn->callback_lock); + cep->c3cn->user_data = NULL; + set_bit(ISCSI_SUSPEND_BIT, &cconn->conn->suspend_rx); + cconn->cep = NULL; + tcp_conn->sock = NULL; + write_unlock_bh(&cep->c3cn->callback_lock); + } + + cxgb3i_log_debug("ep 0x%p, cep 0x%p, release c3cn 0x%p.\n", + ep, cep, cep->c3cn); + c3cn_release(cep->c3cn); + iscsi_destroy_endpoint(ep); +} + +/** + * cxgb3i_session_create - create a new iscsi session + * @cmds_max: max # of commands + * @qdepth: scsi queue depth + * @initial_cmdsn: initial iscsi CMDSN for this session + * @host_no: pointer to return host no + * + * Creates a new iSCSI session + */ +static struct iscsi_cls_session *cxgb3i_session_create(struct iscsi_endpoint + *ep, u16 cmds_max, + u16 qdepth, + u32 initial_cmdsn, + u32 *host_no) +{ + struct cxgb3i_endpoint *cep; + struct cxgb3i_hba *hba; + struct Scsi_Host *shost; + struct iscsi_cls_session *cls_session; + struct iscsi_session *session; + int i; + + if (!ep) { + cxgb3i_log_error("%s, missing endpoint.\n", __func__); + return NULL; + } + + cep = ep->dd_data; + hba = cep->hba; + shost = hba->shost; + cxgb3i_log_debug("ep 0x%p, cep 0x%p, hba 0x%p.\n", ep, cep, hba); + BUG_ON(hba != iscsi_host_priv(shost)); + + *host_no = shost->host_no; + + cls_session = iscsi_session_setup(&cxgb3i_iscsi_transport, shost, + cmds_max, + sizeof(struct iscsi_tcp_task), + initial_cmdsn, ISCSI_MAX_TARGET); + if (!cls_session) + return NULL; + + session = cls_session->dd_data; + + for (i = 0; i < session->cmds_max; i++) { + struct iscsi_task *task = session->cmds[i]; + struct iscsi_tcp_task *tcp_task = task->dd_data; + + task->hdr = &tcp_task->hdr.cmd_hdr; + task->hdr_max = sizeof(tcp_task->hdr) - ISCSI_DIGEST_SIZE; + } + + if (iscsi_r2tpool_alloc(session)) + goto remove_session; + + return cls_session; + +remove_session: + iscsi_session_teardown(cls_session); + return NULL; +} + +/** + * cxgb3i_session_destroy - destroys iscsi session + * @cls_session: pointer to iscsi cls session + * + * Destroys an iSCSI session instance and releases its all resources held + */ +static void cxgb3i_session_destroy(struct iscsi_cls_session *cls_session) +{ + cxgb3i_log_debug("sess 0x%p.\n", cls_session); + iscsi_r2tpool_free(cls_session->dd_data); + iscsi_session_teardown(cls_session); +} + +/** + * cxgb3i_conn_create - create iscsi connection instance + * @cls_session: pointer to iscsi cls session + * @cid: iscsi cid + * + * Creates a new iSCSI connection instance for a given session + */ +static struct iscsi_cls_conn *cxgb3i_conn_create(struct iscsi_cls_session + *cls_session, u32 cid) +{ + struct iscsi_cls_conn *cls_conn; + struct iscsi_conn *conn; + struct cxgb3i_conn *cconn; + + cxgb3i_log_debug("sess 0x%p, cid %u.\n", cls_session, cid); + + cls_conn = iscsi_conn_setup(cls_session, sizeof(*cconn), cid); + if (!cls_conn) + return NULL; + conn = cls_conn->dd_data; + + conn->max_xmit_dlength = conn->max_recv_dlength = 16224 - 56 - 256; + + cconn = conn->dd_data; + cconn->tcp_conn.iscsi_conn = conn; + cconn->conn = conn; + + return cls_conn; +} + +/** + * cxgb3i_conn_bind - binds iscsi sess, conn and endpoint together + * @cls_session: pointer to iscsi cls session + * @cls_conn: pointer to iscsi cls conn + * @transport_eph: 64-bit EP handle + * @is_leading: leading connection on this session? + * + * Binds together an iSCSI session, an iSCSI connection and a + * TCP connection. This routine returns error code if the TCP + * connection does not belong on the device iSCSI sess/conn is bound + */ + +static int cxgb3i_conn_bind(struct iscsi_cls_session *cls_session, + struct iscsi_cls_conn *cls_conn, + u64 transport_eph, int is_leading) +{ + struct iscsi_conn *conn = cls_conn->dd_data; + struct cxgb3i_conn *cconn = conn->dd_data; + struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn; + struct iscsi_endpoint *ep; + struct cxgb3i_endpoint *cep; + struct s3_conn *c3cn; + int err; + + ep = iscsi_lookup_endpoint(transport_eph); + if (!ep) + return -EINVAL; + + cxgb3i_log_debug("ep 0x%p, cls sess 0x%p, cls conn 0x%p.\n", + ep, cls_session, cls_conn); + + err = iscsi_conn_bind(cls_session, cls_conn, is_leading); + if (err) + return -EINVAL; + + cep = ep->dd_data; + c3cn = cep->c3cn; + + read_lock(&c3cn->callback_lock); + tcp_conn->sock = (struct socket *)c3cn; + c3cn->user_data = conn; + read_unlock(&c3cn->callback_lock); + + cconn->hba = cep->hba; + cconn->cep = cep; + cep->cconn = cconn; + + conn->max_recv_dlength = cconn->hba->snic->rx_max_size - + ISCSI_PDU_HEADER_MAX; + conn->max_xmit_dlength = cconn->hba->snic->tx_max_size - + ISCSI_PDU_HEADER_MAX; + + spin_lock_bh(&conn->session->lock); + sprintf(conn->portal_address, NIPQUAD_FMT, + NIPQUAD(c3cn->daddr.sin_addr.s_addr)); + conn->portal_port = ntohs(c3cn->daddr.sin_port); + spin_unlock_bh(&conn->session->lock); + + iscsi_tcp_hdr_recv_prep(tcp_conn); + + return 0; +} + +/** + * cxgb3i_conn_flush - flush tx + * @conn: pointer to iscsi conn + */ +static int cxgb3i_conn_flush(struct iscsi_conn *conn) +{ + struct cxgb3i_conn *cconn = conn->dd_data; + struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn; + struct iscsi_segment *segment = &tcp_conn->out.segment; + + cxgb3i_log_debug("conn 0x%p, segment sent %u/%u.\n", + conn, segment->total_copied, segment->total_size); + + if (segment->total_copied < segment->total_size) + return cxgb3i_conn_ulp2_xmit(conn); + return 0; +} + +/** + * cxgb3i_conn_get_param - return iscsi connection parameter to caller + * @cls_conn: pointer to iscsi cls conn + * @param: parameter type identifier + * @buf: buffer pointer + * + * returns iSCSI connection parameters + */ +static int cxgb3i_conn_get_param(struct iscsi_cls_conn *cls_conn, + enum iscsi_param param, char *buf) +{ + struct iscsi_conn *conn = cls_conn->dd_data; + int len; + + cxgb3i_log_debug("cls_conn 0x%p, param %d.\n", cls_conn, param); + + switch (param) { + case ISCSI_PARAM_CONN_PORT: + spin_lock_bh(&conn->session->lock); + len = sprintf(buf, "%hu\n", conn->portal_port); + spin_unlock_bh(&conn->session->lock); + break; + case ISCSI_PARAM_CONN_ADDRESS: + spin_lock_bh(&conn->session->lock); + len = sprintf(buf, "%s\n", conn->portal_address); + spin_unlock_bh(&conn->session->lock); + break; + default: + return iscsi_conn_get_param(cls_conn, param, buf); + } + + return len; +} + +static int cxgb3i_conn_set_param(struct iscsi_cls_conn *cls_conn, + enum iscsi_param param, char *buf, int buflen) +{ + struct iscsi_conn *conn = cls_conn->dd_data; + struct iscsi_session *session = conn->session; + struct cxgb3i_conn *cconn = conn->dd_data; + int value, err = 0; + + switch (param) { + case ISCSI_PARAM_HDRDGST_EN: + err = iscsi_set_param(cls_conn, param, buf, buflen); + if (!err && conn->hdrdgst_en) + cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en, + conn->datadgst_en); + break; + case ISCSI_PARAM_DATADGST_EN: + err = iscsi_set_param(cls_conn, param, buf, buflen); + if (!err && conn->datadgst_en) + cxgb3i_conn_ulp_setup(cconn, conn->hdrdgst_en, + conn->datadgst_en); + break; + case ISCSI_PARAM_MAX_R2T: + sscanf(buf, "%d", &value); + if (value <= 0 || !is_power_of_2(value)) + return -EINVAL; + if (session->max_r2t == value) + break; + iscsi_r2tpool_free(session); + err = iscsi_set_param(cls_conn, param, buf, buflen); + if (!err && iscsi_r2tpool_alloc(session)) + return -ENOMEM; + case ISCSI_PARAM_MAX_RECV_DLENGTH: + err = iscsi_set_param(cls_conn, param, buf, buflen); + cxgb3i_log_debug("MAX_RECV %u.\n", conn->max_recv_dlength); + break; + case ISCSI_PARAM_MAX_XMIT_DLENGTH: + err = iscsi_set_param(cls_conn, param, buf, buflen); + cxgb3i_log_debug("MAX_XMIT %u.\n", conn->max_xmit_dlength); + break; + default: + return iscsi_set_param(cls_conn, param, buf, buflen); + } + return err; +} + +/** + * cxgb3i_host_get_param - returns host (adapter) related parameters + * @shost: scsi host pointer + * @param: parameter type identifier + * @buf: buffer pointer + */ +static int cxgb3i_host_get_param(struct Scsi_Host *shost, + enum iscsi_host_param param, char *buf) +{ + struct cxgb3i_hba *hba = iscsi_host_priv(shost); + int i; + int len = 0; + + switch (param) { + case ISCSI_HOST_PARAM_HWADDRESS: + for (i = 0; i < 6; i++) + len += + sprintf(buf + len, "%02x.", + hba->ndev->dev_addr[i]); + len--; + buf[len] = '\0'; + break; + case ISCSI_HOST_PARAM_NETDEV_NAME: + len = sprintf(buf, "%s\n", hba->ndev->name); + break; + default: + return iscsi_host_get_param(shost, param, buf); + } + return len; +} + +/** + * cxgb3i_conn_get_stats - returns iSCSI stats + * @cls_conn: pointer to iscsi cls conn + * @stats: pointer to iscsi statistic struct + */ +static void cxgb3i_conn_get_stats(struct iscsi_cls_conn *cls_conn, + struct iscsi_stats *stats) +{ + struct iscsi_conn *conn = cls_conn->dd_data; + + stats->txdata_octets = conn->txdata_octets; + stats->rxdata_octets = conn->rxdata_octets; + stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; + stats->dataout_pdus = conn->dataout_pdus_cnt; + stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; + stats->datain_pdus = conn->datain_pdus_cnt; + stats->r2t_pdus = conn->r2t_pdus_cnt; + stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; + stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; + stats->digest_err = 0; + stats->timeout_err = 0; + stats->custom_length = 1; + strcpy(stats->custom[0].desc, "eh_abort_cnt"); + stats->custom[0].value = conn->eh_abort_cnt; +} + +static inline u32 tag_base(struct cxgb3i_tag_format *format, + unsigned int idx, unsigned int age) +{ + u32 sw_bits = idx | (age << format->idx_bits); + u32 tag = sw_bits >> format->rsvd_shift; + + tag <<= format->rsvd_bits + format->rsvd_shift; + tag |= sw_bits & ((1 << format->rsvd_shift) - 1); + return tag; +} + +static inline void cxgb3i_parse_tag(struct cxgb3i_tag_format *format, + u32 tag, u32 *rsvd_bits, u32 *sw_bits) +{ + if (rsvd_bits) + *rsvd_bits = (tag >> format->rsvd_shift) & format->rsvd_mask; + if (sw_bits) { + *sw_bits = (tag >> (format->rsvd_shift + format->rsvd_bits)) + << format->rsvd_shift; + *sw_bits |= tag & ((1 << format->rsvd_shift) - 1); + } +} + + +static void cxgb3i_parse_itt(struct iscsi_conn *conn, itt_t itt, + int *idx, int *age) +{ + struct cxgb3i_conn *cconn = conn->dd_data; + struct cxgb3i_adapter *snic = cconn->hba->snic; + u32 sw_bits; + + cxgb3i_parse_tag(&snic->tag_format, itt, NULL, &sw_bits); + if (idx) + *idx = sw_bits & ISCSI_ITT_MASK; + if (age) + *age = (sw_bits >> snic->tag_format.idx_bits) & ISCSI_AGE_MASK; +} + +static int cxgb3i_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt) +{ + struct scsi_cmnd *sc = task->sc; + struct iscsi_conn *conn = task->conn; + struct iscsi_session *sess = conn->session; + struct cxgb3i_conn *cconn = conn->dd_data; + struct iscsi_tcp_conn *tcp_conn = &cconn->tcp_conn; + struct cxgb3i_adapter *snic = cconn->hba->snic; + u32 sw_tag = tag_base(&snic->tag_format, task->itt, sess->age); + u32 tag = RESERVED_ITT; + + if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE)) { + struct s3_conn *c3cn = (struct s3_conn *)(tcp_conn->sock); + tag = + cxgb3i_ddp_tag_reserve(snic, c3cn->tid, sw_tag, + scsi_out(sc)->length, + scsi_out(sc)->table.sgl, + scsi_out(sc)->table.nents); + } + if (tag == RESERVED_ITT) + tag = sw_tag | (snic->tag_format.rsvd_mask << + snic->tag_format.rsvd_shift); + *hdr_itt = htonl(tag); + return 0; +} + +static void cxgb3i_release_itt(struct iscsi_task *task, itt_t hdr_itt) +{ + struct scsi_cmnd *sc = task->sc; + struct iscsi_conn *conn = task->conn; + struct cxgb3i_conn *cconn = conn->dd_data; + struct cxgb3i_adapter *snic = cconn->hba->snic; + + hdr_itt = ntohl(hdr_itt); + if (sc && (sc->sc_data_direction == DMA_FROM_DEVICE)) + cxgb3i_ddp_tag_release(snic, hdr_itt, + scsi_out(sc)->table.sgl, + scsi_out(sc)->table.nents); +} + +/** + * cxgb3i_host_template -- Scsi_Host_Template structure + * used when registering with the scsi mid layer + */ +static struct scsi_host_template cxgb3i_host_template = { + .module = THIS_MODULE, + .name = "Chelsio S3xx iSCSI Initiator", + .proc_name = "cxgb3i", + .queuecommand = iscsi_queuecommand, + .change_queue_depth = iscsi_change_queue_depth, + .can_queue = 128 * (ISCSI_DEF_XMIT_CMDS_MAX - 1), + .sg_tablesize = SG_ALL, + .max_sectors = 0xFFFF, + .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, + .eh_abort_handler = iscsi_eh_abort, + .eh_device_reset_handler = iscsi_eh_device_reset, + .eh_host_reset_handler = iscsi_eh_host_reset, + .use_clustering = DISABLE_CLUSTERING, + .this_id = -1, +}; + +static struct iscsi_transport cxgb3i_iscsi_transport = { + .owner = THIS_MODULE, + .name = "cxgb3i", + .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST + | CAP_DATADGST | CAP_DIGEST_OFFLOAD, + .param_mask = ISCSI_MAX_RECV_DLENGTH | + ISCSI_MAX_XMIT_DLENGTH | + ISCSI_HDRDGST_EN | + ISCSI_DATADGST_EN | + ISCSI_INITIAL_R2T_EN | + ISCSI_MAX_R2T | + ISCSI_IMM_DATA_EN | + ISCSI_FIRST_BURST | + ISCSI_MAX_BURST | + ISCSI_PDU_INORDER_EN | + ISCSI_DATASEQ_INORDER_EN | + ISCSI_ERL | + ISCSI_CONN_PORT | + ISCSI_CONN_ADDRESS | + ISCSI_EXP_STATSN | + ISCSI_PERSISTENT_PORT | + ISCSI_PERSISTENT_ADDRESS | + ISCSI_TARGET_NAME | ISCSI_TPGT | + ISCSI_USERNAME | ISCSI_PASSWORD | + ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN | + ISCSI_FAST_ABORT | ISCSI_ABORT_TMO | + ISCSI_LU_RESET_TMO | + ISCSI_PING_TMO | ISCSI_RECV_TMO | + ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME, + .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS | + ISCSI_HOST_INITIATOR_NAME | ISCSI_HOST_NETDEV_NAME, + .get_host_param = cxgb3i_host_get_param, + /* session management */ + .create_session = cxgb3i_session_create, + .destroy_session = cxgb3i_session_destroy, + .get_session_param = iscsi_session_get_param, + /* connection management */ + .create_conn = cxgb3i_conn_create, + .bind_conn = cxgb3i_conn_bind, + .destroy_conn = iscsi_conn_teardown, + .start_conn = iscsi_conn_start, + .stop_conn = iscsi_conn_stop, + .flush_conn = cxgb3i_conn_flush, + .get_conn_param = cxgb3i_conn_get_param, + .set_param = cxgb3i_conn_set_param, + .get_stats = cxgb3i_conn_get_stats, + /* pdu xmit req. from user space */ + .send_pdu = iscsi_conn_send_pdu, + /* task */ + .init_task = iscsi_tcp_task_init, + .xmit_task = iscsi_tcp_task_xmit, + .cleanup_task = iscsi_tcp_cleanup_task, + .parse_itt = cxgb3i_parse_itt, + .reserve_itt = cxgb3i_reserve_itt, + .release_itt = cxgb3i_release_itt, + /* TCP connect/disconnect */ + .ep_connect = cxgb3i_ep_connect, + .ep_poll = cxgb3i_ep_poll, + .ep_disconnect = cxgb3i_ep_disconnect, + /* Error recovery timeout call */ + .session_recovery_timedout = iscsi_session_recovery_timedout, +}; + +int cxgb3i_iscsi_init(void) +{ + cxgb3i_scsi_transport = + iscsi_register_transport(&cxgb3i_iscsi_transport); + if (!cxgb3i_scsi_transport) { + cxgb3i_log_error("Could not register cxgb3i transport.\n"); + return -ENODEV; + } + cxgb3i_log_debug("cxgb3i transport 0x%p.\n", cxgb3i_scsi_transport); + return 0; +} + +void cxgb3i_iscsi_cleanup(void) +{ + if (cxgb3i_scsi_transport) { + cxgb3i_log_debug("cxgb3i transport 0x%p.\n", + cxgb3i_scsi_transport); + iscsi_unregister_transport(&cxgb3i_iscsi_transport); + cxgb3i_scsi_transport = NULL; + } +} diff --git a/drivers/scsi/cxgb3i/cxgb3i_offload.c b/drivers/scsi/cxgb3i/cxgb3i_offload.c new file mode 100644 index 0000000..bb968ae --- /dev/null +++ b/drivers/scsi/cxgb3i/cxgb3i_offload.c @@ -0,0 +1,1929 @@ +/* + * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved. + * + * Written by Dimitris Michailidis (dm@chelsio.com) + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this + * release for licensing terms and conditions. + */ + +#include <linux/if_vlan.h> +#include <linux/version.h> + +#include "cxgb3_defs.h" +#include "cxgb3_ctl_defs.h" +#include "firmware_exports.h" +#include "cxgb3i_offload.h" +#include "cxgb3i_ulp2.h" + +static int rcv_win = 256 * 1024; +module_param(rcv_win, int, 0644); +MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)"); + +static int snd_win = 64 * 1024; +module_param(snd_win, int, 0644); +MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=64KB)"); + +static int rx_credit_thres = 10 * 1024; +module_param(rx_credit_thres, int, 0644); +MODULE_PARM_DESC(rx_credit_thres, + "RX credits return threshold in bytes (default=10KB)"); + +static unsigned int max_connect = 8 * 1024; +module_param(max_connect, uint, 0644); +MODULE_PARM_DESC(max_connect, "Max. # of connections (default=8092)"); + +static unsigned int sport_base = 20000; +module_param(sport_base, uint, 0644); +MODULE_PARM_DESC(sport_start, "starting port number (default=20000)"); + +#ifdef __DEBUG_C3CN_CONN__ +#define c3cn_conn_debug cxgb3i_log_debug +#else +#define c3cn_conn_debug(fmt...) +#endif + +#ifdef __DEBUG_C3CN_TX__ +#define c3cn_tx_debug cxgb3i_log_debug +#else +#define c3cn_tx_debug(fmt...) +#endif + +/* minimal port allocation management scheme */ +static spinlock_t sport_map_lock; +static unsigned int sport_map_next; +static unsigned long *sport_map; + +/* + * Find a free source port in our allocation map. We use a very simple rotor + * scheme to look for the next free port. + * + * If a source port has been specified make sure that it doesn't collide with + * our normal source port allocation map. If it's outside the range of our + * allocation scheme just let them use it. + */ +static int c3cn_get_port(struct s3_conn *c3cn) +{ + unsigned int start; + + if (!sport_map) + goto error_out; + + if (c3cn->saddr.sin_port != 0) { + int sport = ntohs(c3cn->saddr.sin_port) - sport_base; + int err = 0; + + if (sport < 0 || sport >= max_connect) + return 0; + spin_lock(&sport_map_lock); + err = __test_and_set_bit(sport, sport_map); + spin_unlock(&sport_map_lock); + return err ? -EADDRINUSE : 0; + } + + spin_lock(&sport_map_lock); + start = sport_map_next; + do { + unsigned int new = sport_map_next; + if (++sport_map_next >= max_connect) + sport_map_next = 0; + if (!(__test_and_set_bit(new, sport_map))) { + spin_unlock(&sport_map_lock); + c3cn->saddr.sin_port = htons(sport_base + new); + return 0; + } + } while (sport_map_next != start); + spin_unlock(&sport_map_lock); + +error_out: + return -EADDRNOTAVAIL; +} + +/* + * Deallocate a source port from the allocation map. If the source port is + * outside our allocation range just return -- the caller is responsible for + * keeping track of their port usage outside of our allocation map. + */ +static void c3cn_put_port(struct s3_conn *c3cn) +{ + int old = ntohs(c3cn->saddr.sin_port) - sport_base; + c3cn->saddr.sin_port = 0; + + if (old < 0 || old >= max_connect) + return; + + spin_lock(&sport_map_lock); + __clear_bit(old, sport_map); + spin_unlock(&sport_map_lock); +} + +static void c3cn_set_state(struct s3_conn *c3cn, int state) +{ + c3cn_conn_debug("c3cn 0x%p state -> 0x%x.\n", c3cn, state); + if (state == C3CN_STATE_CLOSE) + c3cn_put_port(c3cn); + c3cn->state = state; +} + +static void c3cn_reset_timer(struct s3_conn *c3cn, struct timer_list *timer, + unsigned long expires) +{ + if (!mod_timer(timer, expires)) + c3cn_hold(c3cn); +} + +typedef int (cxgb3_cpl_handler_decl) (struct t3cdev *, + struct sk_buff *, void *); + +static cxgb3_cpl_handler_decl do_act_establish; +static cxgb3_cpl_handler_decl do_act_open_rpl; +static cxgb3_cpl_handler_decl do_wr_ack; +static cxgb3_cpl_handler_decl do_peer_close; +static cxgb3_cpl_handler_decl do_abort_req; +static cxgb3_cpl_handler_decl do_abort_rpl; +static cxgb3_cpl_handler_decl do_close_con_rpl; +static cxgb3_cpl_handler_decl do_iscsi_hdr; + +static LIST_HEAD(cxgb3_list); +static DEFINE_MUTEX(cxgb3_list_lock); + +/* + * For ULP connections HW may inserts digest bytes into the pdu. This array + * contains the compensating extra lengths for ULP packets. It is indexed by + * a packet's ULP submode. + */ +static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 }; + +/* + * Return the length of any HW additions that will be made to a Tx packet. + * Such additions can happen for some types of ULP packets. + */ +static inline unsigned int ulp_extra_len(const struct sk_buff *skb) +{ + return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3]; +} + +/* + * Size of WRs in bytes. Note that we assume all devices we are handling have + * the same WR size. + */ +static unsigned int wrlen __read_mostly; + +/* + * The number of WRs needed for an skb depends on the number of page fragments + * in the skb and whether it has any payload in its main body. This maps the + * length of the gather list represented by an skb into the # of necessary WRs. + */ +static unsigned int skb_wrs[MAX_SKB_FRAGS + 2] __read_mostly; + +static void s3_init_wr_tab(unsigned int wr_len) +{ + int i; + + if (skb_wrs[1]) /* already initialized */ + return; + + for (i = 1; i < ARRAY_SIZE(skb_wrs); i++) { + int sgl_len = (3 * i) / 2 + (i & 1); + + sgl_len += 3; + skb_wrs[i] = (sgl_len <= wr_len + ? 1 : 1 + (sgl_len - 2) / (wr_len - 1)); + } + + wrlen = wr_len * 8; +} + +/* + * Initialization/cleanup cxgb3 API operations. + */ +/* + * large memory chunk allocation/release + */ +void *cxgb3i_alloc_big_mem(unsigned int size) +{ + void *p = kmalloc(size, GFP_KERNEL); + if (!p) + p = vmalloc(size); + if (p) + memset(p, 0, size); + return p; +} + +void cxgb3i_free_big_mem(void *addr) +{ + if (is_vmalloc_addr(addr)) + vfree(addr); + else + kfree(addr); +} + +void cxgb3i_sdev_cleanup(void) +{ + if (sport_map) + cxgb3i_free_big_mem(sport_map); +} + +int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers) +{ + cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish; + cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl; + cpl_handlers[CPL_PEER_CLOSE] = do_peer_close; + cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req; + cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl; + cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl; + cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack; + cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr; + + sport_map = cxgb3i_alloc_big_mem(DIV_ROUND_UP(max_connect, + 8 * + sizeof(unsigned long))); + if (!sport_map) + return -ENOMEM; + return 0; +} + +void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client) +{ + struct cxgb3i_sdev_data *cdata; + struct adap_ports *ports; + struct ofld_page_info rx_page_info; + unsigned int wr_len; + int i; + + cdata = kzalloc(sizeof *cdata, GFP_KERNEL); + if (!cdata) + return; + ports = kzalloc(sizeof *ports, GFP_KERNEL); + if (!ports) + goto free_ports; + cdata->ports = ports; + + if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 || + cdev->ctl(cdev, GET_PORTS, cdata->ports) < 0 || + cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0) + goto free_ports; + + s3_init_wr_tab(wr_len); + + INIT_LIST_HEAD(&cdata->list); + cdata->cdev = cdev; + cdata->client = client; + cdata->rx_page_size = rx_page_info.page_size; + skb_queue_head_init(&cdata->deferq); + + for (i = 0; i < ports->nports; i++) + NDEV2CDATA(ports->lldevs[i]) = cdata; + + mutex_lock(&cxgb3_list_lock); + list_add_tail(&cdata->list, &cxgb3_list); + mutex_unlock(&cxgb3_list_lock); + + return; + +free_ports: + kfree(ports); + kfree(cdata); +} + +void cxgb3i_sdev_remove(struct t3cdev *cdev) +{ + struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev); + struct adap_ports *ports = cdata->ports; + int i; + + for (i = 0; i < ports->nports; i++) + NDEV2CDATA(ports->lldevs[i]) = NULL; + + mutex_lock(&cxgb3_list_lock); + list_del(&cdata->list); + mutex_unlock(&cxgb3_list_lock); + + kfree(ports); + kfree(cdata); +} + +/* + * Return TRUE if the specified net device is for a port on one of our + * registered adapters. + */ +static int is_cxgb3_dev(struct net_device *dev) +{ + struct cxgb3i_sdev_data *cdata; + + mutex_lock(&cxgb3_list_lock); + list_for_each_entry(cdata, &cxgb3_list, list) { + struct adap_ports *ports = cdata->ports; + int i; + + for (i = 0; i < ports->nports; i++) + if (dev == ports->lldevs[i]) { + mutex_unlock(&cxgb3_list_lock); + return 1; + } + } + mutex_unlock(&cxgb3_list_lock); + return 0; +} + +/* + * Primary cxgb3 API operations. + * ============================= + */ + +static int s3_push_frames(struct s3_conn *, int); +static int s3_send_reset(struct s3_conn *, int, struct sk_buff *); + +struct s3_conn *cxgb3i_c3cn_create(void) +{ + struct s3_conn *c3cn; + + c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL); + if (c3cn == NULL) + return NULL; + + c3cn->flags = 0; + spin_lock_init(&c3cn->lock); + atomic_set(&c3cn->refcnt, 1); + skb_queue_head_init(&c3cn->receive_queue); + skb_queue_head_init(&c3cn->write_queue); + setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn); + rwlock_init(&c3cn->callback_lock); + + return c3cn; +} + +static void mk_close_req(struct s3_conn *); +static inline void s3_purge_write_queue(struct s3_conn *c3cn) +{ + struct sk_buff *skb; + + while ((skb = __skb_dequeue(&c3cn->write_queue))) + __kfree_skb(skb); +} + +/* + * Release a connection's local port if the connection is bound. + */ +static inline void release_port(struct s3_conn *c3cn) +{ + c3cn_conn_debug("c3cn 0x%p, port %u.\n", c3cn, c3cn->saddr.sin_port); + if (c3cn->saddr.sin_port) + c3cn_put_port(c3cn); +} + +static void c3cn_done(struct s3_conn *c3cn) +{ + c3cn_conn_debug("c3cn 0x%p.\n", c3cn); + + c3cn_set_state(c3cn, C3CN_STATE_CLOSE); + c3cn->shutdown = C3CN_SHUTDOWN_MASK; + + cxgb3i_conn_closing(c3cn); +} + +void c3cn_close(struct s3_conn *c3cn) +{ + int data_lost, old_state; + + c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n", + c3cn, c3cn->state, c3cn->flags); + + dst_confirm(c3cn->dst_cache); + + spin_lock_bh(&c3cn->lock); + c3cn->shutdown |= C3CN_SHUTDOWN_MASK; + + /* + * We need to flush the receive buffs. We do this only on the + * descriptor close, not protocol-sourced closes, because the + * reader process may not have drained the data yet! Make a note + * of whether any received data will be lost so we can decide whether + * to FIN or RST. + */ + data_lost = skb_queue_len(&c3cn->receive_queue); + __skb_queue_purge(&c3cn->receive_queue); + + if (c3cn->state == C3CN_STATE_CLOSE) + /* Nothing if we are already closed */ + ; + else if (data_lost || c3cn->state == C3CN_STATE_SYN_SENT) { + /* Unread data was tossed, zap the connection. */ + s3_send_reset(c3cn, CPL_ABORT_SEND_RST, NULL); + release_port(c3cn); + goto unlock; + } else if (c3cn->state == C3CN_STATE_ESTABLISHED) { + c3cn_set_state(c3cn, C3CN_STATE_CLOSING); + mk_close_req(c3cn); + } + +unlock: + old_state = c3cn->state; + c3cn_hold(c3cn); /* must last past the potential destroy() */ + + spin_unlock_bh(&c3cn->lock); + + /* + * There are no more user references at this point. Grab the + * connection lock and finish the close. + */ + local_bh_disable(); + spin_lock(&c3cn->lock); + + /* + * Because the connection was orphaned before the spin_lock() + * either the backlog or a BH may have already destroyed it. + * Bail out if so. + */ + if (old_state != C3CN_STATE_CLOSE && c3cn->state == C3CN_STATE_CLOSE) + goto out; + + if (c3cn->state == C3CN_STATE_CLOSE) + s3_purge_write_queue(c3cn); + +out: + spin_unlock(&c3cn->lock); + local_bh_enable(); + c3cn_put(c3cn); +} + +/* + * Local utility routines used to implement primary cxgb3 API operations. + * ====================================================================== + */ + +static int s3_connect(struct s3_conn *); +static u32 s3_send_rx_credits(struct s3_conn *, u32, u32, int); +static void mk_act_open_req(struct s3_conn *, struct sk_buff *, + unsigned int, const struct l2t_entry *); +static void skb_entail(struct s3_conn *, struct sk_buff *, int); + +static inline void reset_wr_list(struct s3_conn *c3cn) +{ + c3cn->wr_pending_head = NULL; +} + +/* + * Add a WR to a connections's list of pending WRs. This is a singly-linked + * list of sk_buffs operating as a FIFO. The head is kept in wr_pending_head + * and the tail in wr_pending_tail. + */ +static inline void enqueue_wr(struct s3_conn *c3cn, + struct sk_buff *skb) +{ + skb->sp = NULL; + + /* + * We want to take an extra reference since both us and the driver + * need to free the packet before it's really freed. We know there's + * just one user currently so we use atomic_set rather than skb_get + * to avoid the atomic op. + */ + atomic_set(&skb->users, 2); + + if (!c3cn->wr_pending_head) + c3cn->wr_pending_head = skb; + else + c3cn->wr_pending_tail->sp = (void *)skb; + c3cn->wr_pending_tail = skb; +} + +/* + * The next two functions calculate the option 0 value for a connection. + */ +static inline int compute_wscale(int win) +{ + int wscale = 0; + while (wscale < 14 && (65535<<wscale) < win) + wscale++; + return wscale; +} + +static inline unsigned int calc_opt0h(struct s3_conn *c3cn) +{ + int wscale = compute_wscale(rcv_win); + return V_KEEP_ALIVE(1) | + F_TCAM_BYPASS | + V_WND_SCALE(wscale) | + V_MSS_IDX(c3cn->mss_idx); +} + +static inline unsigned int calc_opt0l(struct s3_conn *c3cn) +{ + return V_ULP_MODE(ULP_MODE_ISCSI) | + V_RCV_BUFSIZ(rcv_win>>10); +} + +static inline void make_tx_data_wr(struct s3_conn *c3cn, + struct sk_buff *skb, int len) +{ + struct tx_data_wr *req; + + skb_reset_transport_header(skb); + req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req)); + req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)); + req->wr_lo = htonl(V_WR_TID(c3cn->tid)); + req->sndseq = htonl(c3cn->snd_nxt); + /* len includes the length of any HW ULP additions */ + req->len = htonl(len); + req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx)); + /* V_TX_ULP_SUBMODE sets both the mode and submode */ + req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) | + V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1))); + + if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) { + + req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT | + V_TX_CPU_IDX(c3cn->qset)); + + /* Sendbuffer is in units of 32KB. + */ + req->param |= htonl(V_TX_SNDBUF(snd_win >> 15)); + c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT); + } +} + +static struct rtable *find_route(__be32 saddr, __be32 daddr, + __be16 sport, __be16 dport) +{ + struct rtable *rt; + struct flowi fl = { + .oif = 0, + .nl_u = { + .ip4_u = { + .daddr = daddr, + .saddr = saddr, + .tos = 0 } }, + .proto = IPPROTO_TCP, + .uli_u = { + .ports = { + .sport = sport, + .dport = dport } } }; + + if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0)) + return NULL; + return rt; +} + +int cxgb3i_c3cn_connect(struct s3_conn *c3cn, struct sockaddr_in *usin) +{ + struct rtable *rt; + int err; + + if (usin->sin_family != AF_INET) + return -EAFNOSUPPORT; + + /* get a source port if one hasn't been provided */ + err = c3cn_get_port(c3cn); + if (err) + return err; + + c3cn_conn_debug("c3cn 0x%p get port %u.\n", + c3cn, ntohs(c3cn->saddr.sin_port)); + + c3cn->daddr.sin_port = usin->sin_port; + c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr; + + rt = find_route(c3cn->saddr.sin_addr.s_addr, + c3cn->daddr.sin_addr.s_addr, + c3cn->saddr.sin_port, + c3cn->daddr.sin_port); + if (rt == NULL) { + c3cn_conn_debug("NO route to 0x%x, port %u.\n", + c3cn->daddr.sin_addr.s_addr, + ntohs(c3cn->daddr.sin_port)); + return -ENETUNREACH; + } + + if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) { + c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n", + c3cn->daddr.sin_addr.s_addr, + ntohs(c3cn->daddr.sin_port)); + ip_rt_put(rt); + return -ENETUNREACH; + } + + if (!c3cn->saddr.sin_addr.s_addr) + c3cn->saddr.sin_addr.s_addr = rt->rt_src; + + c3cn_conn_debug("c3cn 0x%p -> SYN_SENT.\n", c3cn); + c3cn_set_state(c3cn, C3CN_STATE_SYN_SENT); + + /* now commit destination to connection */ + c3cn->dst_cache = &rt->u.dst; + + if (s3_connect(c3cn)) + return 0; + /* + * If we get here, we don't have an offload connection so simply + * return a failure. + */ + err = -ENOTSUPP; + + /* + * This trashes the connection and releases the local port, + * if necessary. + */ + c3cn_conn_debug("c3cn 0x%p -> CLOSE.\n", c3cn); + c3cn_set_state(c3cn, C3CN_STATE_CLOSE); + ip_rt_put(rt); + c3cn_put_port(c3cn); + c3cn->daddr.sin_port = 0; + return err; +} + +/* + * Set of states for which we should return RX credits. + */ +#define CREDIT_RETURN_STATE (C3CN_STATE_ESTABLISHED) + +/* + * Called after some received data has been read. It returns RX credits + * to the HW for the amount of data processed. + */ +void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied) +{ + struct t3cdev *cdev; + int must_send; + u32 credits, dack = 0; + + if (!c3cn_in_state(c3cn, CREDIT_RETURN_STATE)) + return; + + credits = c3cn->copied_seq - c3cn->rcv_wup; + if (unlikely(!credits)) + return; + + cdev = c3cn->cdev; + + if (unlikely(rx_credit_thres == 0)) + return; + + dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); + + /* + * For coalescing to work effectively ensure the receive window has + * at least 16KB left. + */ + must_send = credits + 16384 >= rcv_win; + + if (must_send || credits >= rx_credit_thres) + c3cn->rcv_wup += s3_send_rx_credits(c3cn, credits, dack, + must_send); +} + +/* + * Generic ARP failure handler that discards the buffer. + */ +static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb) +{ + kfree_skb(skb); +} + +/* + * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a + * connection's send queue and sends them on to T3. Must be called with the + * connection's lock held. Returns the amount of send buffer space that was + * freed as a result of sending queued data to T3. + */ +static int s3_push_frames(struct s3_conn *c3cn, int req_completion) +{ + int total_size = 0; + struct sk_buff *skb; + struct t3cdev *cdev; + struct cxgb3i_sdev_data *cdata; + + if (unlikely(c3cn_in_state(c3cn, + C3CN_STATE_SYN_SENT | C3CN_STATE_CLOSE))) + return 0; + + /* + * We shouldn't really be called at all after an abort but check just + * in case. + */ + if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN))) + return 0; + + cdev = c3cn->cdev; + cdata = CXGB3_SDEV_DATA(cdev); + + while (c3cn->wr_avail + && (skb = skb_peek(&c3cn->write_queue)) != NULL + && !c3cn_flag(c3cn, C3CN_TX_WAIT_IDLE)) { + + int len = skb->len; /* length before skb_push */ + int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len); + int wrs_needed = skb_wrs[frags]; + + if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen) + wrs_needed = 1; + + WARN_ON(frags >= ARRAY_SIZE(skb_wrs) || wrs_needed < 1); + if (c3cn->wr_avail < wrs_needed) + break; + + __skb_unlink(skb, &c3cn->write_queue); + skb->priority = CPL_PRIORITY_DATA; + skb->csum = wrs_needed; /* remember this until the WR_ACK */ + c3cn->wr_avail -= wrs_needed; + c3cn->wr_unacked += wrs_needed; + enqueue_wr(c3cn, skb); + + if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) { + len += ulp_extra_len(skb); + make_tx_data_wr(c3cn, skb, len); + c3cn->snd_nxt += len; + if ((req_completion + && c3cn->wr_unacked == wrs_needed) + || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL) + || c3cn->wr_unacked >= c3cn->wr_max / 2) { + struct work_request_hdr *wr = cplhdr(skb); + + wr->wr_hi |= htonl(F_WR_COMPL); + c3cn->wr_unacked = 0; + } + CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR; + } else if (skb->data[0] == FW_WROPCODE_OFLD_CLOSE_CON) + c3cn_set_flag(c3cn, C3CN_CLOSE_CON_REQUESTED); + + total_size += skb->truesize; + set_arp_failure_handler(skb, arp_failure_discard); + l2t_send(cdev, skb, c3cn->l2t); + } + return total_size; +} + +/* + * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant + * and send it along. + */ +static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb) +{ + struct cpl_abort_req *req = cplhdr(skb); + + req->cmd = CPL_ABORT_NO_RST; + cxgb3_ofld_send(cdev, skb); +} + +/* + * Send an ABORT_REQ message. Cannot fail. This routine makes sure we do + * not send multiple ABORT_REQs for the same connection and also that we do + * not try to send a message after the connection has closed. Returns 1 if + * an ABORT_REQ wasn't generated after all, 0 otherwise. + */ +static int s3_send_reset(struct s3_conn *c3cn, int mode, + struct sk_buff *skb) +{ + struct cpl_abort_req *req; + unsigned int tid = c3cn->tid; + + if (unlikely(c3cn_flag(c3cn, C3CN_ABORT_SHUTDOWN) || !c3cn->cdev)) { + if (skb) + __kfree_skb(skb); + return 1; + } + + c3cn_conn_debug("c3cn 0x%p, mode %d.\n", c3cn, mode); + + c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING); + c3cn_set_flag(c3cn, C3CN_ABORT_SHUTDOWN); + + /* Purge the send queue so we don't send anything after an abort. */ + s3_purge_write_queue(c3cn); + + if (!skb) + skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL); + skb->priority = CPL_PRIORITY_DATA; + set_arp_failure_handler(skb, abort_arp_failure); + + req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); + req->wr.wr_lo = htonl(V_WR_TID(tid)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); + req->rsvd0 = htonl(c3cn->snd_nxt); + req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT); + req->cmd = mode; + + l2t_send(c3cn->cdev, skb, c3cn->l2t); + return 0; +} + +/* + * Add a list of skbs to a connection send queue. This interface is intended + * for use by in-kernel ULPs. The skbs must comply with the max size limit of + * the device and have a headroom of at least TX_HEADER_LEN bytes. + */ +int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb, int flags) +{ + struct sk_buff *next; + int err, copied = 0; + + spin_lock_bh(&c3cn->lock); + + if (!c3cn_in_state(c3cn, C3CN_STATE_ESTABLISHED)) { + err = -EAGAIN; + goto out_err; + } + + err = -EPIPE; + if (c3cn->err || (c3cn->shutdown & C3CN_SEND_SHUTDOWN)) + goto out_err; + + while (skb) { + if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) { + c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn); + err = -EINVAL; + goto out_err; + } + + next = skb->next; + skb->next = NULL; + skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR); + copied += skb->len; + c3cn->write_seq += skb->len + ulp_extra_len(skb); + skb = next; + } +done: + if (likely(skb_queue_len(&c3cn->write_queue))) + s3_push_frames(c3cn, 1); + spin_unlock_bh(&c3cn->lock); + return copied; + +out_err: + if (copied == 0 && err == -EPIPE) + copied = c3cn->err ? c3cn->err : -EPIPE; + goto done; +} + +/* + * Low-level utility routines for primary API functions. + * ===================================================== + */ +/* routines to implement CPL message processing */ +static void c3cn_act_establish(struct s3_conn *, struct sk_buff *); +static void active_open_failed(struct s3_conn *, struct sk_buff *); +static void wr_ack(struct s3_conn *, struct sk_buff *); +static void do_peer_fin(struct s3_conn *, struct sk_buff *); +static void process_abort_req(struct s3_conn *, struct sk_buff *); +static void process_abort_rpl(struct s3_conn *, struct sk_buff *); +static void process_close_con_rpl(struct s3_conn *, struct sk_buff *); +static void process_rx_iscsi_hdr(struct s3_conn *, struct sk_buff *); + +static struct sk_buff *__get_cpl_reply_skb(struct sk_buff *, size_t, gfp_t); + +static int act_open(struct s3_conn *, struct net_device *); +static void fail_act_open(struct s3_conn *, int); +static void init_offload_conn(struct s3_conn *, struct t3cdev *, + struct dst_entry *); + +/* + * Insert a connection into the TID table and take an extra reference. + */ +static inline void c3cn_insert_tid(struct cxgb3i_sdev_data *cdata, + struct s3_conn *c3cn, + unsigned int tid) +{ + c3cn_hold(c3cn); + cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid); +} + +static inline void free_atid(struct t3cdev *cdev, unsigned int tid) +{ + struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid); + if (c3cn) + c3cn_put(c3cn); +} + +/* + * This function is intended for allocations of small control messages. + * Such messages go as immediate data and usually the pakets are freed + * immediately. We maintain a cache of one small sk_buff and use it whenever + * it is available (has a user count of 1). Otherwise we get a fresh buffer. + */ +#define CTRL_SKB_LEN 120 + +static struct sk_buff *alloc_ctrl_skb(const struct s3_conn *c3cn, + int len) +{ + struct sk_buff *skb = c3cn->ctrl_skb_cache; + + if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) { + __skb_trim(skb, 0); + atomic_set(&skb->users, 2); + } else if (likely(!in_atomic())) + skb = alloc_skb(len, GFP_ATOMIC | __GFP_NOFAIL); + else + skb = alloc_skb(len, GFP_ATOMIC); + return skb; +} + +/** + * cxgb3_egress_dev - return the cxgb3 egress device or NULL if the egress + * device isn't one of our ports. + * + * @root_dev: the root device anchoring the search + * @c3cn: the connection used to determine egress port in bonding mode + * @context: in bonding mode, indicates a connection set up or failover + * + * Given a root network device it returns the physical egress device that is a + * descendant of the root device. The root device may be either a physical + * device, in which case it is the device returned, or a virtual device, such + * as a VLAN or bonding device. In case of a bonding device the search + * considers the decisions of the bonding device given its mode to locate the + * correct egress device. + */ +static struct net_device *cxgb3_egress_dev(struct net_device *root_dev, + struct s3_conn *c3cn, + int context) +{ + while (root_dev) { + if (root_dev->priv_flags & IFF_802_1Q_VLAN) + root_dev = vlan_dev_real_dev(root_dev); + else if (is_cxgb3_dev(root_dev)) + return root_dev; + else + return NULL; + } + return NULL; +} + +/* + * Return TRUE if we're able to establish an offload connection; otherwise + * return FALSE. + */ +static int s3_connect(struct s3_conn *c3cn) +{ + struct net_device *dev = cxgb3_egress_dev(c3cn->dst_cache->dev, + c3cn, 0); + if (dev == NULL) { + c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn); + return 0; + } + return act_open(c3cn, dev) == 0; +} + +/* + * Handle an ARP failure for an active open. + */ +static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb) +{ + struct s3_conn *c3cn = (struct s3_conn *)skb->sk; + + c3cn_hold(c3cn); + spin_lock(&c3cn->lock); + if (c3cn->state == C3CN_STATE_SYN_SENT) { + fail_act_open(c3cn, EHOSTUNREACH); + __kfree_skb(skb); + } + spin_unlock(&c3cn->lock); + c3cn_put(c3cn); +} + +/* + * Send an active open request. + */ +static int act_open(struct s3_conn *c3cn, struct net_device *dev) +{ + struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev); + struct t3cdev *cdev = cdata->cdev; + struct dst_entry *dst = c3cn->dst_cache; + struct sk_buff *skb; + + c3cn_conn_debug("c3cn 0x%p.\n", c3cn); + /* + * Initialize connection data. Note that the flags and ULP mode are + * initialized higher up ... + */ + c3cn->dev = dev; + c3cn->cdev = cdev; + c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn); + if (c3cn->tid < 0) + goto out_err; + + c3cn->qset = 0; + c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev); + if (!c3cn->l2t) + goto free_tid; + + skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL); + if (!skb) + goto free_l2t; + + skb->sk = (struct sock *)c3cn; + set_arp_failure_handler(skb, act_open_req_arp_failure); + + c3cn_hold(c3cn); + + init_offload_conn(c3cn, cdev, dst); + c3cn->err = 0; + c3cn_reset_flag(c3cn, C3CN_DONE); + + mk_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t); + l2t_send(cdev, skb, c3cn->l2t); + return 0; + +free_l2t: + l2t_release(L2DATA(cdev), c3cn->l2t); +free_tid: + free_atid(cdev, c3cn->tid); + c3cn->tid = 0; +out_err: + return -1; +} + +/* + * Close a connection by sending a CPL_CLOSE_CON_REQ message. Cannot fail + * under any circumstances. We take the easy way out and always queue the + * message to the write_queue. We can optimize the case where the queue is + * already empty though the optimization is probably not worth it. + */ +static void mk_close_req(struct s3_conn *c3cn) +{ + struct sk_buff *skb; + struct cpl_close_con_req *req; + unsigned int tid = c3cn->tid; + + c3cn_conn_debug("c3cn 0x%p.\n", c3cn); + + skb = alloc_skb(sizeof(struct cpl_close_con_req), + GFP_KERNEL | __GFP_NOFAIL); + req = (struct cpl_close_con_req *)__skb_put(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); + req->wr.wr_lo = htonl(V_WR_TID(tid)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); + req->rsvd = htonl(c3cn->write_seq); + + skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND); + if (c3cn->state != C3CN_STATE_SYN_SENT) + s3_push_frames(c3cn, 1); +} + +static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb, + int flags) +{ + CXGB3_SKB_CB(skb)->seq = c3cn->write_seq; + CXGB3_SKB_CB(skb)->flags = flags; + __skb_queue_tail(&c3cn->write_queue, skb); +} + +/* + * Send RX credits through an RX_DATA_ACK CPL message. If nofail is 0 we are + * permitted to return without sending the message in case we cannot allocate +
