/* * Keystone crypto accelerator driver * * Copyright (C) 2015, 2016 Texas Instruments Incorporated - http://www.ti.com * * Authors: Sandeep Nair * Vitaly Andrianov * * Contributors:Tinku Mannan * Hao Zhang * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * version 2 as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "keystone-sa.h" #include "keystone-sa-hlp.h" #define SA_SW0_EVICT_FL_SHIFT 16 #define SA_SW0_TEAR_FL_SHIFT 17 #define SA_SW0_NOPAYLD_FL_SHIFT 18 #define SA_SW0_CMDL_INFO_SHIFT 20 #define SA_SW0_ENG_ID_SHIFT 25 #define SA_SW0_CPPI_DST_INFO_PRESENT BIT(30) #define SA_CMDL_PRESENT BIT(4) #define SA_SW2_EGRESS_CPPI_FLOW_ID_SHIFT 16 #define SA_SW2_EGRESS_CPPI_STATUS_LEN_SHIFT 24 #define SA_CMDL_UPD_ENC 0x0001 #define SA_CMDL_UPD_AUTH 0x0002 #define SA_CMDL_UPD_ENC_IV 0x0004 #define SA_CMDL_UPD_AUTH_IV 0x0008 #define SA_CMDL_UPD_AUX_KEY 0x0010 /* size of SCCTL structure in bytes */ #define SA_SCCTL_SZ 8 /* Tear down the Security Context */ #define SA_SC_TEAR_RETRIES 5 #define SA_SC_TEAR_DELAY 20 /* msecs */ /* Algorithm interface functions & templates */ struct sa_alg_tmpl { u32 type; /* CRYPTO_ALG_TYPE from */ union { struct crypto_alg crypto; struct aead_alg aead; } alg; bool registered; }; /* Number of elements in scatterlist */ static int sg_count(struct scatterlist *sg, int len) { int sg_nents = 0; while (sg && (len > 0)) { sg_nents++; len -= sg->length; sg = sg_next(sg); } return sg_nents; } /* buffer capacity of scatterlist */ static int sg_len(struct scatterlist *sg) { int len = 0; while (sg) { len += sg->length; sg = sg_next(sg); } return len; } /* Copy buffer content from list of hwdesc-s to DST SG list */ static int sa_hwdesc2sg_copy(struct knav_dma_desc **hwdesc, struct scatterlist *dst, unsigned int src_offset, unsigned int dst_offset, size_t len, int num) { struct scatter_walk walk; int sglen, cplen; int j = 0; sglen = hwdesc[0]->desc_info & KNAV_DMA_DESC_PKT_LEN_MASK; if (unlikely(len + src_offset > sglen)) { pr_err("[%s] src len(%d) less than (%d)\n", __func__, sglen, len + src_offset); return -EINVAL; } sglen = sg_len(dst); if (unlikely(len + dst_offset > sglen)) { pr_err("[%s] dst len(%d) less than (%d)\n", __func__, sglen, len + dst_offset); return -EINVAL; } scatterwalk_start(&walk, dst); scatterwalk_advance(&walk, dst_offset); while ((j < num) && (len > 0)) { cplen = min((int)len, (int)(hwdesc[j]->buff_len - src_offset)); if (likely(cplen)) { scatterwalk_copychunks(((char *)hwdesc[j]->pad[0] + src_offset), &walk, cplen, 1); } len -= cplen; j++; src_offset = 0; } return 0; } static void scatterwalk_copy(void *buf, struct scatterlist *sg, unsigned int start, unsigned int nbytes, int out) { struct scatter_walk walk; unsigned int offset = 0; if (!nbytes) return; for (;;) { scatterwalk_start(&walk, sg); if (start < offset + sg->length) break; offset += sg->length; sg = sg_next(sg); } scatterwalk_advance(&walk, start - offset); scatterwalk_copychunks(buf, &walk, nbytes, out); } /* Command Label Definitions and utility functions */ struct sa_cmdl_cfg { int enc1st; int aalg; u8 enc_eng_id; u8 auth_eng_id; u8 iv_size; const u8 *akey; u16 akey_len; }; /* Format general command label */ static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl, struct sa_cmdl_upd_info *upd_info) { u8 offset = 0; u32 *word_ptr = (u32 *)cmdl; int i; int ret = 0; /* Clear the command label */ memset(cmdl, 0, (SA_MAX_CMDL_WORDS * sizeof(u32))); /* Iniialize the command update structure */ memset(upd_info, 0, sizeof(*upd_info)); upd_info->enc_size.offset = 2; upd_info->enc_size.size = 2; upd_info->enc_offset.size = 1; upd_info->enc_size2.size = 4; upd_info->auth_size.offset = 2; upd_info->auth_size.size = 2; upd_info->auth_offset.size = 1; if (cfg->aalg == SA_AALG_ID_AES_XCBC) { /* Derive K2/K3 subkeys */ ret = sa_aes_xcbc_subkey(NULL, (u8 *)&upd_info->aux_key[0], (u8 *)&upd_info->aux_key[AES_BLOCK_SIZE / sizeof(u32)], cfg->akey, cfg->akey_len); if (ret) return ret; /* * Format the key into 32bit CPU words * from a big-endian stream */ for (i = 0; i < SA_MAX_AUX_DATA_WORDS; i++) upd_info->aux_key[i] = be32_to_cpu(upd_info->aux_key[i]); } if (cfg->enc1st) { if (cfg->enc_eng_id != SA_ENG_ID_NONE) { upd_info->flags |= SA_CMDL_UPD_ENC; upd_info->enc_size.index = 0; upd_info->enc_offset.index = 1; if ((cfg->enc_eng_id == SA_ENG_ID_EM1) && (cfg->auth_eng_id == SA_ENG_ID_EM1)) cfg->auth_eng_id = SA_ENG_ID_EM2; /* Encryption command label */ if (cfg->auth_eng_id != SA_ENG_ID_NONE) cmdl[SA_CMDL_OFFSET_NESC] = cfg->auth_eng_id; else cmdl[SA_CMDL_OFFSET_NESC] = SA_ENG_ID_OUTPORT2; /* Encryption modes requiring IV */ if (cfg->iv_size) { upd_info->flags |= SA_CMDL_UPD_ENC_IV; upd_info->enc_iv.index = SA_CMDL_HEADER_SIZE_BYTES >> 2; upd_info->enc_iv.size = cfg->iv_size; cmdl[SA_CMDL_OFFSET_LABEL_LEN] = SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; cmdl[SA_CMDL_OFFSET_OPTION_CTRL1] = (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3)); offset = SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; } else { cmdl[SA_CMDL_OFFSET_LABEL_LEN] = SA_CMDL_HEADER_SIZE_BYTES; offset = SA_CMDL_HEADER_SIZE_BYTES; } } if (cfg->auth_eng_id != SA_ENG_ID_NONE) { upd_info->flags |= SA_CMDL_UPD_AUTH; upd_info->auth_size.index = offset >> 2; upd_info->auth_offset.index = upd_info->auth_size.index + 1; cmdl[offset + SA_CMDL_OFFSET_NESC] = SA_ENG_ID_OUTPORT2; /* Algorithm with subkeys */ if ((cfg->aalg == SA_AALG_ID_AES_XCBC) || (cfg->aalg == SA_AALG_ID_CMAC)) { upd_info->flags |= SA_CMDL_UPD_AUX_KEY; upd_info->aux_key_info.index = (offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2; cmdl[offset + SA_CMDL_OFFSET_LABEL_LEN] = SA_CMDL_HEADER_SIZE_BYTES + 16; cmdl[offset + SA_CMDL_OFFSET_OPTION_CTRL1] = (SA_CTX_ENC_AUX1_OFFSET | (16 >> 3)); offset += SA_CMDL_HEADER_SIZE_BYTES + 16; } else { cmdl[offset + SA_CMDL_OFFSET_LABEL_LEN] = SA_CMDL_HEADER_SIZE_BYTES; offset += SA_CMDL_HEADER_SIZE_BYTES; } } } else { /* Auth first */ if (cfg->auth_eng_id != SA_ENG_ID_NONE) { upd_info->flags |= SA_CMDL_UPD_AUTH; upd_info->auth_size.index = 0; upd_info->auth_offset.index = 1; if ((cfg->auth_eng_id == SA_ENG_ID_EM1) && (cfg->enc_eng_id == SA_ENG_ID_EM1)) cfg->enc_eng_id = SA_ENG_ID_EM2; /* Authentication command label */ if (cfg->enc_eng_id != SA_ENG_ID_NONE) cmdl[SA_CMDL_OFFSET_NESC] = cfg->enc_eng_id; else cmdl[SA_CMDL_OFFSET_NESC] = SA_ENG_ID_OUTPORT2; /* Algorithm with subkeys */ if ((cfg->aalg == SA_AALG_ID_AES_XCBC) || (cfg->aalg == SA_AALG_ID_CMAC)) { upd_info->flags |= SA_CMDL_UPD_AUX_KEY; upd_info->aux_key_info.index = (SA_CMDL_HEADER_SIZE_BYTES) >> 2; cmdl[SA_CMDL_OFFSET_LABEL_LEN] = SA_CMDL_HEADER_SIZE_BYTES + 16; cmdl[offset + SA_CMDL_OFFSET_OPTION_CTRL1] = (SA_CTX_ENC_AUX1_OFFSET | (16 >> 3)); offset = SA_CMDL_HEADER_SIZE_BYTES + 16; } else { cmdl[SA_CMDL_OFFSET_LABEL_LEN] = SA_CMDL_HEADER_SIZE_BYTES; offset = SA_CMDL_HEADER_SIZE_BYTES; } } if (cfg->enc_eng_id != SA_ENG_ID_NONE) { upd_info->flags |= SA_CMDL_UPD_ENC; upd_info->enc_size.index = offset >> 2; upd_info->enc_offset.index = upd_info->enc_size.index + 1; cmdl[offset + SA_CMDL_OFFSET_NESC] = SA_ENG_ID_OUTPORT2; /* Encryption modes requiring IV */ if (cfg->iv_size) { upd_info->flags |= SA_CMDL_UPD_ENC_IV; upd_info->enc_iv.index = (offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2; upd_info->enc_iv.size = cfg->iv_size; cmdl[offset + SA_CMDL_OFFSET_LABEL_LEN] = SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; cmdl[offset + SA_CMDL_OFFSET_OPTION_CTRL1] = (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3)); offset += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; } else { cmdl[offset + SA_CMDL_OFFSET_LABEL_LEN] = SA_CMDL_HEADER_SIZE_BYTES; offset += SA_CMDL_HEADER_SIZE_BYTES; } } } offset = roundup(offset, 8); for (i = 0; i < offset / 4; i++) word_ptr[i] = be32_to_cpu(word_ptr[i]); return offset; } static inline void sa_copy_iv(u32 *out, const u8 *iv, bool size16) { int j; for (j = 0; j < ((size16) ? 4 : 2); j++) { *out = cpu_to_be32(*((u32 *)iv)); iv += 4; out++; } } /* Update Command label */ static inline void sa_update_cmdl(struct device *dev, u8 enc_offset, u16 enc_size, u8 *enc_iv, u8 auth_offset, u16 auth_size, u8 *auth_iv, u8 aad_size, u8 *aad, struct sa_cmdl_upd_info *upd_info, u32 *cmdl) { switch (upd_info->submode) { case SA_MODE_GEN: if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) { cmdl[upd_info->enc_size.index] &= 0xffff0000; cmdl[upd_info->enc_size.index] |= enc_size; cmdl[upd_info->enc_offset.index] &= 0x00ffffff; cmdl[upd_info->enc_offset.index] |= ((u32)enc_offset << 24); if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) { sa_copy_iv(&cmdl[upd_info->enc_iv.index], enc_iv, (upd_info->enc_iv.size > 8)); } } if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) { cmdl[upd_info->auth_size.index] &= 0xffff0000; cmdl[upd_info->auth_size.index] |= auth_size; cmdl[upd_info->auth_offset.index] &= 0x00ffffff; cmdl[upd_info->auth_offset.index] |= ((u32)auth_offset << 24); if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) { sa_copy_iv(&cmdl[upd_info->auth_iv.index], auth_iv, (upd_info->auth_iv.size > 8)); } if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) { int offset = (auth_size & 0xF) ? 4 : 0; memcpy(&cmdl[upd_info->aux_key_info.index], &upd_info->aux_key[offset], 16); } } break; case SA_MODE_CCM: case SA_MODE_GCM: case SA_MODE_GMAC: default: dev_err(dev, "unsupported mode(%d)\n", upd_info->submode); break; } } /* Format SWINFO words to be sent to SA */ static void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys, u8 cmdl_present, u8 cmdl_offset, u8 flags, u16 queue_id, u8 flow_id, u8 hash_size, u32 *swinfo) { swinfo[0] = sc_id; swinfo[0] |= (flags << SA_SW0_EVICT_FL_SHIFT); if (likely(cmdl_present)) swinfo[0] |= ((cmdl_offset | SA_CMDL_PRESENT) << SA_SW0_CMDL_INFO_SHIFT); swinfo[0] |= (eng_id << SA_SW0_ENG_ID_SHIFT); swinfo[0] |= SA_SW0_CPPI_DST_INFO_PRESENT; swinfo[1] = sc_phys; swinfo[2] = (queue_id | (flow_id << SA_SW2_EGRESS_CPPI_FLOW_ID_SHIFT) | (hash_size << SA_SW2_EGRESS_CPPI_STATUS_LEN_SHIFT)); } /* Security context creation functions */ /* Dump the security context */ static void sa_dump_sc(u8 *buf, u32 dma_addr) { #ifdef DEBUG dev_info(sa_ks2_dev, "Security context dump for %p:\n", (void *)dma_addr); print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, 16, 1, buf, SA_CTX_MAX_SZ, false); #endif } /* Initialize Security context */ static int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key, u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz, const char *cra_name, u8 enc, u32 *swinfo) { struct sa_eng_info *enc_eng, *auth_eng; int ealg_id, aalg_id, use_enc = 0; int enc_sc_offset, auth_sc_offset; u8 php_f, php_e, eng0_f, eng1_f; u8 *sc_buf = ctx->sc; u16 sc_id = ctx->sc_id; u16 aad_len = 0; /* Currently not supporting AEAD algo */ u8 first_engine; u8 hash_size; int ret = 0; memset(sc_buf, 0, SA_CTX_MAX_SZ); sa_conv_calg_to_salg(cra_name, &ealg_id, &aalg_id); enc_eng = sa_get_engine_info(ealg_id); auth_eng = sa_get_engine_info(aalg_id); if (!enc_eng->sc_size && !auth_eng->sc_size) return -EINVAL; if (auth_eng->eng_id <= SA_ENG_ID_EM2) use_enc = 1; /* Determine the order of encryption & Authentication contexts */ if (enc || !use_enc) { eng0_f = SA_CTX_SIZE_TO_DMA_SIZE(enc_eng->sc_size); eng1_f = SA_CTX_SIZE_TO_DMA_SIZE(auth_eng->sc_size); enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; auth_sc_offset = enc_sc_offset + enc_eng->sc_size; } else { eng0_f = SA_CTX_SIZE_TO_DMA_SIZE(auth_eng->sc_size); eng1_f = SA_CTX_SIZE_TO_DMA_SIZE(enc_eng->sc_size); auth_sc_offset = SA_CTX_PHP_PE_CTX_SZ; enc_sc_offset = auth_sc_offset + auth_eng->sc_size; } php_f = SA_CTX_DMA_SIZE_64; php_e = SA_CTX_DMA_SIZE_64; /* SCCTL Owner info: 0=host, 1=CP_ACE */ sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0; /* SCCTL F/E control */ sc_buf[1] = SA_CTX_SCCTL_MK_DMA_INFO(php_f, eng0_f, eng1_f, php_e); memcpy(&sc_buf[2], &sc_id, 2); memcpy(&sc_buf[4], &ctx->sc_phys, 4); /* Initialize the rest of PHP context */ memset(sc_buf + SA_SCCTL_SZ, 0, SA_CTX_PHP_PE_CTX_SZ - SA_SCCTL_SZ); /* Prepare context for encryption engine */ if (enc_eng->sc_size) { ret = sa_set_sc_enc(ealg_id, enc_key, enc_key_sz, aad_len, enc, &sc_buf[enc_sc_offset]); if (ret) return ret; } /* Prepare context for authentication engine */ if (auth_eng->sc_size) { if (use_enc) { if (sa_set_sc_enc(aalg_id, auth_key, auth_key_sz, aad_len, 0, &sc_buf[auth_sc_offset])) return -1; } else sa_set_sc_auth(aalg_id, auth_key, auth_key_sz, &sc_buf[auth_sc_offset]); } /* Set the ownership of context to CP_ACE */ sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80; /* swizzle the security context */ sa_swiz_128(sc_buf, sc_buf, SA_CTX_MAX_SZ); /* Setup SWINFO */ first_engine = enc ? enc_eng->eng_id : auth_eng->eng_id; /* TODO: take care of AEAD algorithms */ hash_size = sa_get_hash_size(aalg_id); if (!hash_size) return -EINVAL; /* Round up the tag size to multiple of 8 */ hash_size = roundup(hash_size, 8); sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0, #ifndef TEST 0, #else /* * For run-time self tests in the cryptographic * algorithm manager framework */ SA_SW_INFO_FLAG_EVICT, #endif ctx->rx_compl_qid, ctx->rx_flow, hash_size, swinfo); sa_dump_sc(sc_buf, ctx->sc_phys); return 0; } static int sa_tear_sc(struct sa_ctx_info *ctx, struct keystone_crypto_data *pdata) { struct device *dev = &pdata->pdev->dev; int own_off, cnt = SA_SC_TEAR_RETRIES; struct knav_dma_desc *hwdesc; struct sa_dma_req_ctx *dma_ctx; int ret = 0; u32 packet_info; int j; dma_addr_t dma_addr; u32 dma_sz; dma_ctx = kmem_cache_alloc(pdata->dma_req_ctx_cache, GFP_KERNEL); if (!dma_ctx) { ret = -ENOMEM; goto err; } dma_ctx->dev_data = pdata; dma_ctx->pkt = false; sa_set_swinfo(SA_ENG_ID_OUTPORT2, ctx->sc_id, ctx->sc_phys, 0, 0, (SA_SW_INFO_FLAG_TEAR | SA_SW_INFO_FLAG_EVICT | SA_SW_INFO_FLAG_NOPD), ctx->rx_compl_qid, ctx->rx_flow, 0, &ctx->epib[1]); ctx->epib[0] = 0; /* map the packet */ packet_info = KNAV_DMA_DESC_HAS_EPIB | (pdata->tx_compl_qid << KNAV_DMA_DESC_RETQ_SHIFT); hwdesc = knav_pool_desc_get(pdata->tx_pool); if (IS_ERR_OR_NULL(hwdesc)) { dev_dbg(dev, "out of tx pool desc\n"); ret = -ENOBUFS; goto err; } memset(hwdesc, 0, sizeof(struct knav_dma_desc)); for (j = 0; j < 4; j++) hwdesc->epib[j] = ctx->epib[j]; hwdesc->packet_info = packet_info; knav_pool_desc_map(pdata->tx_pool, hwdesc, sizeof(hwdesc), &dma_addr, &dma_sz); hwdesc->pad[0] = (u32)dma_addr; hwdesc->pad[1] = dma_sz; hwdesc->pad[2] = (u32)dma_ctx; knav_queue_push(pdata->tx_submit_q, dma_addr, sizeof(struct knav_dma_desc), 0); /* * Check that CP_ACE has released the context * by making sure that the owner bit is 0 */ /* * Security context had been swizzled by 128 bits * before handing to CP_ACE */ own_off = ((SA_CTX_SCCTL_OWNER_OFFSET / 16) * 16) + (15 - (SA_CTX_SCCTL_OWNER_OFFSET % 16)); while (__raw_readb(&ctx->sc[own_off])) { if (!--cnt) return -EAGAIN; msleep_interruptible(SA_SC_TEAR_DELAY); } return 0; err: atomic_inc(&pdata->stats.sc_tear_dropped); if (dma_ctx) kmem_cache_free(pdata->dma_req_ctx_cache, dma_ctx); return ret; } /* Free the per direction context memory */ static int sa_free_ctx_info(struct sa_ctx_info *ctx, struct keystone_crypto_data *data) { unsigned long bn; int ret = 0; ret = sa_tear_sc(ctx, data); if (ret) { dev_err(sa_ks2_dev, "Failed to tear down context id(%x)\n", ctx->sc_id); return ret; } bn = ctx->sc_id - data->sc_id_start; spin_lock(&data->scid_lock); __clear_bit(bn, data->ctx_bm); data->sc_id--; spin_unlock(&data->scid_lock); if (ctx->sc) { dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys); ctx->sc = NULL; } return 0; } /* Initialize the per direction context memory */ static int sa_init_ctx_info(struct sa_ctx_info *ctx, struct keystone_crypto_data *data) { unsigned long bn; int err; spin_lock(&data->scid_lock); if (data->sc_id > data->sc_id_end) { spin_unlock(&data->scid_lock); dev_err(&data->pdev->dev, "Out of SC IDs\n"); return -ENOMEM; } bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX); __set_bit(bn, data->ctx_bm); data->sc_id++; spin_unlock(&data->scid_lock); ctx->sc_id = (u16)(data->sc_id_start + bn); ctx->rx_flow = knav_dma_get_flow(data->rx_chan); ctx->rx_compl_qid = data->rx_compl_qid; ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys); if (!ctx->sc) { dev_err(&data->pdev->dev, "Failed to allocate SC memory\n"); err = -ENOMEM; goto scid_rollback; } return 0; scid_rollback: spin_lock(&data->scid_lock); __clear_bit(bn, data->ctx_bm); data->sc_id--; spin_unlock(&data->scid_lock); return err; } /* Initialize TFM context */ static int sa_init_tfm(struct crypto_tfm *tfm) { struct crypto_alg *alg = tfm->__crt_alg; struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); struct keystone_crypto_data *data = dev_get_drvdata(sa_ks2_dev); int ret; if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AEAD) { memset(ctx, 0, sizeof(*ctx)); ctx->dev_data = data; ret = sa_init_ctx_info(&ctx->enc, data); if (ret) return ret; ret = sa_init_ctx_info(&ctx->dec, data); if (ret) { sa_free_ctx_info(&ctx->enc, data); return ret; } } dev_dbg(sa_ks2_dev, "%s(0x%p) sc-ids(0x%x(0x%x), 0x%x(0x%x))\n", __func__, tfm, ctx->enc.sc_id, ctx->enc.sc_phys, ctx->dec.sc_id, ctx->dec.sc_phys); return 0; } /* Algorithm init */ static int sa_cra_init_aead(struct crypto_aead *tfm) { return sa_init_tfm(crypto_aead_tfm(tfm)); } /* Algorithm context teardown */ static void sa_exit_tfm(struct crypto_tfm *tfm) { struct crypto_alg *alg = tfm->__crt_alg; struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); struct keystone_crypto_data *data = dev_get_drvdata(sa_ks2_dev); dev_dbg(sa_ks2_dev, "%s(0x%p) sc-ids(0x%x(0x%x), 0x%x(0x%x))\n", __func__, tfm, ctx->enc.sc_id, ctx->enc.sc_phys, ctx->dec.sc_id, ctx->dec.sc_phys); if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AEAD) { sa_free_ctx_info(&ctx->enc, data); sa_free_ctx_info(&ctx->dec, data); } } static void sa_exit_tfm_aead(struct crypto_aead *tfm) { return sa_exit_tfm(crypto_aead_tfm(tfm)); } /* AEAD algorithm configuration interface function */ static int sa_aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc); struct crypto_authenc_keys keys; const char *cra_name; struct sa_eng_info *enc_eng, *auth_eng; int ealg_id, aalg_id, cmdl_len; struct sa_cmdl_cfg cfg; if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; cra_name = crypto_tfm_alg_name(crypto_aead_tfm(authenc)); sa_conv_calg_to_salg(cra_name, &ealg_id, &aalg_id); enc_eng = sa_get_engine_info(ealg_id); auth_eng = sa_get_engine_info(aalg_id); memset(&cfg, 0, sizeof(cfg)); cfg.enc1st = 1; cfg.aalg = aalg_id; cfg.enc_eng_id = enc_eng->eng_id; cfg.auth_eng_id = auth_eng->eng_id; cfg.iv_size = crypto_aead_ivsize(authenc); cfg.akey = keys.authkey; cfg.akey_len = keys.authkeylen; /* Setup Encryption Security Context & Command label template */ if (sa_init_sc(&ctx->enc, keys.enckey, keys.enckeylen, keys.authkey, keys.authkeylen, cra_name, 1, &ctx->enc.epib[1])) goto badkey; cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->enc.cmdl, &ctx->enc.cmdl_upd_info); if ((cmdl_len <= 0) || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) goto badkey; ctx->enc.cmdl_size = cmdl_len; /* Setup Decryption Security Context & Command label template */ if (sa_init_sc(&ctx->dec, keys.enckey, keys.enckeylen, keys.authkey, keys.authkeylen, cra_name, 0, &ctx->dec.epib[1])) goto badkey; cfg.enc1st = 0; cfg.enc_eng_id = enc_eng->eng_id; cfg.auth_eng_id = auth_eng->eng_id; cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, &ctx->dec.cmdl_upd_info); if ((cmdl_len <= 0) || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) goto badkey; ctx->dec.cmdl_size = cmdl_len; return 0; badkey: dev_err(sa_ks2_dev, "%s: badkey\n", __func__); crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } /** * sa_prepare_tx_desc() - prepare a chain of tx descriptors * @pdata: struct keystone_crypto_data pinter * @_sg: struct scatterlist source list * @num_sg: number of buffers in the _sg list * @pslen: length of protocol specific data * @psdata: pointer to the protocol specific data * @epiblen: EPIB length * @epib: pointer to EPIB (extended packet info block) * @ctx: struct sa_dma_req_ctx pointer * * For each buffer in the source _sg list the function gets a hardware * descriptor from tx_pool and fills the buffer descriptor fields and maps the * descriptor. For the first descriptor (packet descriptor) it also sets the * psinfo and epib fields. * * Returns dma address of the first descripto on success, NULL otherwise. */ static dma_addr_t sa_prepare_tx_desc(struct keystone_crypto_data *pdata, struct scatterlist *_sg, int num_sg, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib, struct sa_dma_req_ctx *ctx) { struct device *dev = &pdata->pdev->dev; struct knav_dma_desc *hwdesc = NULL; struct scatterlist *sg = _sg; u32 packet_len = 0; u32 nsg; u32 next_desc = 0; u32 packet_info; packet_info = KNAV_DMA_DESC_HAS_EPIB | ((pslen / sizeof(u32)) << KNAV_DMA_DESC_PSLEN_SHIFT) | (pdata->tx_compl_qid << KNAV_DMA_DESC_RETQ_SHIFT); for (sg += num_sg - 1, nsg = num_sg; nsg > 0; sg--, nsg--) { u32 buflen, orig_len; int i; dma_addr_t dma_addr; u32 dma_sz; u32 *out, *in; hwdesc = knav_pool_desc_get(pdata->tx_pool); if (IS_ERR_OR_NULL(hwdesc)) { dev_dbg(dev, "out of tx pool desc\n"); return 0; } buflen = sg_dma_len(sg) & KNAV_DMA_DESC_PKT_LEN_MASK; orig_len = buflen; packet_len += buflen; if (nsg == 1) { /* extra fileds for packed descriptor */ for (out = hwdesc->epib, in = epib, i = 0; i < epiblen / sizeof(u32); i++) *out++ = *in++; for (out = hwdesc->psdata, in = psdata, i = 0; i < pslen / sizeof(u32); i++) *out++ = *in++; } hwdesc->desc_info = packet_len; hwdesc->tag_info = 0; hwdesc->packet_info = packet_info; hwdesc->buff_len = buflen; hwdesc->buff = sg_dma_address(sg); hwdesc->next_desc = next_desc; hwdesc->orig_len = orig_len; hwdesc->orig_buff = sg_dma_address(sg); knav_pool_desc_map(pdata->tx_pool, hwdesc, sizeof(hwdesc), &dma_addr, &dma_sz); hwdesc->pad[0] = (u32)dma_addr; hwdesc->pad[1] = dma_sz; hwdesc->pad[2] = (u32)ctx; next_desc = (u32)dma_addr; } return (unlikely(!hwdesc)) ? 0 : hwdesc->pad[0]; } void sa_tx_completion_process(struct keystone_crypto_data *dev_data) { struct knav_dma_desc *hwdesc = NULL; dma_addr_t dma; struct sa_dma_req_ctx *ctx = NULL; u32 pkt_len; u32 calc_pkt_len; for (;;) { dma = knav_queue_pop(dev_data->tx_compl_q, NULL); if (!dma) { dev_dbg(sa_ks2_dev, "no desc in the queue %d\n", dev_data->tx_compl_qid); break; } ctx = NULL; pkt_len = 0; calc_pkt_len = 0; do { hwdesc = knav_pool_desc_unmap(dev_data->tx_pool, dma, sizeof(hwdesc)); if (!hwdesc) { pr_err("failed to unmap descriptor 0x%08x\n", dma); break; } /* take the req_ctx from the first descriptor */ if (!ctx) { ctx = (struct sa_dma_req_ctx *)hwdesc->pad[2]; pkt_len = hwdesc->desc_info & KNAV_DMA_DESC_PKT_LEN_MASK; } calc_pkt_len += hwdesc->buff_len; dma = hwdesc->next_desc; knav_pool_desc_put(dev_data->tx_pool, hwdesc); } while (dma); #ifdef DEBUG if (pkt_len != calc_pkt_len) pr_err("[%s] calculated packet length doesn't match %d/%d\n", __func__, calc_pkt_len, pkt_len); #endif if ((pkt_len > 0) && ctx) { dma_unmap_sg(&ctx->dev_data->pdev->dev, ctx->src, ctx->src_nents, DMA_TO_DEVICE); if (likely(ctx->pkt)) { atomic_add(ctx->src_nents, &ctx->dev_data->tx_dma_desc_cnt); atomic_inc(&ctx->dev_data->stats.tx_pkts); } } if (ctx) kmem_cache_free(ctx->dev_data->dma_req_ctx_cache, ctx); } } /** * sa_rx_desc_process() - proccess descriptors related * to one trasnsformation received from SA * * @dev_data: struct keystone_crypto_data pointer * @hwdesc: array of pointers to descriptors * @num: number descriptors in the array * * From the first descriptor, which is a packer descriptor, the function * retrieves all algorithm parameters including pointer to original request. * If the transformation was an encryption, it copies calculated authentication * tag to the destination list, otherwise compare received tag with calculated. * * After that it copies all buffers from hw descriptors to the destination list * and call aead_request_complete() callback. * * At the end the function frees all buffers. */ static void sa_rx_desc_process(struct keystone_crypto_data *dev_data, struct knav_dma_desc **hwdesc, int num) { int j; unsigned int alg_type; u32 req_sub_type; alg_type = hwdesc[0]->psdata[0] & CRYPTO_ALG_TYPE_MASK; req_sub_type = hwdesc[0]->psdata[0] >> SA_REQ_SUBTYPE_SHIFT; if (likely(alg_type == CRYPTO_ALG_TYPE_AEAD)) { int auth_words, auth_size, enc_len, enc_offset, i; struct aead_request *req; struct crypto_aead *tfm; int enc, err = 0; unsigned int ivsize; req = (struct aead_request *)hwdesc[0]->psdata[1]; tfm = crypto_aead_reqtfm(req); auth_size = crypto_aead_authsize(tfm); ivsize = crypto_aead_ivsize(tfm); if (req_sub_type == SA_REQ_SUBTYPE_ENC) { enc_offset = req->assoclen + ivsize; enc_len = req->cryptlen - ivsize; enc = 1; } else if (req_sub_type == SA_REQ_SUBTYPE_DEC) { enc_offset = req->assoclen; enc_len = req->cryptlen - auth_size; enc = 0; } else { err = -EBADMSG; goto aead_err; } /* NOTE: We receive the tag as host endian 32bit words */ auth_words = auth_size / sizeof(u32); for (i = 2; i < (auth_words + SA_PSDATA_CTX_WORDS); i++) hwdesc[0]->psdata[i] = htonl(hwdesc[0]->psdata[i]); /* if encryption, copy the authentication tag */ if (enc) { scatterwalk_copy( &hwdesc[0]->psdata[SA_PSDATA_CTX_WORDS], req->dst, enc_offset + enc_len, auth_size, 1); } else { /* Verify the authentication tag */ u8 auth_tag[SA_MAX_AUTH_TAG_SZ]; scatterwalk_copy(auth_tag, req->src, enc_len + req->assoclen, auth_size, 0); err = memcmp(&hwdesc[0]->psdata[SA_PSDATA_CTX_WORDS], auth_tag, auth_size) ? -EBADMSG : 0; if (unlikely(err)) goto aead_err; } /* Copy the encrypted/decrypted data */ if (unlikely(sa_hwdesc2sg_copy(hwdesc, req->dst, enc_offset, enc_offset, enc_len, num))) err = -EBADMSG; aead_err: aead_request_complete(req, err); } /* free buffers here */ for (j = 0; j < num; j++) { if (hwdesc[j]->orig_len == PAGE_SIZE) { __free_page((struct page *)hwdesc[j]->pad[1]); atomic_dec(&dev_data->rx_dma_page_cnt); } else { kfree((void *)hwdesc[j]->pad[0]); } } atomic_inc(&dev_data->stats.rx_pkts); } /** * sa_rx_completion_process() - processes received from SA buffers * * @dev_data: struct keystone_crypto_data pointer * * The function is called from rx tasklet. It retreives one or multiple * chained hw descriptors and calls sa_rx_desc_process(). After that it * returns all descriptors into the rx_pool. */ void sa_rx_completion_process(struct keystone_crypto_data *dev_data) { struct knav_dma_desc *hwdesc[MAX_SKB_FRAGS]; int j, desc_num; dma_addr_t dma; u32 pkt_len; u32 calc_pkt_len; int wait4pkt = 1; for (;;) { dma = knav_queue_pop(dev_data->rx_compl_q, NULL); if (!dma) { dev_dbg(sa_ks2_dev, "no desc in the queue %d\n", dev_data->rx_compl_qid); break; } pkt_len = 0; calc_pkt_len = 0; wait4pkt = 1; desc_num = 0; do { hwdesc[desc_num] = knav_pool_desc_unmap(dev_data->rx_pool, dma, sizeof(hwdesc)); if (!hwdesc[desc_num]) { pr_err("failed to unmap descriptor 0x%08x\n", dma); break; } if (hwdesc[desc_num]->orig_len == PAGE_SIZE) { dma_unmap_page(sa_ks2_dev, hwdesc[desc_num]->orig_buff, PAGE_SIZE, DMA_FROM_DEVICE); } else { dma_unmap_single(sa_ks2_dev, hwdesc[desc_num]->orig_buff, SA_RX_BUF0_SIZE, DMA_FROM_DEVICE); } /* take the req_ctx from the first descriptor */ if (wait4pkt) { pkt_len = hwdesc[desc_num]->desc_info & KNAV_DMA_DESC_PKT_LEN_MASK; wait4pkt = 0; } calc_pkt_len += hwdesc[desc_num]->buff_len; dma = hwdesc[desc_num]->next_desc; desc_num++; } while (dma); #ifdef DEBUG if (pkt_len != calc_pkt_len) pr_err("[%s] calculated packet length doesn't match %d/%d\n", __func__, calc_pkt_len, pkt_len); #endif /* retrieve data and copy it to the destination sg list */ sa_rx_desc_process(dev_data, hwdesc, desc_num); /* return descriptor to the pool */ for (j = 0; j < desc_num; j++) knav_pool_desc_put(dev_data->rx_pool, hwdesc[j]); } } /** * sa_aead_perform() - perform AEAD transformation * @req: struct aead_request pointer * @iv: initial vector * enc: boolean flag true for encryption, false for decryption * * This function prepare * * 1) checks whether the driver has enought buffers to receive transformed * data. * 2) allocates request context and fills appropriate fields in it. * 3) maps source list * 4) prepare tx dma desctiprors and submits them to the SA queue. * * Return: -EINPROGRESS on success, appropriate error code */ static int sa_aead_perform(struct aead_request *req, u8 *iv, bool enc) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); struct sa_ctx_info *sa_ctx = enc ? &ctx->enc : &ctx->dec; dma_addr_t desc_dma_addr; struct keystone_crypto_data *pdata = dev_get_drvdata(sa_ks2_dev); struct sa_dma_req_ctx *req_ctx = NULL; unsigned int ivsize = crypto_aead_ivsize(tfm); u8 enc_offset; int sg_nents; int psdata_offset, ret = 0; u8 auth_offset = 0; u8 *auth_iv = NULL; u8 *aad = NULL; u8 aad_len = 0; u16 enc_len; u16 auth_len; u32 req_type; int n_bufs; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; if (enc) { iv = (u8 *)(sg_virt(req->src) + req->assoclen); enc_offset = req->assoclen + ivsize; enc_len = req->cryptlen - ivsize; auth_len = req->assoclen + req->cryptlen; } else { enc_offset = req->assoclen; enc_len = req->cryptlen - crypto_aead_authsize(tfm); auth_len = req->assoclen + req->cryptlen - crypto_aead_authsize(tfm); } /* Allocate descriptor & submit packet */ sg_nents = sg_count(req->src, enc_len); if (unlikely(atomic_sub_return(sg_nents, &pdata->tx_dma_desc_cnt) < 0)) { ret = -EBUSY; goto err_0; } n_bufs = auth_len - SA_RX_BUF0_SIZE; n_bufs = (n_bufs <= 0) ? 0 : DIV_ROUND_UP(n_bufs, PAGE_SIZE); if (unlikely(atomic_read(&pdata->rx_dma_page_cnt) < n_bufs)) { ret = -EBUSY; goto err_0; } req_ctx = kmem_cache_alloc(pdata->dma_req_ctx_cache, flags); if (unlikely(!req_ctx)) { ret = -ENOMEM; goto err_0; } memcpy(req_ctx->cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size); /* Update Command Label */ sa_update_cmdl(sa_ks2_dev, enc_offset, enc_len, iv, auth_offset, auth_len, auth_iv, aad_len, aad, &sa_ctx->cmdl_upd_info, req_ctx->cmdl); /* * Last 2 words in PSDATA will have the crypto alg type & * crypto request pointer */ req_type = CRYPTO_ALG_TYPE_AEAD; if (enc) req_type |= (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT); else req_type |= (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT); psdata_offset = sa_ctx->cmdl_size / sizeof(u32); req_ctx->cmdl[psdata_offset++] = req_type; req_ctx->cmdl[psdata_offset] = (u32)req; /* map the packet */ req_ctx->src = req->src; req_ctx->src_nents = dma_map_sg(sa_ks2_dev, req_ctx->src, sg_nents, DMA_TO_DEVICE); if (unlikely(req_ctx->src_nents != sg_nents)) { dev_warn_ratelimited(sa_ks2_dev, "failed to map tx pkt\n"); ret = -EIO; goto err; } req_ctx->dev_data = pdata; req_ctx->pkt = true; desc_dma_addr = sa_prepare_tx_desc(pdata, req_ctx->src, sg_nents, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32))), req_ctx->cmdl, sizeof(sa_ctx->epib), sa_ctx->epib, req_ctx); if (desc_dma_addr == 0) { ret = -EIO; goto err; } knav_queue_push(pdata->tx_submit_q, desc_dma_addr, sizeof(struct knav_dma_desc), 0); return -EINPROGRESS; err: if (req_ctx) kmem_cache_free(pdata->dma_req_ctx_cache, req_ctx); err_0: atomic_add(sg_nents, &pdata->tx_dma_desc_cnt); return ret; } /* AEAD algorithm encrypt interface function */ static int sa_aead_encrypt(struct aead_request *req) { return sa_aead_perform(req, req->iv, true); } /* AEAD algorithm decrypt interface function */ static int sa_aead_decrypt(struct aead_request *req) { return sa_aead_perform(req, req->iv, false); } static struct sa_alg_tmpl sa_algs[] = { /* AEAD algorithms */ { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { .cra_name = "authenc(hmac(sha1),cbc(aes))", .cra_driver_name = "authenc(hmac(sha1),cbc(aes))-keystone-sa", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct sa_tfm_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 0, .cra_priority = 3000, }, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, .init = sa_cra_init_aead, .exit = sa_exit_tfm_aead, .setkey = sa_aead_setkey, .encrypt = sa_aead_encrypt, .decrypt = sa_aead_decrypt, } }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", .cra_driver_name = "authenc(hmac(sha1),cbc(des3_ede))-keystone-sa", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct sa_tfm_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 0, .cra_priority = 3000, }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, .init = sa_cra_init_aead, .exit = sa_exit_tfm_aead, .setkey = sa_aead_setkey, .encrypt = sa_aead_encrypt, .decrypt = sa_aead_decrypt, } }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { .cra_name = "authenc(xcbc(aes),cbc(aes))", .cra_driver_name = "authenc(xcbc(aes),cbc(aes))-keystone-sa", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct sa_tfm_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 0, .cra_priority = 3000, }, .ivsize = AES_BLOCK_SIZE, .maxauthsize = AES_XCBC_DIGEST_SIZE, .init = sa_cra_init_aead, .exit = sa_exit_tfm_aead, .setkey = sa_aead_setkey, .encrypt = sa_aead_encrypt, .decrypt = sa_aead_decrypt, } }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { .cra_name = "authenc(xcbc(aes),cbc(des3_ede))", .cra_driver_name = "authenc(xcbc(aes),cbc(des3_ede))-keystone-sa", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, .cra_ctxsize = sizeof(struct sa_tfm_ctx), .cra_module = THIS_MODULE, .cra_alignmask = 0, .cra_priority = 3000, }, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = AES_XCBC_DIGEST_SIZE, .init = sa_cra_init_aead, .exit = sa_exit_tfm_aead, .setkey = sa_aead_setkey, .encrypt = sa_aead_encrypt, .decrypt = sa_aead_decrypt, } }, }; /* Register the algorithms in crypto framework */ void sa_register_algos(const struct device *dev) { char *alg_name; u32 type; int i, err, num_algs = ARRAY_SIZE(sa_algs); for (i = 0; i < num_algs; i++) { type = sa_algs[i].type; if (type == CRYPTO_ALG_TYPE_AEAD) { alg_name = sa_algs[i].alg.aead.base.cra_name; err = crypto_register_aead(&sa_algs[i].alg.aead); } else { dev_err(dev, "un-supported crypto algorithm (%d)", sa_algs[i].type); continue; } if (err) dev_err(dev, "Failed to register '%s'\n", alg_name); else sa_algs[i].registered = true; } } /* un-register the algorithms from crypto framework */ void sa_unregister_algos(const struct device *dev) { char *alg_name; int err = 0, i, num_algs = ARRAY_SIZE(sa_algs); for (i = 0; i < num_algs; i++) { if (sa_algs[i].registered) { if (sa_algs[i].type == CRYPTO_ALG_TYPE_AEAD) { alg_name = sa_algs[i].alg.aead.base.cra_name; crypto_unregister_aead(&sa_algs[i].alg.aead); err = 0; } else { alg_name = sa_algs[i].alg.crypto.cra_name; err = crypto_unregister_alg(&sa_algs[i].alg.crypto); } sa_algs[i].registered = false; } if (err) dev_err(dev, "Failed to unregister '%s'", alg_name); } }