diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index 1bb8ec575352..67528e2d2ce6 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -468,4 +468,35 @@ config BLK_DEV_RSXX To compile this driver as a module, choose M here: the module will be called rsxx. +config NEURON_APP_BLOCK_CLIENT + tristate "Neuron block client" + depends on NEURON + select NEURON_PROT_BLOCK_CLIENT + help + Neuron is a device-sharing framework which is used by guests of the + haven hypervisor to serve or access shared I/O devices and other + inter-VM services. + + This option enables the Neuron block client, which can access block + devices that are shared via Neuron services. These shared devices can + then be accessed as block device special files such as + /dev/neuron-block0, either directly or by mounting filesystems + on them. + + If unsure, say N. + +config NEURON_APP_BLOCK_SERVER + tristate "Neuron block server" + depends on NEURON + select NEURON_PROT_BLOCK_SERVER + help + Neuron is a device-sharing framework which is used by guests of the + haven hypervisors to serve or access shared I/O devices and other + inter-VM services. + + This option enables the Neuron block device server, which exposes a + specified block device to a client in another VM via a Neuron service. + + If unsure, say N. + endif # BLK_DEV diff --git a/drivers/block/Makefile b/drivers/block/Makefile index a53cc1e3a2d3..3787ab3f1ec9 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -43,3 +43,6 @@ null_blk-$(CONFIG_BLK_DEV_ZONED) += null_blk_zoned.o skd-y := skd_main.o swim_mod-y := swim.o swim_asm.o + +obj-$(CONFIG_NEURON_APP_BLOCK_CLIENT) += neuron_block_client.o +obj-$(CONFIG_NEURON_APP_BLOCK_SERVER) += neuron_block_server.o diff --git a/drivers/block/neuron_block_client.c b/drivers/block/neuron_block_client.c new file mode 100644 index 000000000000..e4191c266113 --- /dev/null +++ b/drivers/block/neuron_block_client.c @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define DRIVER_NAME "neuron-application-block-client" +#define DISK_NAME "nd_" +#define BLOCK_NAME "neuron_block" + +#define bio_sector(bio) bio->bi_iter.bi_sector +#define bio_flags(bio) bio->bi_opf + +static int block_client_major_nr; +static struct ida ida; + +struct block_client_dev { + struct device *dev; + struct request_queue *queue; + struct gendisk *gd; + bool read_only; + uint32_t sector_size; + struct bio_list bio_list; + spinlock_t list_lock; + int id; + struct kref kref; + struct work_struct add_disk_work; +}; + +static const struct of_device_id app_block_client_match[] = { + { + .compatible = "qcom,neuron-block-client", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, app_block_client_match); + +static blk_qc_t block_client_make_request(struct request_queue *q, + struct bio *bio) +{ + struct block_client_dev *blk_dev = bio->bi_disk->private_data; + struct neuron_application *app_dev = + to_neuron_application(blk_dev->dev); + unsigned long flags; + + blk_queue_split(q, &bio); + + if ((bio_op(bio) == REQ_OP_WRITE) && (blk_dev->read_only)) { + pr_err("Permission denied! Read-only block device\n"); + bio_io_error(bio); + return BLK_QC_T_NONE; + } + + spin_lock_irqsave(&blk_dev->list_lock, flags); + bio_list_add(&blk_dev->bio_list, bio); + spin_unlock_irqrestore(&blk_dev->list_lock, flags); + + neuron_app_wakeup(app_dev, NEURON_BLOCK_CLIENT_EVENT_REQUEST); + + return BLK_QC_T_NONE; +} + +static int block_client_open(struct block_device *bdev, fmode_t mode) +{ + struct block_client_dev *blk_dev = bdev->bd_disk->private_data; + + if ((blk_dev->read_only) && (mode & FMODE_WRITE)) { + pr_err("Read/write disk should be read-only\n"); + return -EROFS; + } + + return 0; +} + +static const struct block_device_operations block_client_fops = { + .owner = THIS_MODULE, + .open = block_client_open, +}; + +static enum neuron_block_req_type block_to_neuron_type(struct bio *bio) +{ + enum neuron_block_req_type req_type; + + switch (bio_op(bio)) { + + case REQ_OP_READ: + pr_debug("READ REQUEST\n"); + req_type = NEURON_BLOCK_REQUEST_READ; + break; + case REQ_OP_WRITE: + pr_debug("WRITE REQUEST\n"); + req_type = NEURON_BLOCK_REQUEST_WRITE; + break; + case REQ_OP_DISCARD: + pr_debug("DISCARD REQUEST\n"); + req_type = NEURON_BLOCK_REQUEST_DISCARD; + break; + case REQ_OP_SECURE_ERASE: + pr_debug("SECURE ERASE REQUEST\n"); + req_type = NEURON_BLOCK_REQUEST_SECURE_ERASE; + break; + case REQ_OP_WRITE_SAME: + pr_debug("WRITE_SAME REQUEST\n"); + req_type = NEURON_BLOCK_REQUEST_WRITE_SAME; + break; + case REQ_OP_WRITE_ZEROES: + pr_debug("WRITE_ZEROES REQUEST\n"); + req_type = NEURON_BLOCK_REQUEST_WRITE_ZEROES; + break; + case REQ_OP_FLUSH: + pr_debug("REQ_OP_FLUSH REQUEST\n"); + req_type = NEURON_BLOCK_REQUEST_READ; + break; + default: + pr_err("Request operation not found.\n"); + req_type = -EOPNOTSUPP; + break; + } + + return req_type; +} + +static struct sk_buff *bio_to_skb(struct bio *bio) +{ + struct sk_buff *head_skb = NULL; + struct sk_buff *tail_skb = NULL; + struct bio_vec bvec; + struct bvec_iter iter; + int err; + + head_skb = alloc_skb(0, GFP_KERNEL); + tail_skb = head_skb; + + bio_for_each_segment(bvec, bio, iter) { + + if (skb_shinfo(tail_skb)->nr_frags == MAX_SKB_FRAGS) { + struct sk_buff *next_skb = NULL; + + if (head_skb != tail_skb) { + head_skb->len += tail_skb->len; + head_skb->data_len += tail_skb->data_len; + head_skb->truesize += tail_skb->truesize; + } + + next_skb = alloc_skb(0, GFP_KERNEL); + + if (!skb_shinfo(head_skb)->frag_list) { + skb_shinfo(head_skb)->frag_list = next_skb; + } else { + tail_skb->next = next_skb; + next_skb->prev = tail_skb; + } + tail_skb = next_skb; + } + + err = skb_append_pagefrags(tail_skb, bvec.bv_page, + bvec.bv_offset, + bvec.bv_len); + if (err) { + pr_debug("Error appending page to skb.\n"); + return ERR_PTR(err); + } + tail_skb->len += bvec.bv_len; + tail_skb->data_len += bvec.bv_len; + tail_skb->truesize += PAGE_SIZE; + } + + + if (head_skb != tail_skb) { + head_skb->len += tail_skb->len; + head_skb->data_len += tail_skb->data_len; + head_skb->truesize += tail_skb->truesize; + } + + return head_skb; +} + +static int app_block_client_get_request(struct neuron_application *app_dev, + void **opaque_id, + enum neuron_block_req_type *req_type, + uint16_t *flags, + uint64_t *start_sector, + uint32_t *sectors, + struct sk_buff **out_skb) +{ + struct block_client_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + struct bio *bio; + bool flush_flag; + bool commit_flag; + bool sync_flag; + + spin_lock_irq(&blk_dev->list_lock); + if (!bio_list_size(&blk_dev->bio_list)) { + spin_unlock_irq(&blk_dev->list_lock); + return -EAGAIN; + } + + bio = bio_list_pop(&blk_dev->bio_list); + spin_unlock_irq(&blk_dev->list_lock); + + *opaque_id = bio; + *start_sector = bio->bi_iter.bi_sector; + *sectors = bio_sectors(bio); + flush_flag = (bio_flags(bio) & REQ_PREFLUSH); + + commit_flag = (bio_flags(bio) & REQ_FUA); + sync_flag = (bio_flags(bio) & REQ_SYNC); + + *flags = (flush_flag ? (1 << __NEURON_BLOCK_REQ_PREFLUSH) : 0) | + (commit_flag ? (1 << __NEURON_BLOCK_REQ_FUA) : 0) | + (sync_flag ? (1 << __NEURON_BLOCK_REQ_SYNC) : 0); + + *req_type = block_to_neuron_type(bio); + if (*req_type < 0) + goto failed_req; + + if ((*req_type == NEURON_BLOCK_REQUEST_DISCARD) || + (*req_type == NEURON_BLOCK_REQUEST_SECURE_ERASE) || + (*req_type == NEURON_BLOCK_REQUEST_WRITE_ZEROES) || + (*sectors == 0)) { + *out_skb = NULL; + } else { + *out_skb = bio_to_skb(bio); + if (IS_ERR(*out_skb)) + goto failed_req; + } + + pr_debug("vcnt: %d\n", bio->bi_vcnt); + pr_debug("flags: %d\n", *flags); + pr_debug("start_sector: %lld\n", (long long)*start_sector); + pr_debug("sectors: %d\n", *sectors); + + return 0; + +failed_req: + bio_io_error(bio); + return -EAGAIN; +} + +static blk_status_t neuron_to_block_status(enum neuron_block_resp_status status) +{ + blk_status_t ret; + + switch (status) { + case BLOCK_RESP_SUCCESS: + ret = BLK_STS_OK; + break; + case BLOCK_RESP_TIMEOUT: + ret = BLK_STS_TIMEOUT; + break; + case BLOCK_RESP_NOMEM: + ret = BLK_STS_RESOURCE; + break; + case BLOCK_RESP_OPNOTSUPP: + ret = BLK_STS_NOTSUPP; + break; + case BLOCK_RESP_IOERROR: + default: + ret = BLK_STS_IOERR; + break; + } + + return ret; +} + +static int app_block_client_do_response(struct neuron_application *app_dev, + void *opaque_id, + enum neuron_block_resp_status status) +{ + struct bio *bio = opaque_id; + + bio->bi_status = neuron_to_block_status(status); + bio_endio(bio); + + if (status) + pr_err("Request completed with errors.\n"); + + return 0; +} + +static char *bin_to_uuid(char *dst, const void *src, size_t count) +{ + const unsigned char *_src = src; + + while (count--) { + dst = hex_byte_pack(dst, *_src++); + if (count == 12 || count == 10 || count == 8 || count == 6) { + *dst = '-'; + dst++; + } + } + + return dst; +} + +static void clean_blk_dev_obj(struct kref *kref) +{ + struct block_client_dev *blk_dev = + container_of(kref, struct block_client_dev, + kref); + + ida_simple_remove(&ida, blk_dev->id); + ida_destroy(&ida); + del_gendisk(blk_dev->gd); + if (blk_dev->queue) + blk_cleanup_queue(blk_dev->queue); + put_disk(blk_dev->gd); + kfree(blk_dev); +} + +static void app_block_client_add_disk_work(struct work_struct *work) +{ + struct block_client_dev *blk_dev = + container_of(work, struct block_client_dev, + add_disk_work); + + device_add_disk(blk_dev->dev, blk_dev->gd, NULL); + kref_put(&blk_dev->kref, clean_blk_dev_obj); +} + +static int app_block_client_do_set_bd_params(struct neuron_application *app_dev, + struct neuron_block_param *param) +{ + struct block_client_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + int ret = 0; + + blk_dev->queue = blk_alloc_queue(GFP_KERNEL); + if (!blk_dev->queue) { + pr_err("Queue allocation failed.\n"); + ret = -ENOMEM; + goto fail_init_queue; + } + + blk_queue_make_request(blk_dev->queue, block_client_make_request); + + blk_queue_logical_block_size(blk_dev->queue, param->logical_block_size); + blk_queue_physical_block_size(blk_dev->queue, + param->physical_block_size); + blk_dev->sector_size = param->logical_block_size; + blk_queue_alignment_offset(blk_dev->queue, param->alignment_offset); + + blk_dev->queue->limits.max_discard_sectors = + param->discard_max_hw_sectors; + blk_dev->queue->limits.max_hw_discard_sectors = + param->discard_max_sectors; + blk_dev->queue->limits.discard_granularity = param->discard_granularity; + + blk_dev->queue->queuedata = blk_dev; + blk_dev->read_only = param->read_only; + + blk_queue_write_cache(blk_dev->queue, param->wc_flag, param->fua_flag); + + if (param->discard_max_hw_sectors > 0) + blk_queue_flag_set(QUEUE_FLAG_DISCARD, blk_dev->queue); + + /* paravirt device. non-rotational device (SSD). */ + blk_queue_flag_set(QUEUE_FLAG_VIRT, blk_dev->queue); + + blk_dev->gd = alloc_disk(DISK_MAX_PARTS); + if (!blk_dev->gd) { + pr_err("Gendisk allocation failed.\n"); + return -ENOMEM; + } + + set_disk_ro(blk_dev->gd, blk_dev->read_only); + blk_dev->gd->major = block_client_major_nr; + + blk_dev->id = ida_simple_get(&ida, 0, 0, GFP_KERNEL); + if (blk_dev->id < 0) { + pr_err("Error get a new id.\n"); + return blk_dev->id; + } + + blk_dev->gd->first_minor = blk_dev->id * DISK_MAX_PARTS; + blk_dev->gd->fops = &block_client_fops; + blk_dev->gd->queue = blk_dev->queue; + blk_dev->gd->flags |= GENHD_FL_EXT_DEVT; /* allow extended devt */ + blk_dev->gd->private_data = blk_dev; + snprintf(blk_dev->gd->disk_name, PAGE_SIZE - 1, "%s%d", DISK_NAME, + blk_dev->id); + set_capacity(blk_dev->gd, param->num_device_sectors); + + blk_dev->gd->part0.info = + kzalloc(sizeof(struct partition_meta_info), + GFP_KERNEL); + if (!blk_dev->gd->part0.info) + return -ENOMEM; + + bin_to_uuid(blk_dev->gd->part0.info->uuid, + param->uuid.b, + sizeof(param->uuid.b)); + + memcpy(blk_dev->gd->part0.info->volname, param->label, + sizeof(u8)*PARTITION_META_INFO_VOLNAMELTH); + + INIT_WORK(&blk_dev->add_disk_work, app_block_client_add_disk_work); + kref_init(&blk_dev->kref); + + kref_get(&blk_dev->kref); + schedule_work(&blk_dev->add_disk_work); + + kfree(param); + + return 0; + +fail_init_queue: + put_disk(blk_dev->gd); + kfree(param); + return ret; +} + +static int app_block_client_probe(struct neuron_application *app_dev) +{ + struct block_client_dev *blk_dev; + + blk_dev = kzalloc(sizeof(*blk_dev), GFP_KERNEL); + if (!blk_dev) + return -ENOMEM; + + blk_dev->dev = &app_dev->dev; + bio_list_init(&blk_dev->bio_list); + spin_lock_init(&blk_dev->list_lock); + ida_init(&ida); + + dev_set_drvdata(&app_dev->dev, blk_dev); + + return 0; +} + +static void app_block_client_remove(struct neuron_application *app_dev) +{ + struct block_client_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + + flush_work(&blk_dev->add_disk_work); + kref_put(&blk_dev->kref, clean_blk_dev_obj); +} + +static struct neuron_block_app_client_driver app_block_client_drv = { + .base = { + .driver = { + .name = DRIVER_NAME, + .owner = THIS_MODULE, + .of_match_table = app_block_client_match, + }, + .protocol_driver = &protocol_client_block_driver, + .probe = app_block_client_probe, + .remove = app_block_client_remove, + }, + .get_request = app_block_client_get_request, + .do_response = app_block_client_do_response, + .do_set_bd_params = app_block_client_do_set_bd_params, +}; + +static int __init block_client_init(void) +{ + int ret = 0; + + ret = neuron_register_app_driver(&app_block_client_drv.base); + if (ret < 0) { + pr_err("Failed to register driver\n"); + return ret; + } + + block_client_major_nr = register_blkdev(0, BLOCK_NAME); + if (block_client_major_nr < 0) { + pr_err("Major number registration failed.\n"); + return block_client_major_nr; + } + + return 0; +} + +static void block_client_exit(void) +{ + unregister_blkdev(block_client_major_nr, BLOCK_NAME); + neuron_unregister_app_driver(&app_block_client_drv.base); +} + +module_init(block_client_init); +module_exit(block_client_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Neuron block client module"); diff --git a/drivers/block/neuron_block_server.c b/drivers/block/neuron_block_server.c new file mode 100644 index 000000000000..e61f1d52bfae --- /dev/null +++ b/drivers/block/neuron_block_server.c @@ -0,0 +1,628 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define DRIVER_NAME "neuron-application-block-server" +#define BLOCK_NAME "neuron_block" + +struct bio_priv { + struct neuron_application *app_dev; + uint32_t id; + struct sk_buff *skb; +}; + +struct block_server_dev { + struct request_queue *queue; + struct gendisk *gd; + char *bdev_name; + struct block_device *bdev; + uint32_t sector_size; + struct bio_list bio_list; + spinlock_t list_lock; +}; + +static const struct of_device_id app_block_server_match[] = { + { + .compatible = "qcom,neuron-block-server", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, app_block_server_match); + +static int get_blk_dev(struct neuron_application *app_dev); + +static ssize_t blk_name_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(dev); + + return scnprintf(buf, PAGE_SIZE - 1, "%s\n", blk_dev->bdev_name); +} + +static ssize_t blk_name_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(dev); + struct neuron_application *app_dev = to_neuron_application(dev); + char *input; + char *cp; + + if (blk_dev->bdev_name) { + pr_err("No permission to write\n"); + return count; + } + + if (count >= (PAGE_SIZE - 1)) + return -EINVAL; + + input = kstrndup(buf, count, GFP_KERNEL); + if (!input) + return -ENOMEM; + + cp = strnchr(input, count, '\n'); + if (cp) + *cp = '\0'; + + if (strlen(input)) { + blk_dev->bdev_name = input; + } else { + kfree(input); + blk_dev->bdev_name = NULL; + } + + if (get_blk_dev(app_dev)) { + pr_err("Error getting block device %s\n", blk_dev->bdev_name); + blk_dev->bdev_name = NULL; + } + + return count; +} +static DEVICE_ATTR_RW(blk_name); + +static enum neuron_block_resp_status block_to_neuron_status(struct bio *bio) +{ + enum neuron_block_resp_status status; + + switch (bio->bi_status) { + + case BLK_STS_OK: + status = BLOCK_RESP_SUCCESS; + break; + case BLK_STS_TIMEOUT: + status = BLOCK_RESP_TIMEOUT; + break; + case BLK_STS_RESOURCE: + status = BLOCK_RESP_NOMEM; + break; + case BLK_STS_NOTSUPP: + status = BLOCK_RESP_OPNOTSUPP; + break; + case BLK_STS_IOERR: + default: + status = BLOCK_RESP_IOERROR; + break; + } + + return status; +} + +static int app_blk_server_get_response(struct neuron_application *app_dev, + uint32_t *id, + enum neuron_block_resp_status *status, + struct sk_buff **skb) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + struct bio_priv *bio_priv; + struct bio *bio; + + spin_lock_irq(&blk_dev->list_lock); + if (bio_list_empty(&blk_dev->bio_list)) { + pr_debug("Get response EAGAIN\n"); + spin_unlock_irq(&blk_dev->list_lock); + return -EAGAIN; + } + + bio = bio_list_pop(&blk_dev->bio_list); + spin_unlock_irq(&blk_dev->list_lock); + + *status = block_to_neuron_status(bio); + + bio_priv = bio->bi_private; + *id = bio_priv->id; + + if (bio_priv->skb == NULL) { + *skb = NULL; + } else if (bio_op(bio) != REQ_OP_READ) { + consume_skb(bio_priv->skb); + *skb = NULL; + } else { + *skb = bio_priv->skb; + } + + bio_put(bio); + kfree(bio_priv); + + return 0; +} + +static void app_blk_server_request_done(struct bio *bio) +{ + struct bio_priv *bio_priv = bio->bi_private; + struct neuron_application *app_dev = bio_priv->app_dev; + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + unsigned long flags; + + spin_lock_irqsave(&blk_dev->list_lock, flags); + bio_list_add(&blk_dev->bio_list, bio); + spin_unlock_irqrestore(&blk_dev->list_lock, flags); + + neuron_app_wakeup(app_dev, NEURON_BLOCK_SERVER_EVENT_RESPONSE); +} + +static int add_bio_pages_from_frags(struct sk_buff *skb, struct bio *bio) +{ + unsigned int bytes; + struct page *page; + uint32_t offset; + int result; + int i; + + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + + bytes = skb_frag_size(frag); + page = skb_frag_page(frag); + offset = skb_frag_off(frag); + + result = bio_add_page(bio, page, bytes, offset); + if (result < bytes) { + pr_err("Error adding page to bio. result:%d\n", result); + kfree_skb(skb); + bio_io_error(bio); + return -EIO; + } + } + + return 0; +} + +static int app_blk_server_prepare_bio(struct bio **bio, + struct neuron_application *app_dev, + unsigned int op, + uint32_t req_id, + uint64_t start, + uint32_t nr_iovecs, + uint16_t flags, + struct sk_buff *skb) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + struct bio_priv *bio_priv; + bool flush_flag; + bool commit_flag; + bool sync_flag; + unsigned int op_flags; + + bio_priv = kzalloc(sizeof(struct bio_priv), GFP_KERNEL); + if (!bio_priv) + return -ENOMEM; + + *bio = bio_alloc(GFP_KERNEL, nr_iovecs); + if (!*bio) + return -ENOMEM; + + bio_priv->app_dev = app_dev; + bio_priv->id = req_id; + bio_priv->skb = skb; + (*bio)->bi_private = bio_priv; + + bio_set_dev(*bio, blk_dev->bdev); + (*bio)->bi_iter.bi_sector = start; + (*bio)->bi_end_io = app_blk_server_request_done; + + flush_flag = (flags & NEURON_BLOCK_REQ_PREFLUSH); + commit_flag = (flags & NEURON_BLOCK_REQ_FUA); + sync_flag = (flags & NEURON_BLOCK_REQ_SYNC); + + op_flags = (flush_flag ? REQ_PREFLUSH : 0) | + (commit_flag ? REQ_FUA : 0) | + (sync_flag ? REQ_SYNC : 0); + + bio_set_op_attrs(*bio, op, op_flags); + + if (skb) { + struct sk_buff *iter; + + add_bio_pages_from_frags(skb, *bio); + + skb_walk_frags(skb, iter) + add_bio_pages_from_frags(iter, *bio); + } + //blk_recount_segments(bdev_get_queue(blk_dev->bdev), *bio); + + return 0; +} + +static int app_blk_server_do_read(struct neuron_application *app_dev, + uint32_t req_id, + uint64_t start, + uint32_t num, + uint16_t flags) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + struct bio *bio; + int size; + uint32_t nr_iovecs; + int ret; + struct sk_buff *skb = NULL; + + size = num * blk_dev->sector_size; + if (num) + nr_iovecs = 1 + ((size - 1) >> PAGE_SHIFT); + else + nr_iovecs = 0; + + if (num) + skb = neuron_alloc_pskb(size, GFP_KERNEL); + + ret = app_blk_server_prepare_bio(&bio, app_dev, REQ_OP_READ, req_id, + start, nr_iovecs, flags, skb); + if (ret) + return ret; + + generic_make_request(bio); + + return 0; +} + +static int app_blk_server_do_write(struct neuron_application *app_dev, + uint32_t req_id, + uint64_t start, + uint32_t num, + uint16_t flags, + struct sk_buff *skb) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + struct bio *bio; + int size; + int nr_iovecs; + int ret; + + size = num * blk_dev->sector_size; + if (num) + nr_iovecs = 1 + ((size - 1) >> PAGE_SHIFT); + else + nr_iovecs = 0; + + ret = app_blk_server_prepare_bio(&bio, app_dev, REQ_OP_WRITE, req_id, + start, nr_iovecs, flags, skb); + if (ret) + return ret; + + generic_make_request(bio); + + return 0; +} + +static int app_blk_server_do_discard(struct neuron_application *app_dev, + uint32_t req_id, + uint64_t start, + uint32_t num, + uint16_t flags) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + uint32_t size; + struct bio *bio; + int ret; + + size = num * blk_dev->sector_size; + + ret = app_blk_server_prepare_bio(&bio, app_dev, REQ_OP_DISCARD, req_id, + start, 0, flags, NULL); + if (ret) + return ret; + + generic_make_request(bio); + + return 0; +} + +static int app_blk_server_do_secure_erase(struct neuron_application *app_dev, + uint32_t req_id, + uint64_t start, + uint32_t num, + uint16_t flags) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + uint32_t size; + struct bio *bio; + int ret; + + size = num * blk_dev->sector_size; + + ret = app_blk_server_prepare_bio(&bio, app_dev, REQ_OP_SECURE_ERASE, + req_id, start, 0, flags, NULL); + if (ret) + return ret; + + generic_make_request(bio); + + return 0; +} + +static int app_blk_server_do_write_same(struct neuron_application *app_dev, + uint32_t req_id, + uint64_t start, + uint32_t num, + uint16_t flags, + struct sk_buff *skb) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + struct bio *bio; + int size; + int ret; + + size = num * blk_dev->sector_size; + + ret = app_blk_server_prepare_bio(&bio, app_dev, REQ_OP_WRITE_SAME, + req_id, start, 1, flags, skb); + if (ret) + return ret; + + generic_make_request(bio); + + return 0; +} + +static int app_blk_server_do_write_zeroes(struct neuron_application *app_dev, + uint32_t req_id, + uint64_t start, + uint32_t num, + uint16_t flags) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + struct bio *bio; + int size; + int ret; + + size = num * blk_dev->sector_size; + + ret = app_blk_server_prepare_bio(&bio, app_dev, REQ_OP_WRITE_ZEROES, + req_id, start, 0, flags, NULL); + if (ret) + return ret; + generic_make_request(bio); + + return 0; +} + +static int match_dev_name(struct device *dev, const void *name) +{ + if (!dev_name(dev)) + return 0; + return !strcmp(dev_name(dev), (char *)name); +} + +static int app_blk_server_get_bd_params(struct neuron_application *app_dev, + const struct neuron_block_param **param) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + struct neuron_block_param *params; + struct queue_limits limits; + int err; + const char *label_prop; + const char *out_values; + + if (!blk_dev->bdev) + return -EAGAIN; + + limits = bdev_get_queue(blk_dev->bdev)->limits; + + params = kzalloc(sizeof(struct neuron_block_param) + + sizeof(u8)*PARTITION_META_INFO_VOLNAMELTH, + GFP_KERNEL); + if (!params) + return -ENOMEM; + + params->logical_block_size = bdev_logical_block_size(blk_dev->bdev); + params->physical_block_size = bdev_physical_block_size(blk_dev->bdev); + blk_dev->sector_size = bdev_logical_block_size(blk_dev->bdev); + + params->alignment_offset = blk_dev->bdev->bd_part->alignment_offset; + params->read_only = bdev_read_only(blk_dev->bdev); + params->num_device_sectors = blk_dev->bdev->bd_part->nr_sects; + + params->discard_max_sectors = limits.max_discard_sectors; + params->discard_max_hw_sectors = limits.max_hw_discard_sectors; + params->discard_granularity = limits.discard_granularity; + + params->wc_flag = test_bit(QUEUE_FLAG_WC, + &bdev_get_queue(blk_dev->bdev)->queue_flags); + params->fua_flag = test_bit(QUEUE_FLAG_FUA, + &bdev_get_queue(blk_dev->bdev)->queue_flags); + + err = of_property_read_string(app_dev->dev.of_node, "label", + &label_prop); + if (!err) { + err = strscpy(params->label, label_prop, + PARTITION_META_INFO_VOLNAMELTH); + } else { + if (blk_dev->bdev->bd_disk->part0.info && + strlen(blk_dev->bdev->bd_disk->part0.info->volname)) { + memcpy(params->label, + blk_dev->bdev->bd_disk->part0.info->volname, + sizeof(u8)*PARTITION_META_INFO_VOLNAMELTH); + } else { + memcpy(params->label, blk_dev->bdev_name, + strlen(blk_dev->bdev_name)); + } + } + + err = of_property_read_string(app_dev->dev.of_node, "uuid-string", + &out_values); + if (err < 0) { + if (blk_dev->bdev->bd_disk->part0.info) { + const char *uuid; + + uuid = blk_dev->bdev->bd_disk->part0.info->uuid; + err = uuid_parse(uuid, ¶ms->uuid); + if (err) + pr_err("Invalid uuid.\n"); + } else + uuid_copy(¶ms->uuid, &uuid_null); + } else { + err = uuid_parse(out_values, ¶ms->uuid); + if (err) + pr_err("Invalid uuid.\n"); + } + + *param = params; + + return 0; +} + +static int get_blk_dev(struct neuron_application *app_dev) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + struct device *dev; + + dev = class_find_device(&block_class, NULL, + (const void *)blk_dev->bdev_name, + match_dev_name); + if (!dev) { + pr_err("Device with name %s not found.\n", blk_dev->bdev_name); + return -ENODEV; + } + + blk_dev->bdev = blkdev_get_by_dev(dev->devt, + FMODE_READ|FMODE_WRITE, NULL); + if (IS_ERR(blk_dev->bdev)) { + pr_err("Getting block device %s by device number failed.\n", + blk_dev->bdev_name); + return PTR_ERR(blk_dev->bdev); + } + + neuron_app_wakeup(app_dev, NEURON_BLOCK_SERVER_EVENT_BD_PARAMS); + + return 0; +} + +static int app_block_server_probe(struct neuron_application *app_dev) +{ + struct block_server_dev *blk_dev; + int err; + int ret; + const char *name_prop; + + blk_dev = kzalloc(sizeof(*blk_dev), GFP_KERNEL); + if (!blk_dev) + return -ENOMEM; + + bio_list_init(&blk_dev->bio_list); + spin_lock_init(&blk_dev->list_lock); + + dev_set_drvdata(&app_dev->dev, blk_dev); + + ret = device_create_file(&app_dev->dev, &dev_attr_blk_name); + if (ret) { + pr_err("Sysfs creation failed with error: %d\n", ret); + goto fail; + } + + err = of_property_read_string(app_dev->dev.of_node, "device-name", + &name_prop); + if (!err) { + + blk_dev->bdev_name = kzalloc(strlen(name_prop) + 1, + GFP_KERNEL); + if (!blk_dev->bdev_name) { + ret = -ENOMEM; + goto fail; + } + + err = strscpy(blk_dev->bdev_name, name_prop, + strlen(name_prop) + 1); + } else { + pr_err("No device name found in device tree.\n"); + return 0; + } + + if (get_blk_dev(app_dev)) { + ret = -EPROBE_DEFER; + goto fail; + } + + return 0; +fail: + device_remove_file(&app_dev->dev, &dev_attr_blk_name); + dev_set_drvdata(&app_dev->dev, NULL); + kfree(blk_dev); + return ret; +} + +static void app_block_server_remove(struct neuron_application *app_dev) +{ + struct block_server_dev *blk_dev = dev_get_drvdata(&app_dev->dev); + + device_remove_file(&app_dev->dev, &dev_attr_blk_name); + dev_set_drvdata(&app_dev->dev, NULL); + kfree(blk_dev); +} + +static struct neuron_block_app_server_driver app_block_server_drv = { + .base = { + .driver = { + .name = DRIVER_NAME, + .owner = THIS_MODULE, + .of_match_table = app_block_server_match, + }, + .protocol_driver = &protocol_server_block_driver, + .probe = app_block_server_probe, + .remove = app_block_server_remove, + }, + .get_bd_params = app_blk_server_get_bd_params, + .get_response = app_blk_server_get_response, + .do_read = app_blk_server_do_read, + .do_write = app_blk_server_do_write, + .do_discard = app_blk_server_do_discard, + .do_secure_erase = app_blk_server_do_secure_erase, + .do_write_same = app_blk_server_do_write_same, + .do_write_zeroes = app_blk_server_do_write_zeroes, +}; + +static int __init block_server_init(void) +{ + int ret = 0; + + ret = neuron_register_app_driver(&app_block_server_drv.base); + if (ret < 0) { + pr_err("Failed to register driver\n"); + return ret; + } + + return 0; +} + +static void block_server_exit(void) +{ + neuron_unregister_app_driver(&app_block_server_drv.base); +} + +module_init(block_server_init); +module_exit(block_server_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Neuron block server module"); diff --git a/include/linux/neuron.h b/include/linux/neuron.h new file mode 100644 index 000000000000..00df21de3aa1 --- /dev/null +++ b/include/linux/neuron.h @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020 The Linux Foundation. All rights reserved. + */ + +#ifndef __NEURON_H__ +#define __NEURON_H__ + +#include +#include +#include + +/* + * Communication channels + * + * These are link-layer devices which abstract the details of inter-VM + * communication mechanisms away from the upper layers. + */ + +enum neuron_channel_type { + NEURON_CHANNEL_MESSAGE_QUEUE = 1, + NEURON_CHANNEL_NOTIFICATION, + NEURON_CHANNEL_SHARED_MEMORY +}; + +enum neuron_channel_direction { + NEURON_CHANNEL_SEND = (1 << 0), + NEURON_CHANNEL_RECEIVE = (1 << 1), + NEURON_CHANNEL_BIDIRECTIONAL = NEURON_CHANNEL_SEND | + NEURON_CHANNEL_RECEIVE +}; + +struct buffer_list { + struct sk_buff *head; + off_t offset; + size_t size; +}; + +struct neuron_channel { + enum neuron_channel_type type; + enum neuron_channel_direction direction; + + /* For message queue channels, the maximum guaranteed message size + * and the minimum guaranteed message queue length. These may be + * zero until handshaking with the peer has completed; in this case, + * the channel driver will call the wakeup callback after they have + * been set. + * + * Note that it may be transiently possible to exceed these limits; + * they are merely the lower bounds guaranteed by the driver. + */ + size_t max_size; + unsigned int queue_length; + + struct device dev; + struct neuron_protocol *protocol; + unsigned int id; + + /* Writes protected by the protocol device lock */ + struct neuron_protocol_driver __rcu *protocol_drv; +}; + +#define to_neuron_channel(drv) container_of(drv, struct neuron_channel, dev) + +struct neuron_channel_driver { + enum neuron_channel_type type; + enum neuron_channel_direction direction; + struct device_driver driver; + + int (*probe)(struct neuron_channel *channel_dev); + void (*remove)(struct neuron_channel *channel_dev); + + /* Message queue send callback. + * @skb sk_buff pointer for sending. + * @return 0 for success, others for failure. + */ + int (*send_msg)(struct neuron_channel *channel_dev, + struct sk_buff *skb); + /* Message queue send callback. + * @buf buffer_list object, which contains the offset of the sk buffer + * and size to send. + * @return 0 for success, others for failure. + */ + int (*send_msgv)(struct neuron_channel *channel_dev, + struct buffer_list buf); + /* Message queue receive callbacks + * The caller is responsible for allocating and freeing receiving buffer + * @skb sk_buff pointer for receiving. + * @return positive number for received data length, negative number for + * failure. Never return 0. + */ + ssize_t (*receive_msg)(struct neuron_channel *channel_dev, + struct sk_buff *skb); + /* Message queue send callback. + * @buf buffer_list object, which contains the offset of the sk buffer + * to start taking the received data. + * @return positive number for received data length, negative number for + * failure. Never return 0. + */ + ssize_t (*receive_msgv)(struct neuron_channel *channel_dev, + struct buffer_list buf); + + /* Notification callbacks */ + int (*send_notify)(struct neuron_channel *channel_dev, uint32_t bits); + uint32_t (*receive_notify)(struct neuron_channel *channel_dev); +}; + +#define to_neuron_channel_driver(drv) container_of(drv, \ + struct neuron_channel_driver, driver) + +struct neuron_channel *neuron_channel_add(struct device_node *node, + struct device *parent); +int neuron_register_channel_driver(struct neuron_channel_driver *drv); +void neuron_unregister_channel_driver(struct neuron_channel_driver *drv); + + +/* + * Protocol drivers (typically autogenerated) + * + * These drivers translate between messages that are sent over the + * communication channels and high-level interfaces that are used by the + * application layers. + * + * Each driver has its own set of callbacks that communicate with compatible + * application drivers, and expects a particular set of channel devices. + * These will typically be defined by enclosing struct neuron_protocol_driver + * in a protocol-specific structure which the application driver accesses. + */ + +struct neuron_protocol { + struct device dev; + struct neuron_application *application; + + unsigned int process_count; + const char **processes; + + unsigned int channel_count; + struct neuron_channel *channels[]; +}; + +#define to_neuron_protocol(drv) container_of(drv, struct neuron_protocol, dev) + +struct neuron_channel_match_table { + enum neuron_channel_type type; + enum neuron_channel_direction direction; +}; + +struct neuron_protocol_driver { + unsigned int channel_count; + const struct neuron_channel_match_table *channels; + + unsigned int process_count; + const char *const *processes; + + struct device_driver driver; + + int (*probe)(struct neuron_protocol *protocol_dev); + void (*remove)(struct neuron_protocol *protocol_dev); + + int (*channel_wakeup)(struct neuron_protocol *protocol, + unsigned int id); + int (*app_wakeup)(struct neuron_protocol *dev, unsigned int ev); +}; + +#define to_neuron_protocol_driver(drv) container_of(drv, \ + struct neuron_protocol_driver, driver) + +struct neuron_protocol *neuron_protocol_add(struct device_node *node, + unsigned int channel_count, struct neuron_channel **channels, + struct device *parent, struct neuron_application *app_dev); +int neuron_register_protocol_driver(struct neuron_protocol_driver *drv); +void neuron_unregister_protocol_driver(struct neuron_protocol_driver *drv); + +/** + * neuron_channel_wakeup() - tell a protocol that a channel is ready + * + * This function should be called by the channel driver when its channel first + * becomes fully initialised, and also when the channel becomes ready to send + * or receive data. It will call a method provided by the protocol driver + * which will typically wake up a wait queue or schedule a tasklet to process + * the data. The wakeup method will not block. + * + * For message queue channels, this is triggered: + * - after the channel's maximum message size and queue length are known and + * handshaking with the peer has completed; + * - when a send side channel that was previously full is no longer full; and + * - when a receive side channel that was previously empty is no longer empty. + * + * For notification channels, this is triggered when a receive side channel + * may have received a notification from its remote partner. It is not used on + * send side notification channels. + * + * This is unused for shared-memory channels. + */ +static inline int neuron_channel_wakeup(struct neuron_channel *channel) +{ + struct neuron_protocol_driver *protocol_drv; + int ret = -ECONNRESET; + + rcu_read_lock(); + + protocol_drv = rcu_dereference(channel->protocol_drv); + if (protocol_drv != NULL) + if (protocol_drv->channel_wakeup) + ret = protocol_drv->channel_wakeup(channel->protocol, + channel->id); + + rcu_read_unlock(); + + return ret; +} + +/* + * Application drivers + * + * These drivers contain hand-written glue between the high-level API provided + * by a protocol driver, and the guest kernel's internal interfaces. + */ + +struct neuron_application { + const char *type; + struct device dev; + struct neuron_protocol *protocol; + + /* Writes protected by the protocol device lock */ + struct neuron_protocol_driver __rcu *protocol_drv; +}; + +#define to_neuron_application(drv) container_of(drv, \ + struct neuron_application, dev) + +struct neuron_app_driver { + struct device_driver driver; + + const struct neuron_protocol_driver *protocol_driver; + + int (*probe)(struct neuron_application *dev); + void (*remove)(struct neuron_application *dev); + void (*start)(struct neuron_application *dev); +}; + +#define to_neuron_app_driver(drv) container_of(drv, \ + struct neuron_app_driver, driver) + +struct neuron_application *neuron_app_add(struct device_node *node, + struct device *parent); +int neuron_register_app_driver(struct neuron_app_driver *drv); +void neuron_unregister_app_driver(struct neuron_app_driver *drv); + +/** + * neuron_app_wakeup() - tell a protocol that the application is ready + * + * This function should be called by the application driver when there is a + * wakeup that needs to be sent to the protocol driver. + * + */ +static inline int neuron_app_wakeup(struct neuron_application *application, + unsigned int ev) +{ + struct neuron_protocol_driver *protocol_drv; + int ret = -ECONNRESET; + + rcu_read_lock(); + + protocol_drv = rcu_dereference(application->protocol_drv); + if (protocol_drv != NULL) + if (protocol_drv->app_wakeup) + ret = protocol_drv->app_wakeup(application->protocol, + ev); + + rcu_read_unlock(); + + return ret; +} + +/** + * Allocate sk_buff with pages as many as you want. + */ +static inline struct sk_buff *neuron_alloc_pskb(size_t data_len, gfp_t gfp) +{ + struct sk_buff *head_skb = NULL; + struct sk_buff *second_skb = NULL; + struct sk_buff *new_frag, *prev; + int ret; + + do { + size_t frag_len = min_t(size_t, data_len, + MAX_SKB_FRAGS << PAGE_SHIFT); + + new_frag = alloc_skb_with_frags(0, frag_len, + 0, &ret, gfp); + if (!new_frag) { + if (head_skb) + kfree_skb(head_skb); + return ERR_PTR(ret); + } + new_frag->data_len = frag_len; + new_frag->len = frag_len; + + if (!head_skb) { + head_skb = new_frag; + } else { + if (!second_skb) { + skb_shinfo(head_skb)->frag_list = new_frag; + second_skb = new_frag; + } else { + prev->next = new_frag; + } + prev = new_frag; + + head_skb->len += new_frag->len; + head_skb->data_len += new_frag->data_len; + head_skb->truesize += new_frag->truesize; + } + + data_len -= frag_len; + } while (data_len); + + return head_skb; +} + +#endif /* __LINUX_NEURON_H */ diff --git a/include/linux/neuron_block.h b/include/linux/neuron_block.h new file mode 100644 index 000000000000..b403613f73d6 --- /dev/null +++ b/include/linux/neuron_block.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020 The Linux Foundation. All rights reserved. + */ + +#include +#include + +/* Block I/O request type */ +enum neuron_block_req_type { + NEURON_BLOCK_REQUEST_READ = 0, + NEURON_BLOCK_REQUEST_WRITE, + NEURON_BLOCK_REQUEST_DISCARD, + NEURON_BLOCK_REQUEST_SECURE_ERASE, + NEURON_BLOCK_REQUEST_WRITE_SAME, + NEURON_BLOCK_REQUEST_WRITE_ZEROES +}; + +/* Block I/O request flags bit position */ +enum { + __NEURON_BLOCK_REQ_PREFLUSH = 0, + __NEURON_BLOCK_REQ_FUA, + __NEURON_BLOCK_REQ_SYNC +}; + +/* Block I/O request flags */ +#define NEURON_BLOCK_REQ_PREFLUSH (1U << __NEURON_BLOCK_REQ_PREFLUSH) +#define NEURON_BLOCK_REQ_FUA (1U << __NEURON_BLOCK_REQ_FUA) +#define NEURON_BLOCK_REQ_SYNC (1U << __NEURON_BLOCK_REQ_SYNC) + +/* Block I/O response status */ +enum neuron_block_resp_status { + BLOCK_RESP_SUCCESS = 0, + BLOCK_RESP_TIMEOUT, + BLOCK_RESP_IOERROR, + BLOCK_RESP_INVAL, + BLOCK_RESP_ROFS, + BLOCK_RESP_NOMEM, + BLOCK_RESP_NODEV, + BLOCK_RESP_OPNOTSUPP +}; + +/* Block device params */ +struct neuron_block_param { + u32 logical_block_size; + u32 physical_block_size; + u64 num_device_sectors; + u64 discard_max_hw_sectors; + u64 discard_max_sectors; + u32 discard_granularity; + u16 alignment_offset; + bool read_only; + bool discard_zeroes_data; + bool wc_flag; + bool fua_flag; + uuid_t uuid; + u8 label[]; +}; + +enum neuron_protocol_block_client_event { + NEURON_BLOCK_CLIENT_EVENT_REQUEST = 0, + NEURON_BLOCK_CLIENT_EVENT__COUNT, +}; + +struct neuron_block_app_client_driver { + struct neuron_app_driver base; + + /* Called by client protocol driver to read the I/O request. + * @param opaque_id An opaque ID to associate with this request + * @param skb A pointer to a generated skb, which contains pages shared + * with request bio. + * @return 0 for success, others for failure + */ + int (*get_request)(struct neuron_application *dev, + void **opaque_id, + enum neuron_block_req_type *req_type, + u16 *flags, + u64 *start_sector, + u32 *sectors, + struct sk_buff **skb); + + /* Called by client protocol driver when it receives + * block parameter from the server. + */ + int (*do_set_bd_params)(struct neuron_application *dev, + struct neuron_block_param *param); + + /* Called by client protocol driver when it receives i/o + * resopnse. + * @param opaque_id The one retrieved from read_request. + */ + int (*do_response)(struct neuron_application *dev, + void *opaque_id, + enum neuron_block_resp_status status); +}; + +enum neuron_protocol_block_server_event { + NEURON_BLOCK_SERVER_EVENT_BD_PARAMS = 0, + NEURON_BLOCK_SERVER_EVENT_RESPONSE, + NEURON_BLOCK_SERVER_EVENT__COUNT, +}; + +struct neuron_block_app_server_driver { + struct neuron_app_driver base; + // Called by server protocol driver to get block params + int (*get_bd_params)(struct neuron_application *dev, + const struct neuron_block_param **param); + + /* Called by server protocol driver to read the I/O response. + * @param id An opaque ID to associate with this response + * @param status A pointer to status result + * @param skb A pointer to a generated skb pointer + * @return 0 for success, others for failure + */ + int (*get_response)(struct neuron_application *dev, u32 *id, + enum neuron_block_resp_status *status, + struct sk_buff **skb); + + /* These APIs are called by server protocol for + * the received request. + */ + int (*do_read)(struct neuron_application *dev, u32 req_id, u64 start, + u32 num, u16 flags); + int (*do_write)(struct neuron_application *dev, u32 req_id, u64 start, + u32 num, u16 flags, struct sk_buff *skb); + int (*do_discard)(struct neuron_application *dev, u32 req_id, + u64 start, u32 num, u16 flags); + int (*do_secure_erase)(struct neuron_application *dev, u32 req_id, + u64 start, u32 num, u16 flags); + int (*do_write_same)(struct neuron_application *dev, u32 req_id, + u64 start, u32 num, u16 flags, + struct sk_buff *skb); + int (*do_write_zeroes)(struct neuron_application *dev, u32 req_id, + u64 start, u32 num, u16 flags); +}; + +extern struct neuron_protocol_driver protocol_client_block_driver; +extern struct neuron_protocol_driver protocol_server_block_driver; diff --git a/net/Kconfig b/net/Kconfig index 3101bfcbdd7a..a8e998604662 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -391,6 +391,7 @@ source "net/ceph/Kconfig" source "net/nfc/Kconfig" source "net/psample/Kconfig" source "net/ife/Kconfig" +source "net/neuron/Kconfig" config LWTUNNEL bool "Network light weight tunnels" diff --git a/net/Makefile b/net/Makefile index 449fc0b221f8..88d1bdc6d592 100644 --- a/net/Makefile +++ b/net/Makefile @@ -87,3 +87,4 @@ endif obj-$(CONFIG_QRTR) += qrtr/ obj-$(CONFIG_NET_NCSI) += ncsi/ obj-$(CONFIG_XDP_SOCKETS) += xdp/ +obj-$(CONFIG_NEURON) += neuron/ diff --git a/net/neuron/Kconfig b/net/neuron/Kconfig new file mode 100644 index 000000000000..79871b935142 --- /dev/null +++ b/net/neuron/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "Neuron Device-sharing Framework" + +config NEURON + tristate "Support for Neuron device-sharing framework" + help + This option enables the Neuron device-sharing framework. It is used + by guests of the haven hypervisors to serve or access shared I/O + devices and other inter-VM services. The Neuron framework is composed + of three buses that represent different layers in the framework + (channel, protocol, application). + +config NEURON_SERVICE + tristate "Static configuration of services from the device tree" + depends on OF + select NEURON + help + This option enables a platform driver that can construct a Neuron + service stack from a description in a device tree. The device tree + node should contain a channel, application and protocol subnode. + Multiple services can run on a system. + +source "net/neuron/channel/Kconfig" +source "net/neuron/protocol/Kconfig" + +endmenu diff --git a/net/neuron/Makefile b/net/neuron/Makefile new file mode 100644 index 000000000000..6ef838400454 --- /dev/null +++ b/net/neuron/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +# bus types +obj-$(CONFIG_NEURON) += channel_bus.o +obj-$(CONFIG_NEURON) += protocol_bus.o +obj-$(CONFIG_NEURON) += application_bus.o + +# platform device +obj-$(CONFIG_NEURON_SERVICE) += neuron_service.o + +# drivers +obj-y += channel/ +obj-y += protocol/ diff --git a/net/neuron/application_bus.c b/net/neuron/application_bus.c new file mode 100644 index 000000000000..0cf3011d0625 --- /dev/null +++ b/net/neuron/application_bus.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +/* Neuron application bus type driver + * + * This driver creates an application bus type device and registers application + * driver. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +static int app_match(struct device *dev, struct device_driver *driver) +{ + if (of_driver_match_device(dev, driver)) + return 1; + + return 0; +} + +static int app_probe(struct device *dev) +{ + int ret; + struct neuron_application *app_dev = to_neuron_application(dev); + struct neuron_app_driver *app_drv = to_neuron_app_driver(dev->driver); + + ret = 0; + if (app_drv->probe) + ret = app_drv->probe(app_dev); + else if (dev->driver->probe) + ret = dev->driver->probe(dev); + + return ret; +} + +static int app_remove(struct device *dev) +{ + struct neuron_application *app_dev = to_neuron_application(dev); + struct neuron_app_driver *app_drv = to_neuron_app_driver(dev->driver); + + if (app_drv->remove) + app_drv->remove(app_dev); + else if (dev->driver->remove) + dev->driver->remove(dev); + + return 0; +} + +static struct bus_type app_bus_type = { + .name = "neuron_application", + .match = app_match, + .probe = app_probe, + .remove = app_remove, +}; + +static void app_dev_release(struct device *dev) +{ + struct neuron_application *app_dev = to_neuron_application(dev); + + put_device(&app_dev->protocol->dev); + kfree(app_dev); +} + +struct neuron_application *neuron_app_add(struct device_node *node, + struct device *parent) +{ + struct neuron_application *app_dev; + int err; + + app_dev = kzalloc(sizeof(*app_dev), GFP_KERNEL); + if (!app_dev) + return ERR_PTR(-ENOMEM); + device_initialize(&app_dev->dev); + + app_dev->dev.of_node = node; + app_dev->dev.bus = &app_bus_type; + app_dev->dev.parent = parent; + app_dev->dev.release = app_dev_release; + dev_set_name(&app_dev->dev, "%s:%s", dev_name(parent), node->name); + + err = device_add(&app_dev->dev); + if (err) + goto fail_device_add; + + return app_dev; + +fail_device_add: + put_device(&app_dev->dev); + return ERR_PTR(err); +} +EXPORT_SYMBOL(neuron_app_add); + +int neuron_register_app_driver(struct neuron_app_driver *drv) +{ + int ret; + + drv->driver.bus = &app_bus_type; + ret = driver_register(&drv->driver); + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL(neuron_register_app_driver); + +void neuron_unregister_app_driver(struct neuron_app_driver *drv) +{ + driver_unregister(&drv->driver); +} +EXPORT_SYMBOL(neuron_unregister_app_driver); + +static int __init app_bus_init(void) +{ + int ret; + + ret = bus_register(&app_bus_type); + if (ret < 0) { + pr_err("Unable to register bus\n"); + return ret; + } + + return 0; +} + +static void app_bus_exit(void) +{ + bus_unregister(&app_bus_type); +} + +subsys_initcall(app_bus_init); +module_exit(app_bus_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Neuron application bus module"); diff --git a/net/neuron/channel/Kconfig b/net/neuron/channel/Kconfig new file mode 100644 index 000000000000..2990e9aec077 --- /dev/null +++ b/net/neuron/channel/Kconfig @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +config NEURON_CH_HAVEN + tristate "Shared memory channel drivers for Haven guests" + depends on OF && HAVEN_DRIVERS + help + This option enables receive and send Neuron channel drivers that use + a shared memory buffer and a pair of notification objects to + communicate with another VM. This is the Haven version. + + If unsure, say Y. diff --git a/net/neuron/channel/Makefile b/net/neuron/channel/Makefile new file mode 100644 index 000000000000..6a431c7a510b --- /dev/null +++ b/net/neuron/channel/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_NEURON_CH_HAVEN) += ch_haven_recv.o +obj-$(CONFIG_NEURON_CH_HAVEN) += ch_haven_send.o diff --git a/net/neuron/channel/ch_haven_recv.c b/net/neuron/channel/ch_haven_recv.c new file mode 100644 index 000000000000..ad654c489239 --- /dev/null +++ b/net/neuron/channel/ch_haven_recv.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ch_mq_shmem_common.h" + +#define CHANNEL_VERSION NEURON_SHMEM_CHANNEL_V1 +#define CH_DBL_MASK 0x1 + +/* Cache line size, 64 bytes is picked since it is the largest one currently */ +#define CACHE_LINE_SIZE 64 + +/* Messages has to be aligned for performance's concern. */ +#define ALIGNMENT_BYTES 16 + +static inline void shm_clear_header(struct neuron_mq_data_priv *priv) +{ + struct neuron_shmem_channel_header *hdr = priv->base; + struct neuron_msg_queue *msgq = &priv->msgq; + + /* clear space_for_next field */ + hdr->space_for_next = 0; + /* clear tail offset */ + *msgq->tailp = 0; + /* Make sure memory writing is finished */ + mb(); + /* Set tail_offset to correct value to indicate + * to the sender that we are synced. + */ + smp_store_release(&hdr->tail_offset, CACHE_LINE_SIZE); +} + +/* Read data from the ring buffer */ +static inline int ring_read_msg(struct neuron_msg_queue *msgq, + struct buffer_list dest, size_t n) +{ + int ret; + void *src; + off_t offset = msgq->offset; + + offset = round_up(offset + PACKET_HEADER_SIZE, + msgq->message_alignment); + if (offset >= msgq->ring_buffer_len) + offset -= msgq->ring_buffer_len; + + src = msgq->ring_buffer_p + offset; + + if (offset + n <= msgq->ring_buffer_len) { + ret = skb_store_bits(dest.head, dest.offset, src, n); + if (ret) + return ret; + } else { + size_t n_1 = msgq->ring_buffer_len - offset; + size_t n_2 = n - n_1; + + ret = skb_store_bits(dest.head, dest.offset, src, n_1); + if (ret) + return ret; + ret = skb_store_bits(dest.head, dest.offset + n_1, + msgq->ring_buffer_p, n_2); + if (ret) + return ret; + } + + offset = round_up(offset + n, msgq->message_alignment); + if (offset >= msgq->ring_buffer_len) + offset -= msgq->ring_buffer_len; + + msgq->offset = offset; + + return 0; +} + +static inline int channel_hh_kick(struct neuron_mq_data_priv *priv) +{ + hh_dbl_flags_t dbl_mask = CH_DBL_MASK; + int ret; + + ret = hh_dbl_send(priv->tx_dbl, &dbl_mask); + if (ret) + pr_err("failed to raise virq to the sender %d\n", ret); + + return ret; +} + +static ssize_t channel_hh_receivev(struct neuron_channel *channel_dev, + struct buffer_list buf) +{ + ssize_t ret; + size_t left_space, new_left_space, space_for_next, len; + off_t head, tail; + struct neuron_msg_hdr *msg; + struct neuron_mq_data_priv *priv = dev_get_drvdata(&channel_dev->dev); + struct neuron_msg_queue *msgq = &priv->msgq; + + /* read shared variable from memory */ + if (unlikely(!smp_load_acquire(&priv->synced))) + return -EAGAIN; + + /* Get head offset from the shared header */ + head = smp_load_acquire(msgq->headp); + tail = msgq->offset; + + /* The sender rebooting has been detected. */ + if (unlikely(head >= msgq->ring_buffer_len)) { + dev_warn(&channel_dev->dev, + "The sender rebooted. Start to sync again!\n"); + return -ECONNRESET; + } + + if (head == tail) { + /* Set notification flag so that the sender can wake me up. */ + dev_dbg(&channel_dev->dev, "empty buffer!\n"); + return -EAGAIN; + } + + msg = (struct neuron_msg_hdr *)((char *)msgq->ring_buffer_p + tail); + len = msg->size + 1; + + WARN_ON(len > S32_MAX); + + if (len > buf.size) { + dev_dbg(&channel_dev->dev, "message too long to fill"); + return -EMSGSIZE; + } + + ret = ring_read_msg(msgq, buf, len); + if (ret) + return ret; + + /* Publish to the shared header */ + smp_store_release(msgq->tailp, msgq->offset); + + /* Making sure memory writing finished */ + mb(); + + /* Get space_for_next from the shared header */ + space_for_next = smp_load_acquire(msgq->space_for_next_p); + + head = READ_ONCE(*msgq->headp); + + if (head < tail) + left_space = tail - head - 1; + else + left_space = msgq->ring_buffer_len + tail - head - 1; + + if (head < msgq->offset) + new_left_space = msgq->offset - head - 1; + else + new_left_space = msgq->ring_buffer_len + msgq->offset - + head - 1; + + /* Wake up the sender when + * 1. There is not enough space when the sender tries to send last time + * 2. There is enough space after this receiving + */ + if (left_space < space_for_next && new_left_space >= space_for_next) { + dev_dbg(&channel_dev->dev, "Waking the sender up"); + /* wake up the sender */ + channel_hh_kick(priv); + } + + return (ssize_t)len; +} + +static ssize_t channel_hh_receive(struct neuron_channel *channel_dev, + struct sk_buff *skb) +{ + struct buffer_list buf = { + .head = skb, + .offset = 0, + .size = skb->len + }; + return channel_hh_receivev(channel_dev, buf); +} + +static void channel_hh_cb(int irq, void *data) +{ + struct neuron_mq_data_priv *priv = data; + + wake_up(&priv->wait_q); + neuron_channel_wakeup(priv->dev); +} + +static int msgq_init(struct neuron_mq_data_priv *priv) +{ + int ret; + struct neuron_shmem_channel_header *hdr = priv->base; + struct neuron_msg_queue *msgq = &priv->msgq; + struct neuron_channel *channel = priv->dev; + + msgq->offset = 0; + msgq->message_alignment = ALIGNMENT_BYTES; + msgq->headp = &hdr->head; + msgq->tailp = (u32 *)((u8 *)hdr + CACHE_LINE_SIZE); + msgq->space_for_next_p = &hdr->space_for_next; + msgq->ring_buffer_p = (void *)round_up((uintptr_t)hdr + + 2 * CACHE_LINE_SIZE, msgq->message_alignment); + msgq->ring_buffer_len = resource_size(&priv->buffer) - + 2 * CACHE_LINE_SIZE; + + ret = channel_set_limits(channel, msgq); + if (ret) + return ret; + + hdr->version = CHANNEL_VERSION; + /* Make sure version is visible when the sender sees tail_offset */ + mb(); + /* Set tail_offset to -1 to indicate it is unsynced. + */ + hdr->tail_offset = -1; + /* Set message_alignment field. */ + hdr->message_alignment = msgq->message_alignment; + /* Set ring_buffer_offset field as 2*CACHE_LINE_SIZE */ + hdr->ring_buffer_offset = (uintptr_t)msgq->ring_buffer_p - + (uintptr_t)hdr; + /* Set ring_buffer_len field */ + hdr->ring_buffer_len = msgq->ring_buffer_len; + /* Set max msg size field */ + hdr->max_msg_size = channel->max_size; + + /* Notify the sender that the channel has been reset. */ + channel_hh_kick(priv); + + return 0; +} + +/* Thread to sync with the sender. Note: this thread might never finishes if + * it fails to sync with the peer. + */ +static int channel_sync_thread(void *data) +{ + struct neuron_shmem_channel_header *hdr; + struct neuron_mq_data_priv *priv = (struct neuron_mq_data_priv *)data; + + hdr = (struct neuron_shmem_channel_header *)priv->base; + + /* Waiting for head being updated by the sender. */ + wait_event_killable(priv->wait_q, smp_load_acquire(&hdr->head) == -1 || + kthread_should_stop()); + + if (kthread_should_stop()) + return 0; + + /* If the version doesn't match (it could also be case the peer is of + * version without version field), quit the sync. + */ + if (READ_ONCE(hdr->version) != CHANNEL_VERSION) { + pr_err("Mismatched channel version: Me: %u, Peer: %u\n", + CHANNEL_VERSION, hdr->version); + return -EPROTO; + } + + shm_clear_header(priv); + + /* Notify the sender that shared memory header has been initialized. */ + channel_hh_kick(priv); + + /* Waiting for the sender's readiness. */ + wait_event_killable(priv->wait_q, smp_load_acquire(&hdr->head) != -1 || + kthread_should_stop()); + + if (kthread_should_stop()) + return 0; + + /* flush shared variable to memory */ + smp_store_release(&priv->synced, 1); + + neuron_channel_wakeup(priv->dev); + + return 0; +} + +static int channel_hh_map_memory(struct neuron_mq_data_priv *priv, + struct device *dev) +{ + struct device_node *np; + resource_size_t size; + struct resource *r; + int ret; + + np = of_parse_phandle(dev->of_node, "shared-buffer", 0); + if (!np) { + dev_err(dev, "shared-buffer node missing!\n"); + return -EINVAL; + } + + ret = of_address_to_resource(np, 0, &priv->buffer); + of_node_put(np); + if (ret) { + dev_err(dev, "of_address_to_resource failed!\n"); + return -EINVAL; + } + size = resource_size(&priv->buffer); + + /* buffer parameters checking */ + if (!priv->buffer.start || size % PAGE_SIZE) { + dev_err(dev, "invalid memory region: start:%llx, size:%llx\n", + priv->buffer.start, size); + return -EINVAL; + } + + r = devm_request_mem_region(dev, priv->buffer.start, size, + dev_name(dev)); + if (!r) { + dev_err(dev, "request memory region failed!\n"); + return -ENXIO; + } + priv->base = ioremap_cache(priv->buffer.start, size); + if (!priv->base) { + dev_err(dev, "ioremap failed!\n"); + return -ENXIO; + } + + if (of_property_read_bool(dev->of_node, "qcom,primary")) + memset(priv->base, 0, + sizeof(struct neuron_shmem_channel_header)); + + return 0; +} + +static int channel_hh_probe(struct neuron_channel *cdev) +{ + struct device_node *node = cdev->dev.of_node; + struct device *dev = &cdev->dev; + struct neuron_mq_data_priv *priv; + enum hh_dbl_label dbl_label; + int ret; + + if (!node) + return -ENODEV; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = cdev; + + ret = channel_hh_map_memory(priv, dev); + if (ret) { + dev_err(dev, "failed to map memory %d\n", ret); + return ret; + } + + /* Get outgoing haven doorbell information */ + ret = of_property_read_u32(node, "haven-label", &dbl_label); + if (ret) { + dev_err(dev, "failed to read label info %d\n", ret); + goto fail_tx_dbl; + } + + priv->tx_dbl = hh_dbl_tx_register(dbl_label); + if (IS_ERR_OR_NULL(priv->tx_dbl)) { + ret = PTR_ERR(priv->tx_dbl); + dev_err(dev, "failed to get haven tx dbl %d\n", ret); + goto fail_tx_dbl; + } + + priv->rx_dbl = hh_dbl_rx_register(dbl_label, channel_hh_cb, priv); + if (IS_ERR_OR_NULL(priv->rx_dbl)) { + ret = PTR_ERR(priv->rx_dbl); + dev_err(dev, "failed to get haven rx dbl %d\n", ret); + goto fail_rx_dbl; + } + + /* Init the shared memory header and local message queue. */ + ret = msgq_init(priv); + if (ret) + goto fail_mask; + init_waitqueue_head(&priv->wait_q); + + /* Start the thread for syncing with the sender. */ + priv->sync_thread = kthread_run(channel_sync_thread, priv, + "recv_sync_thread"); + + dev_set_drvdata(&cdev->dev, priv); + + return 0; + +fail_mask: + hh_dbl_rx_unregister(priv->rx_dbl); +fail_rx_dbl: + hh_dbl_tx_unregister(priv->tx_dbl); +fail_tx_dbl: + iounmap(priv->base); + return ret; +} + +static void channel_hh_remove(struct neuron_channel *cdev) +{ + struct neuron_mq_data_priv *priv = dev_get_drvdata(&cdev->dev); + + /* Stop it anyway. */ + kthread_stop(priv->sync_thread); + iounmap(priv->base); + devm_release_mem_region(&cdev->dev, priv->buffer.start, + resource_size(&priv->buffer)); + + hh_dbl_tx_unregister(priv->tx_dbl); + hh_dbl_rx_unregister(priv->rx_dbl); +} + +static const struct of_device_id channel_hh_match[] = { + { .compatible = "qcom,neuron-channel-haven-shmem" }, + {}, +}; +MODULE_DEVICE_TABLE(of, channel_hh_match); + +static struct neuron_channel_driver channel_hh_recv_driver = { + .driver = { + .name = "ch_haven_recv", + .of_match_table = channel_hh_match, + }, + .type = NEURON_CHANNEL_MESSAGE_QUEUE, + .direction = NEURON_CHANNEL_RECEIVE, + .receive_msgv = channel_hh_receivev, + .receive_msg = channel_hh_receive, + .probe = channel_hh_probe, + .remove = channel_hh_remove, +}; + +static int __init channel_hh_init(void) +{ + int ret; + + ret = neuron_register_channel_driver(&channel_hh_recv_driver); + if (ret < 0) { + pr_err("Failed to register driver:%d\n", ret); + return ret; + } + + return 0; +} + +static void __exit channel_hh_exit(void) +{ + neuron_unregister_channel_driver(&channel_hh_recv_driver); +} +module_init(channel_hh_init); +module_exit(channel_hh_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Neuron channel haven shared memory receiver driver"); + diff --git a/net/neuron/channel/ch_haven_send.c b/net/neuron/channel/ch_haven_send.c new file mode 100644 index 000000000000..f803d7a04485 --- /dev/null +++ b/net/neuron/channel/ch_haven_send.c @@ -0,0 +1,440 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ch_mq_shmem_common.h" + +#define CHANNEL_VERSION NEURON_SHMEM_CHANNEL_V1 +#define CH_DBL_MASK 0x1 + +/* Write data to the ring buffer */ +static inline void ring_write_msg(struct neuron_msg_queue *msgq, + struct buffer_list src, size_t n) +{ + off_t offset = msgq->offset; + struct neuron_msg_hdr *hdr; + void *dest; + + hdr = (struct neuron_msg_hdr *)(msgq->ring_buffer_p + offset); + + /* Size index is from 0. */ + hdr->size = n - 1; + + offset = round_up(offset + PACKET_HEADER_SIZE, + msgq->message_alignment); + if (offset >= msgq->ring_buffer_len) + offset -= msgq->ring_buffer_len; + + dest = msgq->ring_buffer_p + offset; + + if (offset + n <= msgq->ring_buffer_len) { + skb_copy_bits(src.head, src.offset, dest, n); + } else { + size_t n_1 = msgq->ring_buffer_len - offset; + size_t n_2 = n - n_1; + + skb_copy_bits(src.head, src.offset, dest, n_1); + skb_copy_bits(src.head, src.offset + n_1, msgq->ring_buffer_p, + n_2); + } + + offset = round_up(offset + n, msgq->message_alignment); + if (offset >= msgq->ring_buffer_len) + offset -= msgq->ring_buffer_len; + + msgq->offset = offset; +} + +static inline int channel_hh_kick(struct neuron_mq_data_priv *priv) +{ + hh_dbl_flags_t dbl_mask = CH_DBL_MASK; + int ret; + + ret = hh_dbl_send(priv->tx_dbl, &dbl_mask); + if (ret) + pr_err("failed to raise virq to the sender %d\n", ret); + + return ret; +} + +static int channel_hh_sendv(struct neuron_channel *channel_dev, + struct buffer_list buf) +{ + size_t space, space_needed, len; + off_t tail, head; + int prev_empty = 0; + + struct neuron_mq_data_priv *priv = dev_get_drvdata(&channel_dev->dev); + struct neuron_msg_queue *msgq = &priv->msgq; + + /* read shared variable from memory */ + if (unlikely(!smp_load_acquire(&priv->synced))) + return -EAGAIN; + + len = buf.size; + + /* Nothing to send */ + if (len == 0) { + dev_err(&channel_dev->dev, "len is 0\n"); + return -EINVAL; + } + + if (len > channel_dev->max_size) { + dev_err(&channel_dev->dev, + "buffer is oversized - len is %zd and %lu is supported\n", + len, channel_dev->max_size); + return -EMSGSIZE; + } + + space_needed = + round_up(PACKET_HEADER_SIZE, msgq->message_alignment) + + round_up(len, msgq->message_alignment); + WRITE_ONCE(*msgq->space_for_next_p, space_needed); + + /* Order the space_for_next write before the tail read */ + mb(); + + /* Get the tail from the shared memory */ + tail = smp_load_acquire(msgq->tailp); + head = msgq->offset; + + /* The receiver rebooting has been detected. */ + if (unlikely(tail >= msgq->ring_buffer_len)) { + dev_warn(&channel_dev->dev, + "The receiver rebooted. Start to sync again!\n"); + return -ECONNRESET; + } + + if (head < tail) + space = tail - head - 1; + else + space = msgq->ring_buffer_len + tail - head - 1; + + if (space < space_needed) + return -EAGAIN; + + prev_empty = msgq->offset == head ? 1 : 0; + + ring_write_msg(msgq, buf, len); + + /* Publish to the shared header */ + smp_store_release(msgq->headp, msgq->offset); + + if (prev_empty) + channel_hh_kick(priv); + + return 0; +} + +static int channel_hh_send(struct neuron_channel *channel_dev, + struct sk_buff *skb) +{ + struct buffer_list buf = { + .head = skb, + .offset = 0, + .size = skb->len + }; + + return channel_hh_sendv(channel_dev, buf); +} + +static void channel_hh_cb(int irq, void *data) +{ + struct neuron_mq_data_priv *priv = data; + + wake_up(&priv->wait_q); + neuron_channel_wakeup(priv->dev); +} + +static int read_config(struct neuron_mq_data_priv *priv) +{ + int ret; + u32 offset; + struct neuron_shmem_channel_header *hdr = priv->base; + struct neuron_msg_queue *msgq = &priv->msgq; + struct neuron_channel *channel = priv->dev; + + /* Clear local offset. */ + msgq->offset = 0; + /* Get message_alignment value. */ + msgq->message_alignment = hdr->message_alignment; + /* Get ring_buffer_len value. */ + msgq->ring_buffer_len = hdr->ring_buffer_len; + /* Get tail_offset, making sure the value is valid. */ + offset = hdr->tail_offset; + if (offset > resource_size(&priv->buffer)) + return -ECONNRESET; + offset = array_index_nospec(offset, resource_size(&priv->buffer)); + msgq->tailp = (u32 *)((u8 *)hdr + offset); + + msgq->space_for_next_p = &hdr->space_for_next; + /* Get ring_buffer_offset, making sure the value is valid*/ + offset = hdr->ring_buffer_offset; + if (offset > resource_size(&priv->buffer)) + return -ECONNRESET; + offset = array_index_nospec(offset, resource_size(&priv->buffer)); + msgq->ring_buffer_p = (void *)round_up((uintptr_t)hdr + offset, + msgq->message_alignment); + + /* send channel max message size is: + * 1. From device tree definition if it is not bigger than the receiver + * max message size. Or + * 2. The receiver max message size if not defined. + */ + if (channel->max_size && channel->max_size > hdr->max_msg_size) { + dev_err(&channel->dev, + "send max message size is bigger than receiver max message size!\n"); + return -EINVAL; + } else if (channel->max_size == 0) { + channel->max_size = hdr->max_msg_size; + } + + ret = channel_set_limits(channel, msgq); + if (ret < 0) + return ret; + + /* write shared variable to memory */ + smp_store_release(&priv->synced, 1); + + return 0; +} + +/* Thread to sync with the receiver. Note: this thread might never finishes if + * it fails to sync with the peer. + */ +static int channel_sync_thread(void *data) +{ + struct neuron_mq_data_priv *priv = (struct neuron_mq_data_priv *)data; + struct neuron_shmem_channel_header *hdr; + + hdr = (struct neuron_shmem_channel_header *)priv->base; + hdr->version = CHANNEL_VERSION; + hdr->tail_offset = -1; + + /* Make sure the memory writing is in order. */ + mb(); + hdr->head = -1; + channel_hh_kick(priv); + + /* Waiting for tail_offset being updated by the send. */ + wait_event_killable(priv->wait_q, kthread_should_stop() || + /* read shared variable from memory */ + smp_load_acquire(&hdr->tail_offset) != -1); + + if (kthread_should_stop()) + return 0; + + /* If the version doesn't match (it could also be case the peer is of + * version without version field), quit the sync. + */ + if (smp_load_acquire(&hdr->version) != CHANNEL_VERSION) { + pr_err("Mismatched channel version: Me: %u, Peer: %u\n", + CHANNEL_VERSION, hdr->version); + return -EPROTO; + } + /* Clear head offset to 0 */ + smp_store_release(&hdr->head, 0); + + if (read_config(priv)) { + pr_err("config failed\n"); + return -EINVAL; + } + + channel_hh_kick(priv); + + neuron_channel_wakeup(priv->dev); + + return 0; +} + +static void msgq_init(struct neuron_mq_data_priv *priv) +{ + struct neuron_shmem_channel_header *hdr = priv->base; + struct neuron_msg_queue *msgq = &priv->msgq; + + msgq->headp = &hdr->head; + msgq->space_for_next_p = &hdr->space_for_next; + msgq->offset = (u32)-1; + + /* Set it to -1 as UNINITIALIZED */ + smp_store_release(msgq->headp, (u32)-1); +} + +static int channel_hh_map_memory(struct neuron_mq_data_priv *priv, + struct device *dev) +{ + struct device_node *np; + resource_size_t size; + struct resource *r; + int ret; + + np = of_parse_phandle(dev->of_node, "shared-buffer", 0); + if (!np) { + dev_err(dev, "shared-buffer node missing!\n"); + return -EINVAL; + } + + ret = of_address_to_resource(np, 0, &priv->buffer); + of_node_put(np); + if (ret) { + dev_err(dev, "of_address_to_resource failed!\n"); + return -EINVAL; + } + size = resource_size(&priv->buffer); + + /* buffer parameters checking */ + if (!priv->buffer.start || size % PAGE_SIZE) { + dev_err(dev, "invalid memory region: start:%llx, size:%llx\n", + priv->buffer.start, size); + return -EINVAL; + } + + r = devm_request_mem_region(dev, priv->buffer.start, size, + dev_name(dev)); + if (!r) { + dev_err(dev, "request memory region failed!\n"); + return -ENXIO; + } + priv->base = ioremap_cache(priv->buffer.start, size); + if (!priv->base) { + dev_err(dev, "ioremap failed!\n"); + return -ENXIO; + } + + if (of_property_read_bool(dev->of_node, "qcom,primary")) + memset(priv->base, 0, + sizeof(struct neuron_shmem_channel_header)); + + return 0; +} + +static int channel_hh_probe(struct neuron_channel *cdev) +{ + struct device_node *node = cdev->dev.of_node; + struct device *dev = &cdev->dev; + struct neuron_mq_data_priv *priv; + enum hh_dbl_label dbl_label; + int ret; + + if (!node) + return -ENODEV; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = cdev; + + ret = channel_hh_map_memory(priv, dev); + if (ret) + return ret; + + /* Get outgoing haven doorbell information */ + ret = of_property_read_u32(node, "haven-label", &dbl_label); + if (ret) { + dev_err(dev, "failed to read label info %d\n", ret); + goto fail_tx_dbl; + } + + priv->tx_dbl = hh_dbl_tx_register(dbl_label); + if (IS_ERR_OR_NULL(priv->tx_dbl)) { + ret = PTR_ERR(priv->tx_dbl); + dev_err(dev, "failed to get haven tx dbl %d\n", ret); + goto fail_tx_dbl; + } + + priv->rx_dbl = hh_dbl_rx_register(dbl_label, channel_hh_cb, priv); + if (IS_ERR_OR_NULL(priv->rx_dbl)) { + ret = PTR_ERR(priv->rx_dbl); + dev_err(dev, "failed to get haven rx dbl %d\n", ret); + goto fail_rx_dbl; + } + + msgq_init(priv); + init_waitqueue_head(&priv->wait_q); + /* Start the thread for syncing with the receiver. */ + priv->sync_thread = kthread_run(channel_sync_thread, priv, + "send_sync_thread"); + + dev_set_drvdata(dev, priv); + return 0; + +fail_rx_dbl: + hh_dbl_tx_unregister(priv->tx_dbl); +fail_tx_dbl: + iounmap(priv->base); + return ret; +} + +static void channel_hh_remove(struct neuron_channel *cdev) +{ + struct neuron_mq_data_priv *priv = dev_get_drvdata(&cdev->dev); + + /* Stop it anyway. */ + kthread_stop(priv->sync_thread); + iounmap(priv->base); + devm_release_mem_region(&cdev->dev, priv->buffer.start, + resource_size(&priv->buffer)); + + hh_dbl_tx_unregister(priv->tx_dbl); + hh_dbl_rx_unregister(priv->rx_dbl); +} + +static const struct of_device_id channel_hh_match[] = { + { .compatible = "qcom,neuron-channel-haven-shmem" }, + {}, +}; +MODULE_DEVICE_TABLE(of, channel_hh_match); + +static struct neuron_channel_driver channel_hh_send_driver = { + .driver = { + .name = "ch_haven_send", + .of_match_table = channel_hh_match, + }, + .type = NEURON_CHANNEL_MESSAGE_QUEUE, + .direction = NEURON_CHANNEL_SEND, + .send_msgv = channel_hh_sendv, + .send_msg = channel_hh_send, + .probe = channel_hh_probe, + .remove = channel_hh_remove, +}; + +static int __init channel_hh_init(void) +{ + int ret; + + ret = neuron_register_channel_driver(&channel_hh_send_driver); + if (ret < 0) { + pr_err("Failed to register driver: %d\n", ret); + return ret; + } + + return 0; +} + +static void __exit channel_hh_exit(void) +{ + neuron_unregister_channel_driver(&channel_hh_send_driver); +} +module_init(channel_hh_init); +module_exit(channel_hh_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Neuron channel haven shared memory send driver"); diff --git a/net/neuron/channel/ch_mq_shmem_common.h b/net/neuron/channel/ch_mq_shmem_common.h new file mode 100644 index 000000000000..3a74e81703e0 --- /dev/null +++ b/net/neuron/channel/ch_mq_shmem_common.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +/* Neuron Message Queue transport layer header file */ +#ifndef __NEURON_MQ_H + +#include +#include + +/* Current version number */ +#define NEURON_SHMEM_CHANNEL_V1 0xcafe0001 + +/* Shared Memory header definition. + * It locates on the beginning of the shared memory. + */ +struct neuron_shmem_channel_header { + /* channel header version */ + u32 version; + /* field for keeping head value, which is an offset value + * against the ring buffer + */ + u32 head; + /* tail offset */ + u32 tail_offset; + /* field for the expecting length of the next message.*/ + u32 space_for_next; + /* Maximum message size */ + u32 max_msg_size; + /* ring buffer offset */ + u32 ring_buffer_offset; + /* ring buffer lenghth */ + u32 ring_buffer_len; + /* message alignment bytes */ + u16 message_alignment; + /* notification flags. Currently unused. */ + u16 notification_flags; +} __packed; + +/* Message header. The message payload is somewhere aligned with + * message_alignment field after this header. + */ +struct neuron_msg_hdr { + u32 size; // size +} __packed; + +/* Packet header size*/ +#define PACKET_HEADER_SIZE sizeof(struct neuron_msg_hdr) + +/* Message Queue definition */ +struct neuron_msg_queue { + /* Point to field head in shared memory header */ + u32 *headp; + /* Point to tail offset field in shared memory header */ + u32 *tailp; + /* Point to space_for_next field in shared memory header */ + u32 *space_for_next_p; + /* ring buffer address */ + void *ring_buffer_p; + /* ring buffer length */ + u32 ring_buffer_len; + /* message alignment bytes */ + u16 message_alignment; + /* current local offset. It is the head offset for the sender + * and the tail offset for the receiver. + */ + u32 offset; +}; + +/* A struct for driver private data */ +struct neuron_mq_data_priv { + /* outgoing vIRQ */ + u32 virq_line; + /* incoming vIRQ */ + u32 virq; + /* unused at the moment */ + atomic64_t virq_payload; + /* A counter to calculate the interrupt received. */ + u32 interrupt_counter; + /* haven tx doorbell descriptor */ + void *tx_dbl; + /* haven rx doorbell descriptor */ + void *rx_dbl; + /* pointer to the device structure */ + struct neuron_channel *dev; + /* shared memory mapped address */ + void *base; + /* shared memory resource */ + struct resource buffer; + /* Flag to show whether the channel is synced. */ + int synced; + /* The pointer of thread doing the sync work. */ + struct task_struct *sync_thread; + /* Waiting queue for sync thread */ + wait_queue_head_t wait_q; + /* Message Queue */ + struct neuron_msg_queue msgq; +}; + +/* Setting limits for each side. */ +static inline int channel_set_limits(struct neuron_channel *channel, + struct neuron_msg_queue *msgq) +{ + /* Default limit is one message filling the whole buffer */ + if (!channel->max_size && !channel->queue_length) + channel->queue_length = 1; + + /* Calculate max size based on given queue length */ + if (!channel->max_size) + channel->max_size = + round_down((msgq->ring_buffer_len - 1) / + channel->queue_length, + msgq->message_alignment) - + round_up(PACKET_HEADER_SIZE, + msgq->message_alignment); + + /* Calculate queue length based on given max size */ + if (!channel->queue_length) + channel->queue_length = (msgq->ring_buffer_len - 1) / + (round_up(PACKET_HEADER_SIZE, + msgq->message_alignment) + + round_up(channel->max_size, + msgq->message_alignment)); + + /* Assert that max_size and queue_length are sane. */ + if ((size_t)msgq->ring_buffer_len - 1 < + channel->queue_length * + (size_t)(round_up(channel->max_size, + msgq->message_alignment) + + round_up(PACKET_HEADER_SIZE, + msgq->message_alignment))) { + dev_err(&channel->dev, + "ring buf size %zu too small for %u * %zu messages\n", + (size_t)msgq->ring_buffer_len, + (unsigned int)channel->queue_length, + channel->max_size); + return -EINVAL; + } + + return 0; +} + +#endif diff --git a/net/neuron/channel_bus.c b/net/neuron/channel_bus.c new file mode 100644 index 000000000000..9508457d2108 --- /dev/null +++ b/net/neuron/channel_bus.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +/* Neuron channel bus type driver + * + * This driver creates a channel bus type device and registers a channel driver. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +static int channel_match(struct device *dev, struct device_driver *driver) +{ + struct neuron_channel *channel = to_neuron_channel(dev); + struct neuron_channel_driver *channel_drv = + to_neuron_channel_driver(driver); + + /* The channel's type must match the driver's. */ + if (channel->type != channel_drv->type) + return 0; + + /* The driver must be able to drive the channel in the required + * direction(s). + */ + if ((channel->direction & channel_drv->direction) != + channel->direction) + return 0; + + if (of_driver_match_device(dev, driver)) + return 1; + + return 0; +} + +static int channel_probe(struct device *dev) +{ + struct neuron_channel *channel_dev = to_neuron_channel(dev); + struct neuron_channel_driver *channel_drv = + to_neuron_channel_driver(dev->driver); + + if (channel_drv->probe) + return channel_drv->probe(channel_dev); + else if (dev->driver->probe) + return dev->driver->probe(dev); + + return 0; +} + +static int channel_remove(struct device *dev) +{ + struct neuron_channel *channel_dev = to_neuron_channel(dev); + struct neuron_channel_driver *channel_drv = + to_neuron_channel_driver(dev->driver); + + if (channel_drv->remove) + channel_drv->remove(channel_dev); + else if (dev->driver->remove) + dev->driver->remove(dev); + + return 0; +} + +static struct bus_type channel_bus_type = { + .name = "neuron_channel", + .match = channel_match, + .probe = channel_probe, + .remove = channel_remove, +}; + +static void channel_dev_release(struct device *dev) +{ + struct neuron_channel *channel_dev = to_neuron_channel(dev); + + put_device(&channel_dev->protocol->dev); + kfree(channel_dev); +} + +struct neuron_channel *neuron_channel_add(struct device_node *node, + struct device *parent) +{ + struct neuron_channel *channel_dev; + const char *str; + int reg = 0; + int err; + + channel_dev = kzalloc(sizeof(*channel_dev), GFP_KERNEL); + if (!channel_dev) + return ERR_PTR(-ENOMEM); + device_initialize(&channel_dev->dev); + + channel_dev->dev.of_node = node; + channel_dev->dev.bus = &channel_bus_type; + channel_dev->dev.parent = parent; + channel_dev->dev.release = channel_dev_release; + + err = of_property_read_u32(node, "reg", ®); + if (err < 0) { + dev_err(parent, "channel %s has no reg property\n", + node->full_name); + goto fail_properties; + } + channel_dev->id = reg; + + err = of_property_read_string(node, "direction", &str); + if (err < 0) { + dev_err(parent, "channel %d: channel direction is undefined\n", + reg); + goto fail_properties; + } + + err = -EINVAL; + if (!strcmp(str, "send")) { + channel_dev->direction = NEURON_CHANNEL_SEND; + } else if (!strcmp(str, "receive")) { + channel_dev->direction = NEURON_CHANNEL_RECEIVE; + } else if (!strcmp(str, "both")) { + channel_dev->direction = NEURON_CHANNEL_BIDIRECTIONAL; + } else { + dev_err(parent, "channel %d: bad channel direction \"%s\"\n", + reg, str); + goto fail_properties; + } + + err = of_property_read_string(node, "class", &str); + if (err < 0) { + dev_err(parent, "channel %d: channel type is undefined\n", + reg); + goto fail_properties; + } + + err = -EINVAL; + if (!strcmp(str, "message-queue")) { + u64 max_size = 0; + u32 queue_length = 0; + + channel_dev->type = NEURON_CHANNEL_MESSAGE_QUEUE; + + /* If these reads fail, we let the driver decide */ + of_property_read_u64(node, "max-size", &max_size); + channel_dev->max_size = (size_t)max_size; + of_property_read_u32(node, "queue-length", &queue_length); + channel_dev->queue_length = (unsigned int)queue_length; + } else if (!strcmp(str, "notification")) { + channel_dev->type = NEURON_CHANNEL_NOTIFICATION; + } else if (!strcmp(str, "shared-memory")) { + channel_dev->type = NEURON_CHANNEL_SHARED_MEMORY; + } else { + dev_err(parent, "channel %d: unknown channel type \"%s\"\n", + reg, str); + goto fail_properties; + } + + dev_set_name(&channel_dev->dev, "%s:%s%d", dev_name(parent), + node->name, reg); + + err = device_add(&channel_dev->dev); + if (err) + goto fail_device_add; + + return channel_dev; + +fail_device_add: +fail_properties: + put_device(&channel_dev->dev); + kfree(channel_dev); + return ERR_PTR(err); +} +EXPORT_SYMBOL(neuron_channel_add); + +int neuron_register_channel_driver(struct neuron_channel_driver *drv) +{ + int ret; + + drv->driver.bus = &channel_bus_type; + ret = driver_register(&drv->driver); + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL(neuron_register_channel_driver); + +void neuron_unregister_channel_driver(struct neuron_channel_driver *drv) +{ + driver_unregister(&drv->driver); +} +EXPORT_SYMBOL(neuron_unregister_channel_driver); + +static int __init channel_bus_init(void) +{ + int ret; + + ret = bus_register(&channel_bus_type); + if (ret < 0) { + pr_err("Unable to register bus\n"); + return ret; + } + + return 0; +} + +static void channel_bus_exit(void) +{ + bus_unregister(&channel_bus_type); +} + +subsys_initcall(channel_bus_init); +module_exit(channel_bus_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Neuron channel bus module"); diff --git a/net/neuron/neuron_service.c b/net/neuron/neuron_service.c new file mode 100644 index 000000000000..57a1fb845586 --- /dev/null +++ b/net/neuron/neuron_service.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +/* Neuron service driver + * + * This driver parses DT description of a Neuron service and creates protocol, + * application, and channel devices based on the DT nodes. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "neuron_service.h" + +#define DRIVER_NAME "neuron-service" +#define DEVICE_NAME "neuron-service" + +#ifndef CONFIG_OF +#error "neuron service driver only supported on device tree kernels" +#endif + +static const struct of_device_id neuron_service_match[] = { + { + .compatible = "qcom,neuron-service", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, neuron_service_match); + +static int neuron_service_probe(struct platform_device *pdev) +{ + struct neuron_service *neuron_serv; + static struct device_node *node; + static struct device_node *pnode; + int count = 0; + int err; + int i = 0; + + node = NULL; + pnode = pdev->dev.of_node; + + for_each_child_of_node(pnode, node) + if (node->name && (of_node_cmp(node->name, "channel") == 0)) + count++; + + if (!count) { + err = -ENODEV; + goto fail_find_channels; + } + + neuron_serv = kzalloc(sizeof(*neuron_serv) + + sizeof(struct neuron_channel *) * count, + GFP_KERNEL); + if (!neuron_serv) { + err = -ENOMEM; + goto fail_alloc_neuron_serv; + } + + neuron_serv->channel_count = count; + node = NULL; + + for_each_child_of_node(pnode, node) { + if (node->name && (of_node_cmp(node->name, "channel") == 0)) { + neuron_serv->channels[i] = + neuron_channel_add(node, &pdev->dev); + if (IS_ERR(neuron_serv->channels[i])) { + err = PTR_ERR(neuron_serv->channels[i]); + goto fail_add_channel; + } + neuron_serv->channels[i]->id = i; + i++; + } + } + + node = of_get_child_by_name(pnode, "application"); + if (!node) { + err = -ENODEV; + goto fail_find_application; + } + + neuron_serv->application = + neuron_app_add(node, &pdev->dev); + + if (IS_ERR(neuron_serv->application)) { + err = PTR_ERR(neuron_serv->application); + goto fail_add_application; + } + + node = of_get_child_by_name(pnode, "protocol"); + if (!node) { + err = -ENODEV; + goto fail_find_protocol; + } + + neuron_serv->protocol = + neuron_protocol_add(node, count, neuron_serv->channels, + &pdev->dev, neuron_serv->application); + if (IS_ERR(neuron_serv->protocol)) { + err = PTR_ERR(neuron_serv->protocol); + goto fail_add_protocol; + } + + for (i = 0; i < neuron_serv->channel_count; i++) { + neuron_serv->channels[i]->protocol = neuron_serv->protocol; + get_device(&neuron_serv->protocol->dev); + } + + /* Save protocol pointer */ + neuron_serv->application->protocol = neuron_serv->protocol; + get_device(&neuron_serv->protocol->dev); + + dev_set_drvdata(&pdev->dev, neuron_serv); + + return 0; + +fail_add_application: +fail_find_application: + device_unregister(&neuron_serv->protocol->dev); + put_device(&neuron_serv->protocol->dev); +fail_add_protocol: +fail_find_protocol: + for (i = 0; i < (neuron_serv->channel_count); i++) { + device_unregister(&neuron_serv->channels[i]->dev); + put_device(&neuron_serv->channels[i]->dev); + } +fail_add_channel: + dev_set_drvdata(&pdev->dev, NULL); + kfree(neuron_serv); +fail_alloc_neuron_serv: +fail_find_channels: + return err; +} + +static int neuron_service_remove(struct platform_device *pdev) +{ + struct neuron_service *neuron_serv; + int i; + + neuron_serv = dev_get_drvdata(&pdev->dev); + + /* Clearing all pointers to devices */ + neuron_serv->application->protocol = NULL; + neuron_serv->protocol->application = NULL; + for (i = 0; i < (neuron_serv->channel_count); i++) { + neuron_serv->protocol->channels[i] = NULL; + neuron_serv->channels[i]->protocol = NULL; + } + + device_unregister(&neuron_serv->application->dev); + put_device(&neuron_serv->application->dev); + + device_unregister(&neuron_serv->protocol->dev); + put_device(&neuron_serv->protocol->dev); + + for (i = 0; i < (neuron_serv->channel_count); i++) { + device_unregister(&neuron_serv->channels[i]->dev); + put_device(&neuron_serv->channels[i]->dev); + } + + dev_set_drvdata(&pdev->dev, NULL); + kfree(neuron_serv); + + return 0; +} + +static struct platform_driver neuron_service_driver = { + .driver = { + .name = DRIVER_NAME, + .of_match_table = neuron_service_match, + }, + .probe = neuron_service_probe, + .remove = neuron_service_remove, +}; + +static int __init neuron_service_init(void) +{ + int ret; + + ret = platform_driver_register(&neuron_service_driver); + if (ret < 0) { + pr_err("Failed to register driver\n"); + return ret; + } + + return 0; +} + +static void __exit neuron_service_exit(void) +{ + platform_driver_unregister(&neuron_service_driver); +} + +module_init(neuron_service_init); +module_exit(neuron_service_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Neuron service - configuration layer"); diff --git a/net/neuron/neuron_service.h b/net/neuron/neuron_service.h new file mode 100644 index 000000000000..e9415bebfaf5 --- /dev/null +++ b/net/neuron/neuron_service.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +#include + +/* Device's driver data */ +struct neuron_service { + struct neuron_protocol *protocol; + struct neuron_application *application; + unsigned int channel_count; + struct neuron_channel *channels[]; +}; diff --git a/net/neuron/protocol/Kconfig b/net/neuron/protocol/Kconfig new file mode 100644 index 000000000000..f3a5f976c0ca --- /dev/null +++ b/net/neuron/protocol/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +config NEURON_PROT_BLOCK_CLIENT + tristate "Protocol block client driver" + help + This option enables a Neuron block protocol client driver. It + contains the commands to virtualize a block device. This driver + will read from a block device being virtualized by a server. + + If unsure, say N. + +config NEURON_PROT_BLOCK_SERVER + tristate "Protocol block server driver" + help + This option enables a Neuron block protocol server driver. It + contains the commands to virtualize a block device. This driver + will send commands to virtualize a block device owned by this + virtual machine. + + If unsure, say N. diff --git a/net/neuron/protocol/Makefile b/net/neuron/protocol/Makefile new file mode 100644 index 000000000000..b77134bf8c5e --- /dev/null +++ b/net/neuron/protocol/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_NEURON_PROT_BLOCK_CLIENT) += prot_block_client.o +obj-$(CONFIG_NEURON_PROT_BLOCK_SERVER) += prot_block_server.o diff --git a/net/neuron/protocol/prot_block.h b/net/neuron/protocol/prot_block.h new file mode 100644 index 000000000000..c644691b0d45 --- /dev/null +++ b/net/neuron/protocol/prot_block.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +#ifndef __H_PROT_BLOCK_COMMON_H +#define __H_PROT_BLOCK_COMMON_H + +/* Block Server Advertisement message */ +struct neuron_block_advertise { + u32 logical_block_size; + u32 physical_block_size; + u64 num_device_sectors; + u32 flags; + u32 discard_granularity; + u64 discard_max_hw_sectors; + u64 discard_max_sectors; + u16 alignment_offset; + bool wc_flag; + bool fua_flag; + u8 uuid[16]; + u8 label[]; +} __packed; + +/* Request message */ +struct neuron_block_req { + u32 req_id; + u16 req_type; + u16 flags; + u64 start_sector; + u32 sectors; +} __packed; + +/* Response message */ +struct neuron_block_resp { + u32 resp_id; + u16 resp_status; +} __packed; + +/* Bit position definition for flags field in struct neuron_block_req */ +enum neuron_block_param_flags_bit { + NEURON_BLOCK_PARAM_FLAG_READONLY = 0, + NEURON_BLOCK_PARAM_FLAG_DISCARD_ZEROES, +}; + +#define NEURON_BLOCK_READONLY BIT(NEURON_BLOCK_PARAM_FLAG_READONLY) +#define NEURON_BLOCK_DISCARD_ZEROES BIT(NEURON_BLOCK_PARAM_FLAG_DISCARD_ZEROES) + +#endif diff --git a/net/neuron/protocol/prot_block_client.c b/net/neuron/protocol/prot_block_client.c new file mode 100644 index 000000000000..4a0041140976 --- /dev/null +++ b/net/neuron/protocol/prot_block_client.c @@ -0,0 +1,618 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +/* Protocol block client drivers + * + * This driver receives data from the application layer and sends it to the + * channel layer and vice versa. Receives from channel and sends to app. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "prot_block.h" + +#define DRIVER_NAME_CLIENT "neuron-protocol-client-block" + +#define C_IN 1 +#define C_OUT 0 +#define CHANNEL_COUNT 2 + +#define EV_REQUEST NEURON_BLOCK_CLIENT_EVENT_REQUEST +#define EVENT_COUNT NEURON_BLOCK_CLIENT_EVENT__COUNT + +#define CHANNEL_BIT(n) BIT(n) +#define EVENT_BIT(n) BIT(CHANNEL_COUNT + (n)) + +#define MIN_NEURON_BLOCK_AD_LEN offsetof(struct neuron_block_advertise, label) + +struct req_context { + enum neuron_block_resp_status status; + enum neuron_block_req_type req_type; + int req_id; + u16 flags; + u32 sectors; + void *opaque_id; + off_t offset; + u64 start_sector; + struct buffer_list buf; +}; + +struct client_data { + struct task_struct *thread; + wait_queue_head_t wait_q; + unsigned long wakeups; + u32 sector_size; + struct idr req_idr; +}; + +static const struct of_device_id protocol_block_client_match[] = { + { + .compatible = "qcom,neuron-protocol-block", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, protocol_block_client_match); + +/* Acquire a req id and store its associated context. + * @return A req id if success, others for failure. + */ +static int acquire_req_id(struct client_data *kdata, struct req_context *ctx) +{ + return idr_alloc(&kdata->req_idr, ctx, 1, INT_MAX, GFP_KERNEL); +} + +/* Retrieve req id associated context. */ +static int get_req_id_associated(struct client_data *kdata, int req_id, + struct req_context **ctx) +{ + struct req_context *context; + + context = (struct req_context *)idr_find(&kdata->req_idr, req_id); + if (!context) + return -ENODEV; + *ctx = context; + return 0; +} + +static void release_req_id(struct client_data *kdata, int req_id) +{ + idr_remove(&kdata->req_idr, req_id); +} + +static int protocol_block_channel_wakeup(struct neuron_protocol *protocol, + unsigned int id) +{ + struct client_data *kdata; + + if (WARN_ON_ONCE(id >= CHANNEL_COUNT)) + return -EINVAL; + + kdata = dev_get_drvdata(&protocol->dev); + set_bit(id, &kdata->wakeups); + wake_up(&kdata->wait_q); + + return 0; +} + +static int protocol_block_app_wakeup(struct neuron_protocol *prot_dev, + unsigned int ev) +{ + struct client_data *kdata; + + if (WARN_ON_ONCE(ev >= EVENT_COUNT)) + return -EINVAL; + + kdata = dev_get_drvdata(&prot_dev->dev); + set_bit(CHANNEL_COUNT + ev, &kdata->wakeups); + wake_up(&kdata->wait_q); + + return 0; +} + +static int protocol_block_client_thread(void *data) +{ + struct neuron_protocol *prot_dev = (struct neuron_protocol *)data; + + struct client_data *kdata = dev_get_drvdata(&prot_dev->dev); + + struct device *dev = &prot_dev->channels[C_OUT]->dev; + struct neuron_channel *channel_dev = to_neuron_channel(dev); + struct neuron_channel_driver *channel_drv = + to_neuron_channel_driver(dev->driver); + struct device *dev_r = &prot_dev->channels[C_IN]->dev; + struct neuron_channel *channel_dev_r = to_neuron_channel(dev_r); + struct neuron_channel_driver *channel_drv_r = + to_neuron_channel_driver(dev_r->driver); + + struct device *dev_app = &prot_dev->application->dev; + struct neuron_application *app_dev = to_neuron_application(dev_app); + + struct neuron_app_driver *app_drv = + to_neuron_app_driver(dev_app->driver); + struct neuron_block_app_client_driver *block_drv = + container_of(app_drv, + struct neuron_block_app_client_driver, base); + + struct sk_buff *skb_r, *skb, *skb_resp, *skb_req; + struct req_context *req_ctx = NULL, *resp_ctx = NULL; + struct neuron_block_resp *resp; + struct neuron_block_req *req; + + void *opaque_id; + u16 flags; + u64 start_sector; + u32 sectors; + enum neuron_block_req_type req_type; + int req_id; + + u32 left; + u32 to_send; + + struct neuron_block_advertise *ad; + struct neuron_block_param *param; + + size_t label_size; + + unsigned long wakeup_mask; + + bool req_valid = false; + bool req_header_sent = false; + bool resp_valid = false; + bool resp_ready = false; + int ret = 0; + + skb_resp = NULL; + skb_req = NULL; + + /* Wait for the channels to start */ + wakeup_mask = 0; + while (!kthread_should_stop()) { + if (!channel_dev->max_size || !channel_dev->queue_length) + wakeup_mask |= CHANNEL_BIT(C_IN); + + if (!channel_dev_r->max_size || !channel_dev_r->queue_length) + wakeup_mask |= CHANNEL_BIT(C_OUT); + + if (!wakeup_mask) + break; + + wait_event_killable(kdata->wait_q, + kthread_should_stop() || + (kdata->wakeups & wakeup_mask)); + + wakeup_mask &= ~xchg(&kdata->wakeups, 0); + /* flush shared variable to memory */ + smp_mb__after_atomic(); + } + if (kthread_should_stop()) + return 0; + + WARN_ON(channel_dev_r->max_size <= + sizeof(struct neuron_block_advertise)); + + skb_r = alloc_skb(channel_dev_r->max_size, GFP_KERNEL); + if (!skb_r) + return -ENOMEM; + + skb_put(skb_r, channel_dev_r->max_size); + /* Wait and get block params */ + do { + clear_bit(C_IN, &kdata->wakeups); + /* flush shared variable to memory */ + smp_mb__after_atomic(); + + ret = channel_drv_r->receive_msg(channel_dev_r, skb_r); + if (ret == -EAGAIN) { + wait_event_killable(kdata->wait_q, + kthread_should_stop() || + (kdata->wakeups & + CHANNEL_BIT(C_IN))); + } + + } while (ret == -EAGAIN); + + if (ret <= 0) { + pr_err("Receiving Block param failed with error %d\n", ret); + kfree_skb(skb_r); + return ret; + } + + if (ret < MIN_NEURON_BLOCK_AD_LEN) { + pr_err("Invalid block param length\n"); + kfree_skb(skb_r); + return -EINVAL; + } + + label_size = ret - MIN_NEURON_BLOCK_AD_LEN; + + ad = (struct neuron_block_advertise *)skb_r->data; + + /* Allocate 1 more byte to hold '\0' in case label is not + * NULL terminated. + */ + param = kzalloc(sizeof(*param) + label_size + 1, GFP_KERNEL); + if (!param) { + consume_skb(skb_r); + return -ENOMEM; + } + + /* Save logical block size */ + kdata->sector_size = ad->logical_block_size; + + param->logical_block_size = ad->logical_block_size; + WARN_ON(param->logical_block_size > channel_dev->max_size); + param->physical_block_size = ad->physical_block_size; + param->num_device_sectors = ad->num_device_sectors; + param->discard_max_hw_sectors = ad->discard_max_hw_sectors; + param->discard_max_sectors = ad->discard_max_sectors; + param->discard_granularity = ad->discard_granularity; + param->alignment_offset = ad->alignment_offset; + param->read_only = ad->flags & NEURON_BLOCK_READONLY; + param->discard_zeroes_data = ad->flags & NEURON_BLOCK_DISCARD_ZEROES; + param->wc_flag = ad->wc_flag; + param->fua_flag = ad->fua_flag; + memcpy(param->uuid.b, ad->uuid, sizeof(param->uuid.b)); + strlcpy(param->label, ad->label, label_size); + /* Ensure it is NULL terminated always. */ + param->label[label_size] = '\0'; + + consume_skb(skb_r); + + /* Set block device params. */ + block_drv->do_set_bd_params(app_dev, param); + + /* Pre-allocate a request SKB. This will be reallocated after each + * successful send; we can't reuse it in case the channel driver has + * cloned it. + */ + skb_req = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb_req) + return -ENOMEM; + req = (struct neuron_block_req *)skb_put(skb_req, sizeof(*req)); + + /* Pre-allocate the response SKB. This can be safely reused + * because it is never sent to any asynchronous function outside + * this driver. + */ + skb_resp = __alloc_skb(sizeof(*resp), GFP_KERNEL, SKB_ALLOC_RX, + NUMA_NO_NODE); + if (!skb_resp) { + kfree_skb(skb_req); + return -ENOMEM; + } + resp = (struct neuron_block_resp *)skb_put(skb_resp, sizeof(*resp)); + + /* Main request handling loop */ + wakeup_mask = 0; + while (!kthread_should_stop()) { + unsigned long old_wakeups, consumed_wakeups; + + /*Try to get request from client*/ + if (!req_valid && !(wakeup_mask & EVENT_BIT(EV_REQUEST))) { + ret = block_drv->get_request(app_dev, &opaque_id, + &req_type, &flags, + &start_sector, + §ors, &skb); + if (ret == -EAGAIN) { + wakeup_mask |= EVENT_BIT(EV_REQUEST); + continue; + } else if (ret < 0) { + dev_err(&prot_dev->dev, + "get_request returned %d\n", ret); + goto fail; + } + /* Create request context */ + req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL); + if (!req_ctx) { + ret = -ENOMEM; + goto fail; + } + req_ctx->opaque_id = opaque_id; + req_ctx->req_type = req_type; + req_ctx->flags = flags; + req_ctx->start_sector = start_sector; + req_ctx->sectors = sectors; + req_ctx->buf.head = skb; + req_ctx->buf.offset = 0; + + /* Acquire req id */ + req_id = acquire_req_id(kdata, req_ctx); + if (req_id < 0) { + dev_err(&prot_dev->dev, + "Can't acquire req id: %d\n", req_id); + ret = req_id; + kfree(req_ctx); + goto fail; + } + req_ctx->req_id = req_id; + + dev_dbg(&prot_dev->dev, "Getting request: req_id =%d, req_type =%d, sectors = %d\n", + req_id, req_type, sectors); + + req_valid = true; + req_header_sent = false; + continue; + } + + /* If we have a request header, try to send it. */ + if (req_valid && !req_header_sent && + !(wakeup_mask & CHANNEL_BIT(C_OUT))) { + WARN_ON(!req_ctx); + req->req_id = req_ctx->req_id; + req->req_type = req_ctx->req_type; + req->flags = req_ctx->flags; + req->start_sector = req_ctx->start_sector; + req->sectors = req_ctx->sectors; + + ret = channel_drv->send_msg(channel_dev, skb_req); + if (ret == -EAGAIN) { + wakeup_mask |= CHANNEL_BIT(C_OUT); + continue; + } else if (ret < 0) { + dev_err(&prot_dev->dev, + "Can't send request: %d\n", + ret); + goto fail; + } + + dev_dbg(&prot_dev->dev, "Sending req header for req_id = %d, req_type = %d\n", + req->req_id, req->req_type); + /* Consume and replace the request skb. */ + consume_skb(skb_req); + skb_req = alloc_skb(sizeof(*req), GFP_KERNEL); + if (!skb_req) { + kfree_skb(skb_resp); + return -ENOMEM; + } + req = (struct neuron_block_req *)skb_put(skb_req, + sizeof(*req)); + memset(req, 0, sizeof(*req)); + + /* Is there any data to send? */ + if (req_ctx->sectors && + (req_ctx->req_type == NEURON_BLOCK_REQUEST_WRITE || + req_ctx->req_type == + NEURON_BLOCK_REQUEST_WRITE_SAME)) + req_header_sent = true; + else + req_valid = false; + + continue; + } + + /* If we have a request data, try to send it. */ + if (req_valid && req_header_sent && + !(wakeup_mask & CHANNEL_BIT(C_OUT))) { + left = req_ctx->buf.head->len - req_ctx->buf.offset; + WARN_ON(left % kdata->sector_size); + to_send = min_t(u32, left, channel_dev->max_size); + to_send = round_down(to_send, kdata->sector_size); + req_ctx->buf.size = to_send; + ret = channel_drv->send_msgv(channel_dev, req_ctx->buf); + if (ret == -EAGAIN) { + wakeup_mask |= CHANNEL_BIT(C_OUT); + continue; + } else if (ret < 0) { + dev_err(&prot_dev->dev, + "Sending request data failed %d\n", + ret); + goto fail; + } + req_ctx->buf.offset += to_send; + if (req_ctx->buf.offset == req_ctx->buf.head->len) { + req_valid = false; + req_header_sent = false; + } + + dev_dbg(&prot_dev->dev, "Sending req data for req_id = %d with %d bytes\n", + req_ctx->req_id, to_send); + + continue; + } + + /* Try to obtain a response from the server. */ + if (!resp_valid && !(wakeup_mask & CHANNEL_BIT(C_IN))) { + ret = channel_drv_r->receive_msg(channel_dev_r, + skb_resp); + if (ret == -EAGAIN) { + wakeup_mask |= CHANNEL_BIT(C_IN); + continue; + } else if (ret < 0) { + dev_err(&prot_dev->dev, "receive_msg for resp header returned %d\n", + ret); + goto fail; + } else if (skb_resp->len < sizeof(*resp)) { + dev_err(&prot_dev->dev, "Truncated response\n"); + ret = -EBADMSG; + goto fail; + } + + if (get_req_id_associated(kdata, resp->resp_id, + &resp_ctx)) { + dev_err(&prot_dev->dev, "Incorrect response id %d\n", + resp->resp_id); + ret = -EBADMSG; + goto fail; + } + resp_ctx->status = resp->resp_status; + release_req_id(kdata, resp->resp_id); + + dev_dbg(&prot_dev->dev, "Receiving response for req_id %d\n", + resp->resp_id); + + resp_valid = true; + if (!resp->resp_status && + resp_ctx->req_type == NEURON_BLOCK_REQUEST_READ && + resp_ctx->sectors) { + WARN_ON(!resp_ctx->buf.head); + resp_ctx->buf.offset = 0; + resp_ctx->buf.size = resp_ctx->sectors * + kdata->sector_size; + resp_ready = false; + } else { + resp_ready = true; + } + continue; + } + + /* Try to receive response data if we are waiting for any. */ + if (resp_valid && !resp_ready && + !(wakeup_mask & CHANNEL_BIT(C_IN))) { + if (WARN_ON(!resp_ctx)) + goto fail; + ret = channel_drv_r->receive_msgv(channel_dev_r, + resp_ctx->buf); + if (ret == -EAGAIN) { + wakeup_mask |= CHANNEL_BIT(C_IN); + continue; + } else if (ret < 0) { + dev_err(&prot_dev->dev, + "receive_msg for resp header returned %d\n", + ret); + goto fail; + } + + dev_dbg(&prot_dev->dev, + "Receiving response data for req_id %d with %d bytes\n", + resp_ctx->req_id, ret); + + resp_ctx->buf.offset += ret; + resp_ctx->buf.size -= ret; + if (resp_ctx->buf.size == 0) + resp_ready = true; + continue; + } + + /* Deal with response if the response is ready. */ + if (resp_ready) { + if (WARN_ON(!resp_ctx)) + goto fail; + dev_dbg(&prot_dev->dev, "Response is done for req %d\n", + resp_ctx->req_id); + resp_ready = false; + resp_valid = false; + consume_skb(resp_ctx->buf.head); + block_drv->do_response(app_dev, resp_ctx->opaque_id, + resp_ctx->status); + kfree(resp_ctx); + continue; + } + + if (WARN_ON_ONCE(!wakeup_mask)) + continue; + + dev_dbg(&prot_dev->dev, "wait for events %#lx (now: %#lx)\n", + wakeup_mask, kdata->wakeups); + wait_event_killable(kdata->wait_q, + kthread_should_stop() || + (kdata->wakeups & wakeup_mask)); + + do { + old_wakeups = READ_ONCE(kdata->wakeups); + consumed_wakeups = old_wakeups & wakeup_mask; + } while (cmpxchg(&kdata->wakeups, old_wakeups, + old_wakeups & ~consumed_wakeups) != + old_wakeups); + /* flush shared variable to memory */ + smp_mb__after_atomic(); + wakeup_mask &= ~consumed_wakeups; + } + + consume_skb(skb_req); + consume_skb(skb_resp); + +fail: + kfree_skb(skb_req); + kfree_skb(skb_resp); + + return ret; +} + +static int protocol_block_client_probe(struct neuron_protocol *prot_dev) +{ + struct client_data *kdata; + + kdata = kzalloc(sizeof(*kdata), GFP_KERNEL); + if (!kdata) + return -ENOMEM; + + init_waitqueue_head(&kdata->wait_q); + + idr_init(&kdata->req_idr); + + dev_set_drvdata(&prot_dev->dev, kdata); + + kdata->thread = kthread_run(protocol_block_client_thread, prot_dev, + "%s", prot_dev->dev.driver->name); + return 0; +} + +static void protocol_block_client_remove(struct neuron_protocol *prot_dev) +{ + struct client_data *kdata; + + kdata = dev_get_drvdata(&prot_dev->dev); + kthread_stop(kdata->thread); + dev_set_drvdata(&prot_dev->dev, NULL); + kfree(kdata); +} + +static const struct neuron_channel_match_table channels_client_block[] = { + {NEURON_CHANNEL_MESSAGE_QUEUE, NEURON_CHANNEL_SEND}, + {NEURON_CHANNEL_MESSAGE_QUEUE, NEURON_CHANNEL_RECEIVE} +}; + +static const char * const processes_client_block[] = {"client"}; + +struct neuron_protocol_driver protocol_client_block_driver = { + .channel_count = 2, + .channels = channels_client_block, + .process_count = 1, + .processes = processes_client_block, + .driver = { + .name = DRIVER_NAME_CLIENT, + .owner = THIS_MODULE, + .of_match_table = protocol_block_client_match, + }, + .probe = protocol_block_client_probe, + .remove = protocol_block_client_remove, + .channel_wakeup = protocol_block_channel_wakeup, + .app_wakeup = protocol_block_app_wakeup, +}; +EXPORT_SYMBOL(protocol_client_block_driver); + +static int __init protocol_client_block_init(void) +{ + int ret; + + ret = neuron_register_protocol_driver(&protocol_client_block_driver); + if (ret < 0) { + pr_err("Failed to register driver\n"); + return ret; + } + return 0; +} + +static void __exit protocol_client_block_exit(void) +{ + neuron_unregister_protocol_driver(&protocol_client_block_driver); +} + +module_init(protocol_client_block_init); +module_exit(protocol_client_block_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Protocol block client drivers"); diff --git a/net/neuron/protocol/prot_block_server.c b/net/neuron/protocol/prot_block_server.c new file mode 100644 index 000000000000..1ba0fe48e386 --- /dev/null +++ b/net/neuron/protocol/prot_block_server.c @@ -0,0 +1,606 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +/* Protocol block server drivers + * + * This driver receives data from the application layer and sends it to the + * channel layer and vice versa. Receives from channel and sends to app. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "prot_block.h" + +#define DRIVER_NAME_SERVER "neuron-protocol-server-block" + +#define C_IN 0 +#define C_OUT 1 +#define CHANNEL_COUNT 2 + +#define EV_BD_PARAMS NEURON_BLOCK_SERVER_EVENT_BD_PARAMS +#define EV_RESPONSE NEURON_BLOCK_SERVER_EVENT_RESPONSE +#define EVENT_COUNT NEURON_BLOCK_SERVER_EVENT__COUNT + +#define CHANNEL_BIT(n) BIT(n) +#define EVENT_BIT(n) BIT(CHANNEL_COUNT + (n)) + +struct server_data { + struct task_struct *thread; + wait_queue_head_t wait_q; + unsigned long wakeups; +}; + +static const struct of_device_id protocol_block_server_match[] = { + { + .compatible = "qcom,neuron-protocol-block", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, protocol_block_server_match); + +static int protocol_block_channel_wakeup(struct neuron_protocol *prot, + unsigned int id) +{ + struct server_data *kdata; + + if (WARN_ON_ONCE(id >= CHANNEL_COUNT)) + return -EINVAL; + + kdata = dev_get_drvdata(&prot->dev); + set_bit(id, &kdata->wakeups); + dev_dbg(&prot->dev, "wake ch %d (now: %#lx)\n", id, kdata->wakeups); + wake_up(&kdata->wait_q); + + return 0; +} + +static int protocol_block_app_wakeup(struct neuron_protocol *prot, + unsigned int ev) +{ + struct server_data *kdata; + + if (WARN_ON_ONCE(ev >= EVENT_COUNT)) + return -EINVAL; + + kdata = dev_get_drvdata(&prot->dev); + set_bit(CHANNEL_COUNT + ev, &kdata->wakeups); + dev_dbg(&prot->dev, "wake ev %d (now: %#lx)\n", ev, kdata->wakeups); + wake_up(&kdata->wait_q); + + return 0; +} + +static int handle_request(struct neuron_block_app_server_driver *block_drv, + struct neuron_application *app_dev, + struct neuron_block_req *req, + struct sk_buff *skb_data) +{ + switch (req->req_type) { + case NEURON_BLOCK_REQUEST_READ: + WARN_ON(skb_data); + return block_drv->do_read(app_dev, req->req_id, + req->start_sector, req->sectors, + req->flags); + case NEURON_BLOCK_REQUEST_WRITE: + return block_drv->do_write(app_dev, req->req_id, + req->start_sector, req->sectors, + req->flags, skb_data); + case NEURON_BLOCK_REQUEST_DISCARD: + WARN_ON(skb_data); + return block_drv->do_discard(app_dev, req->req_id, + req->start_sector, req->sectors, + req->flags); + case NEURON_BLOCK_REQUEST_SECURE_ERASE: + WARN_ON(skb_data); + return block_drv->do_secure_erase(app_dev, req->req_id, + req->start_sector, + req->sectors, req->flags); + case NEURON_BLOCK_REQUEST_WRITE_SAME: + return block_drv->do_write_same(app_dev, req->req_id, + req->start_sector, + req->sectors, req->flags, + skb_data); + /* fall through return */ + case NEURON_BLOCK_REQUEST_WRITE_ZEROES: + WARN_ON(skb_data); + return block_drv->do_write_zeroes(app_dev, req->req_id, + req->start_sector, + req->sectors, req->flags); + default: + pr_err("Wrong req type: %d\n", req->req_type); + if (skb_data) + kfree_skb(skb_data); + return -EINVAL; + } +} + +static int protocol_block_server_thread(void *data) +{ + struct neuron_protocol *prot = (struct neuron_protocol *)data; + struct neuron_channel *c_in = prot->channels[C_IN]; + struct neuron_channel_driver *c_in_drv = + to_neuron_channel_driver(c_in->dev.driver); + struct neuron_channel *c_out = prot->channels[C_OUT]; + struct neuron_channel_driver *c_out_drv = + to_neuron_channel_driver(c_out->dev.driver); + struct neuron_application *app = prot->application; + struct neuron_app_driver *app_drv = + to_neuron_app_driver(app->dev.driver); + struct neuron_block_app_server_driver *block_drv = + container_of(app_drv, + struct neuron_block_app_server_driver, + base); + + struct server_data *kdata = dev_get_drvdata(&prot->dev); + + int ret; + size_t size; + + struct sk_buff *skb_ad = NULL; + struct neuron_block_advertise *ad; + const struct neuron_block_param *param; + + size_t sector_size; + + unsigned long wakeup_mask; + + bool req_valid = false; + bool req_ready = false; + struct neuron_block_req *req; + struct sk_buff *skb_req = NULL; + struct sk_buff *skb_req_data = NULL; + size_t req_data_pos = 0; + + bool resp_valid = false; + bool resp_sent_header = false; + struct neuron_block_resp *resp; + struct sk_buff *skb_resp = NULL; + struct sk_buff *skb_resp_data = NULL; + size_t resp_data_pos = 0; + + kdata->wakeups = 0; + + /* Wait for the channels to start */ + wakeup_mask = 0; + while (!kthread_should_stop()) { + if (!c_in->max_size || !c_in->queue_length) + wakeup_mask |= CHANNEL_BIT(C_IN); + + if (!c_out->max_size || !c_out->queue_length) + wakeup_mask |= CHANNEL_BIT(C_OUT); + + if (!wakeup_mask) + break; + + wait_event_killable(kdata->wait_q, + kthread_should_stop() || + (kdata->wakeups & wakeup_mask)); + + wakeup_mask &= ~xchg(&kdata->wakeups, 0); + /* flush shared variable to memory */ + smp_mb__after_atomic(); + } + if (kthread_should_stop()) + return 0; + + /* Wait for the block params to be ready */ + while (!kthread_should_stop()) { + clear_bit(EVENT_BIT(EV_BD_PARAMS), &kdata->wakeups); + /* flush shared variable to memory */ + smp_mb__after_atomic(); + + ret = block_drv->get_bd_params(app, ¶m); + if (ret == 0) { + break; + } else if (ret != -EAGAIN) { + dev_err(&prot->dev, "Failed to get params: %d\n", ret); + return ret; + } + + wait_event_killable(kdata->wait_q, + kthread_should_stop() || + (kdata->wakeups & EVENT_BIT(EV_BD_PARAMS))); + } + if (kthread_should_stop()) + return 0; + + /* Save the logical sector size; the protocol will need it later */ + sector_size = param->logical_block_size; + + size = sizeof(*ad) + strlen(param->label); + + skb_ad = alloc_skb(ret, GFP_KERNEL); + ad = (struct neuron_block_advertise *)skb_put(skb_ad, sizeof(*ad)); + memset(ad, 0, sizeof(*ad)); + ad->logical_block_size = param->logical_block_size; + ad->physical_block_size = param->physical_block_size; + ad->num_device_sectors = param->num_device_sectors; + ad->discard_max_hw_sectors = param->discard_max_hw_sectors; + ad->discard_max_sectors = param->discard_max_sectors; + ad->discard_granularity = param->discard_granularity; + ad->alignment_offset = param->alignment_offset; + ad->wc_flag = param->wc_flag; + ad->fua_flag = param->fua_flag; + memcpy(ad->uuid, param->uuid.b, sizeof(param->uuid.b)); + strlcpy(ad->label, param->label, strlen(param->label)); + if (param->read_only) + ad->flags |= NEURON_BLOCK_READONLY; + if (param->discard_zeroes_data) + ad->flags |= NEURON_BLOCK_DISCARD_ZEROES; + if (param->logical_block_size > c_in->max_size) + return -EPROTO; + if (param->logical_block_size > c_out->max_size) + return -EPROTO; + kfree(param); + + while (!kthread_should_stop()) { + clear_bit(CHANNEL_BIT(C_OUT), &kdata->wakeups); + /* flush shared variable to memory */ + smp_mb__after_atomic(); + + ret = c_out_drv->send_msg(c_out, skb_ad); + if (ret == 0) { + break; + } else if (ret != -EAGAIN) { + dev_err(&prot->dev, "Failed to send params: %d\n", ret); + kfree_skb(skb_ad); + return ret; + } + wait_event_killable(kdata->wait_q, + kthread_should_stop() || + (kdata->wakeups & CHANNEL_BIT(C_OUT))); + } + consume_skb(skb_ad); + if (kthread_should_stop()) + return 0; + + /* Pre-allocate the request SKB. This can be safely reused + * because it is never sent to any asynchronous function outside + * this driver. + */ + skb_req = __alloc_skb(sizeof(*req), GFP_KERNEL, SKB_ALLOC_RX, + NUMA_NO_NODE); + if (!skb_req) { + ret = -ENOMEM; + goto fail; + } + req = (struct neuron_block_req *)skb_put(skb_req, sizeof(*req)); + + /* Pre-allocate a response SKB. This will be reallocated after each + * successful send; we can't reuse it in case the channel driver has + * cloned it. + */ + skb_resp = alloc_skb(sizeof(*resp), GFP_KERNEL); + if (!skb_resp) { + ret = -ENOMEM; + goto fail; + } + resp = (struct neuron_block_resp *)skb_put(skb_resp, sizeof(*resp)); + memset(resp, 0, sizeof(*resp)); + + /* Main request handling loop */ + wakeup_mask = 0; + while (!kthread_should_stop()) { + unsigned long old_wakeups, consumed_wakeups; + + /* Try to obtain a request if we don't have one */ + if (!req_valid && !(wakeup_mask & CHANNEL_BIT(C_IN))) { + size_t size; + + /* New request */ + WARN_ON(skb_req_data); + + ret = c_in_drv->receive_msg(c_in, skb_req); + if (ret == -EAGAIN) { + wakeup_mask |= CHANNEL_BIT(C_IN); + continue; + } else if (ret < 0) { + dev_err(&prot->dev, + "%d Recv error %d\n", __LINE__, ret); + goto fail; + } else if (ret < sizeof(*req)) { + dev_err(&prot->dev, "Truncated request\n"); + ret = -EBADMSG; + goto fail; + } + req_valid = true; + + if (req->req_type == NEURON_BLOCK_REQUEST_WRITE) + size = req->sectors * sector_size; + else if (req->req_type == + NEURON_BLOCK_REQUEST_WRITE_SAME) + size = sector_size; + else + size = 0; + + if (size > 0) { + WARN_ON(skb_req_data); + skb_req_data = neuron_alloc_pskb(size, + GFP_KERNEL); + if (IS_ERR(skb_req_data)) { + ret = PTR_ERR(skb_req_data); + goto fail; + } + req_data_pos = 0; + req_ready = false; + } else { + req_ready = true; + } + continue; + } + + /* Try to receive request data if we are waiting for any */ + if (req_valid && !req_ready && + !(wakeup_mask & CHANNEL_BIT(C_IN))) { + struct buffer_list buf; + + WARN_ON(!skb_req_data); + + buf.head = skb_req_data; + buf.offset = req_data_pos; + buf.size = min_t(size_t, c_in->max_size, + skb_req_data->len - req_data_pos); + buf.size = round_down(buf.size, sector_size); + + WARN_ON(!buf.size); + + ret = c_in_drv->receive_msgv(c_in, buf); + if (ret == -EAGAIN) { + wakeup_mask |= CHANNEL_BIT(C_IN); + continue; + } else if (ret < 0) { + dev_err(&prot->dev, + "%d Recv error %d\n", __LINE__, ret); + goto fail; + } else if (!ret) { + dev_err(&prot->dev, "Empty payload\n"); + ret = -EBADMSG; + goto fail; + } else if (ret % sector_size) { + dev_err(&prot->dev, "Bad payload length %d\n", + ret); + ret = -EBADMSG; + goto fail; + } + req_data_pos += ret; + + if (req_data_pos == skb_req_data->len) + req_ready = true; + continue; + } + + /* Try to handle a request if there is one ready */ + if (req_valid && req_ready) { + ret = handle_request(block_drv, app, req, skb_req_data); + skb_req_data = NULL; + + if (ret < 0) { + dev_err(&prot->dev, + "Request failed: %d\n", ret); + ret = -EBADMSG; + goto fail; + } + + req_data_pos = 0; + req_ready = false; + req_valid = false; + continue; + } + + /* Try to obtain a response from the application layer. */ + if (!resp_valid && !(wakeup_mask & EVENT_BIT(EV_RESPONSE))) { + enum neuron_block_resp_status resp_status; + + WARN_ON(skb_resp_data); + + ret = block_drv->get_response(app, &resp->resp_id, + &resp_status, &skb_resp_data); + + if (ret == -EAGAIN) { + wakeup_mask |= EVENT_BIT(EV_RESPONSE); + continue; + } else if (ret < 0) { + dev_err(&prot->dev, "Can't get response: %d\n", + ret); + goto fail; + } + + resp->resp_status = (u16)resp_status; + resp_valid = true; + resp_sent_header = false; + resp_data_pos = 0; + continue; + } + + /* If we have a response header, try to send it. */ + if (resp_valid && !resp_sent_header && + !(wakeup_mask & CHANNEL_BIT(C_OUT))) { + ret = c_out_drv->send_msg(c_out, skb_resp); + if (ret == -EAGAIN) { + wakeup_mask |= CHANNEL_BIT(C_OUT); + continue; + } else if (ret < 0) { + dev_err(&prot->dev, "Can't send response: %d\n", + ret); + goto fail; + } + + /* Consume and replace the response skb */ + consume_skb(skb_resp); + skb_resp = alloc_skb(sizeof(*resp), GFP_KERNEL); + if (!skb_resp) { + ret = -ENOMEM; + goto fail; + } + resp = (struct neuron_block_resp *)skb_put(skb_resp, + sizeof(*resp)); + memset(resp, 0, sizeof(*resp)); + + /* Is there any data to send? */ + if (skb_resp_data) + resp_sent_header = true; + else + resp_valid = false; + continue; + } + + /* If we have response data, try to send it. */ + if (resp_valid && resp_sent_header && + !(wakeup_mask & CHANNEL_BIT(C_OUT))) { + struct buffer_list buf; + + WARN_ON(!skb_resp_data); + + buf.head = skb_resp_data; + buf.offset = resp_data_pos; + buf.size = min_t(size_t, c_out->max_size, + skb_resp_data->len - resp_data_pos); + buf.size = round_down(buf.size, sector_size); + + WARN_ON(!buf.size); + + ret = c_out_drv->send_msgv(c_out, buf); + if (ret == -EAGAIN) { + wakeup_mask |= CHANNEL_BIT(C_OUT); + continue; + } else if (ret < 0) { + dev_err(&prot->dev, "Send error %d\n", ret); + goto fail; + } + resp_data_pos += buf.size; + + WARN_ON(resp_data_pos > skb_resp_data->len); + + if (resp_data_pos == skb_resp_data->len) { + resp_valid = false; + consume_skb(skb_resp_data); + skb_resp_data = NULL; + } + continue; + } + + if (WARN_ON_ONCE(!wakeup_mask)) + continue; + + dev_dbg(&prot->dev, "wait for events %#lx (now: %#lx)\n", + wakeup_mask, kdata->wakeups); + wait_event_killable(kdata->wait_q, + kthread_should_stop() || + (kdata->wakeups & wakeup_mask)); + + do { + old_wakeups = READ_ONCE(kdata->wakeups); + consumed_wakeups = old_wakeups & wakeup_mask; + } while (cmpxchg(&kdata->wakeups, old_wakeups, + old_wakeups & ~consumed_wakeups) != + old_wakeups); + /* flush shared variable to memory */ + smp_mb__after_atomic(); + wakeup_mask &= ~consumed_wakeups; + } + + consume_skb(skb_req); + consume_skb(skb_resp); + + return 0; + +fail: + if (skb_req) + kfree_skb(skb_req); + if (skb_req_data) + kfree_skb(skb_req_data); + if (skb_resp) + kfree_skb(skb_resp); + if (skb_resp_data) + kfree_skb(skb_resp_data); + + return ret; +} + +static int protocol_block_server_probe(struct neuron_protocol *protocol_dev) +{ + struct server_data *kdata; + + kdata = kzalloc(sizeof(*kdata), GFP_KERNEL); + if (!kdata) + return -ENOMEM; + + init_waitqueue_head(&kdata->wait_q); + kdata->wakeups = 0; + + dev_set_drvdata(&protocol_dev->dev, kdata); + + kdata->thread = kthread_run(protocol_block_server_thread, protocol_dev, + "%s", dev_name(&protocol_dev->dev)); + + return 0; +} + +static void protocol_block_server_remove(struct neuron_protocol *protocol_dev) +{ + struct server_data *kdata; + + kdata = dev_get_drvdata(&protocol_dev->dev); + kthread_stop(kdata->thread); + dev_set_drvdata(&protocol_dev->dev, NULL); + kfree(kdata); +} + +static const struct neuron_channel_match_table channels_server_block[] = { + [C_IN] = {NEURON_CHANNEL_MESSAGE_QUEUE, NEURON_CHANNEL_RECEIVE}, + [C_OUT] = {NEURON_CHANNEL_MESSAGE_QUEUE, NEURON_CHANNEL_SEND}, +}; + +static const char * const processes_server_block[] = {"server"}; + +struct neuron_protocol_driver protocol_server_block_driver = { + .channel_count = CHANNEL_COUNT, + .channels = channels_server_block, + .process_count = 1, + .processes = processes_server_block, + .driver = { + .name = DRIVER_NAME_SERVER, + .owner = THIS_MODULE, + .of_match_table = protocol_block_server_match, + }, + .probe = protocol_block_server_probe, + .remove = protocol_block_server_remove, + .channel_wakeup = protocol_block_channel_wakeup, + .app_wakeup = protocol_block_app_wakeup, +}; +EXPORT_SYMBOL(protocol_server_block_driver); + +static int __init protocol_server_block_init(void) +{ + int ret; + + ret = neuron_register_protocol_driver(&protocol_server_block_driver); + if (ret < 0) { + pr_err("Failed to register driver\n"); + return ret; + } + return 0; +} + +static void __exit protocol_server_block_exit(void) +{ + neuron_unregister_protocol_driver(&protocol_server_block_driver); +} + +module_init(protocol_server_block_init); +module_exit(protocol_server_block_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Neuron block server protocol driver"); diff --git a/net/neuron/protocol_bus.c b/net/neuron/protocol_bus.c new file mode 100644 index 000000000000..999e3f0fd0dd --- /dev/null +++ b/net/neuron/protocol_bus.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */ + +/* Neuron protocol bus type driver + * + * This driver creates a protocol bus type device and registers a protocol + * driver. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +static int protocol_match(struct device *dev, struct device_driver *driver) +{ + struct neuron_protocol *protocol_dev = to_neuron_protocol(dev); + struct neuron_protocol_driver *protocol_drv = + to_neuron_protocol_driver(driver); + int i; + + /* The list of channel types and directions must exactly match those + * expected by the driver. + */ + if (protocol_dev->channel_count != protocol_drv->channel_count) + return 0; + + for (i = 0; i < protocol_drv->channel_count; i++) { + if (protocol_dev->channels[i]->type != + protocol_drv->channels[i].type) + return 0; + + if (protocol_dev->channels[i]->direction != + protocol_drv->channels[i].direction) + return 0; + } + + /* The list of processes must exactly match those implemented by the + * driver. Typically this will just be "client" or "server", but some + * protocols might have multiple processes. + */ + if (protocol_dev->process_count != protocol_drv->process_count) + return 0; + for (i = 0; i < protocol_dev->process_count; i++) + if (strcmp(protocol_dev->processes[i], + protocol_drv->processes[i])) + return 0; + + /* The protocol driver requires the application to be bound by a + * specific application driver, but that may not have happened yet so + * we can't check it here. It's checked in probe instead. + * + * It should not be possible to bind the wrong app driver anyway, + * unless the device tree has incorrect compatible strings. That is + * unlikely enough that we can live with the added overhead of not + * failing until probe. + */ + + if (of_driver_match_device(dev, driver)) + return 1; + + return 0; +} + +static int protocol_probe(struct device *dev) +{ + int i, ret; + struct neuron_protocol *protocol_dev = to_neuron_protocol(dev); + struct neuron_protocol_driver *protocol_drv = + to_neuron_protocol_driver(dev->driver); + struct neuron_app_driver *app_drv = + to_neuron_app_driver(protocol_dev->application->dev.driver); + + /* Fail if the application driver is not the expected one. */ + if (protocol_drv != app_drv->protocol_driver) + return -EINVAL; + + ret = 0; + if (protocol_drv->probe) + ret = protocol_drv->probe(protocol_dev); + else if (dev->driver->probe) + ret = dev->driver->probe(dev); + + if (ret < 0) + return ret; + + for (i = 0; i < protocol_dev->channel_count; i++) + rcu_assign_pointer(protocol_dev->channels[i]->protocol_drv, + protocol_drv); + + if (ret >= 0) + rcu_assign_pointer(protocol_dev->application->protocol_drv, + protocol_drv); + + if (app_drv->start) + app_drv->start(protocol_dev->application); + + return ret; +} + +static int protocol_remove(struct device *dev) +{ + int i; + struct neuron_protocol *protocol_dev = to_neuron_protocol(dev); + struct neuron_protocol_driver *protocol_drv = + to_neuron_protocol_driver(dev->driver); + + /* Disconnect the channels and application, and wait for them to + * notice + */ + for (i = 0; i < protocol_dev->channel_count; i++) + rcu_assign_pointer(protocol_dev->channels[i]->protocol_drv, + NULL); + rcu_assign_pointer(protocol_dev->application->protocol_drv, NULL); + + synchronize_rcu(); + + if (protocol_drv->remove) + protocol_drv->remove(protocol_dev); + else if (dev->driver->remove) + dev->driver->remove(dev); + + return 0; +} + +static struct bus_type protocol_bus_type = { + .name = "neuron_protocol", + .match = protocol_match, + .probe = protocol_probe, + .remove = protocol_remove, +}; + +static void protocol_dev_release(struct device *dev) +{ + int i; + struct neuron_protocol *protocol_dev = to_neuron_protocol(dev); + + for (i = 0; i < protocol_dev->channel_count; i++) + put_device(&protocol_dev->channels[i]->dev); + + kfree(protocol_dev->processes); + kfree(protocol_dev); +} + +struct neuron_protocol *neuron_protocol_add(struct device_node *node, + unsigned int channel_count, + struct neuron_channel **channels, + struct device *parent, + struct neuron_application *app_dev) +{ + struct neuron_protocol *protocol_dev; + const char *process; + struct property *prop; + int err; + int i; + + protocol_dev = kzalloc(sizeof(*protocol_dev) + + (sizeof(struct neuron_channel *) * channel_count), + GFP_KERNEL); + if (!protocol_dev) + return ERR_PTR(-ENOMEM); + device_initialize(&protocol_dev->dev); + + protocol_dev->dev.of_node = node; + protocol_dev->dev.bus = &protocol_bus_type; + protocol_dev->dev.parent = parent; + protocol_dev->dev.release = protocol_dev_release; + dev_set_name(&protocol_dev->dev, "%s:%s", dev_name(parent), + node->name); + + /* Save process names */ + err = of_property_count_strings(node, "processes"); + if (err < 0) + goto fail_properties; + protocol_dev->process_count = err; + protocol_dev->processes = kcalloc(protocol_dev->process_count, + sizeof(const char *), + GFP_KERNEL); + if (!protocol_dev->processes) + goto fail_properties; + i = 0; + of_property_for_each_string(node, "processes", prop, process) { + protocol_dev->processes[i] = process; + i++; + } + + /* Save channel pointers */ + protocol_dev->channel_count = 0; + for (i = 0; i < channel_count; i++) { + protocol_dev->channels[i] = &(*channels[i]); + get_device(&(*channels[i]).dev); + protocol_dev->channel_count++; + } + + /* Save application pointer in protocol */ + protocol_dev->application = app_dev; + get_device(&app_dev->dev); + + if (!device_link_add(&protocol_dev->dev, &app_dev->dev, 0)) { + err = -ENODEV; + goto fail_link; + } + + err = device_add(&protocol_dev->dev); + if (err) + goto fail_device_add; + + return protocol_dev; + +fail_device_add: +fail_link: +fail_properties: + put_device(&protocol_dev->dev); + return ERR_PTR(err); +} +EXPORT_SYMBOL(neuron_protocol_add); + +int neuron_register_protocol_driver(struct neuron_protocol_driver *drv) +{ + int ret; + + drv->driver.bus = &protocol_bus_type; + ret = driver_register(&drv->driver); + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL(neuron_register_protocol_driver); + +void neuron_unregister_protocol_driver(struct neuron_protocol_driver *drv) +{ + driver_unregister(&drv->driver); +} +EXPORT_SYMBOL(neuron_unregister_protocol_driver); + +static int __init protocol_bus_init(void) +{ + int ret; + + ret = bus_register(&protocol_bus_type); + if (ret < 0) { + pr_err("Unable to register bus\n"); + return ret; + } + + return 0; +} + +static void protocol_bus_exit(void) +{ + bus_unregister(&protocol_bus_type); +} + +subsys_initcall(protocol_bus_init); +module_exit(protocol_bus_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Neuron protocol bus module");