A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-30 19:51:31 -04:00
Code Issues Projects Releases Packages Wiki Activity

net: Add Neuron Framework

Browse source 
Neuron is a device-sharing framework which is used by guests of the
haven hypervisor to serve or access shared I/O devices and other
inter-VM services.

There are three main layers that make up a neuron service.
channel - the physical layer transport that uses the hypervisor
          provided transports.

protocol - defines the syntax and semantics to virtualize a specific
           device across VMs. Block and Net are examples of protocols.

application - integrates the neuron service components into the rest of
              the system. There would be front and back end application
              drivers for the net protocol.

Change-Id: Ic7278fdaee1cd30147e91e1126643bce79c05e52
Signed-off-by: Chris Lew <[email protected]>
This commit is contained in:
Chris Lew 2020-03-30 21:44:24 -07:00
commit 6493564fe3
25 changed files with 4839 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

26
net/neuron/Kconfig Normal file
View file

@ -0,0 +1,26 @@
# SPDX-License-Identifier: GPL-2.0-only
menu "Neuron Device-sharing Framework"
config NEURON
tristate "Support for Neuron device-sharing framework"
help
This option enables the Neuron device-sharing framework. It is used
by guests of the haven hypervisors to serve or access shared I/O
devices and other inter-VM services. The Neuron framework is composed
of three buses that represent different layers in the framework
(channel, protocol, application).
config NEURON_SERVICE
tristate "Static configuration of services from the device tree"
depends on OF
select NEURON
help
This option enables a platform driver that can construct a Neuron
service stack from a description in a device tree. The device tree
node should contain a channel, application and protocol subnode.
Multiple services can run on a system.
source "net/neuron/channel/Kconfig"
source "net/neuron/protocol/Kconfig"
endmenu

12
net/neuron/Makefile Normal file
View file

@ -0,0 +1,12 @@
# SPDX-License-Identifier: GPL-2.0-only
# bus types
obj-$(CONFIG_NEURON) += channel_bus.o
obj-$(CONFIG_NEURON) += protocol_bus.o
obj-$(CONFIG_NEURON) += application_bus.o
# platform device
obj-$(CONFIG_NEURON_SERVICE) += neuron_service.o
# drivers
obj-y += channel/
obj-y += protocol/

139
net/neuron/application_bus.c Normal file
View file

@ -0,0 +1,139 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
/* Neuron application bus type driver
*
* This driver creates an application bus type device and registers application
* driver.
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/of_device.h>
#include <linux/version.h>
#include <linux/neuron.h>
static int app_match(struct device *dev, struct device_driver *driver)
{
if (of_driver_match_device(dev, driver))
return 1;
return 0;
}
static int app_probe(struct device *dev)
{
int ret;
struct neuron_application *app_dev = to_neuron_application(dev);
struct neuron_app_driver *app_drv = to_neuron_app_driver(dev->driver);
ret = 0;
if (app_drv->probe)
ret = app_drv->probe(app_dev);
else if (dev->driver->probe)
ret = dev->driver->probe(dev);
return ret;
}
static int app_remove(struct device *dev)
{
struct neuron_application *app_dev = to_neuron_application(dev);
struct neuron_app_driver *app_drv = to_neuron_app_driver(dev->driver);
if (app_drv->remove)
app_drv->remove(app_dev);
else if (dev->driver->remove)
dev->driver->remove(dev);
return 0;
}
static struct bus_type app_bus_type = {
.name = "neuron_application",
.match = app_match,
.probe = app_probe,
.remove = app_remove,
};
static void app_dev_release(struct device *dev)
{
struct neuron_application *app_dev = to_neuron_application(dev);
put_device(&app_dev->protocol->dev);
kfree(app_dev);
}
struct neuron_application *neuron_app_add(struct device_node *node,
struct device *parent)
{
struct neuron_application *app_dev;
int err;
app_dev = kzalloc(sizeof(*app_dev), GFP_KERNEL);
if (!app_dev)
return ERR_PTR(-ENOMEM);
device_initialize(&app_dev->dev);
app_dev->dev.of_node = node;
app_dev->dev.bus = &app_bus_type;
app_dev->dev.parent = parent;
app_dev->dev.release = app_dev_release;
dev_set_name(&app_dev->dev, "%s:%s", dev_name(parent), node->name);
err = device_add(&app_dev->dev);
if (err)
goto fail_device_add;
return app_dev;
fail_device_add:
put_device(&app_dev->dev);
return ERR_PTR(err);
}
EXPORT_SYMBOL(neuron_app_add);
int neuron_register_app_driver(struct neuron_app_driver *drv)
{
int ret;
drv->driver.bus = &app_bus_type;
ret = driver_register(&drv->driver);
if (ret)
return ret;
return 0;
}
EXPORT_SYMBOL(neuron_register_app_driver);
void neuron_unregister_app_driver(struct neuron_app_driver *drv)
{
driver_unregister(&drv->driver);
}
EXPORT_SYMBOL(neuron_unregister_app_driver);
static int __init app_bus_init(void)
{
int ret;
ret = bus_register(&app_bus_type);
if (ret < 0) {
pr_err("Unable to register bus\n");
return ret;
}
return 0;
}
static void app_bus_exit(void)
{
bus_unregister(&app_bus_type);
}
subsys_initcall(app_bus_init);
module_exit(app_bus_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Neuron application bus module");

10
net/neuron/channel/Kconfig Normal file
View file

@ -0,0 +1,10 @@
# SPDX-License-Identifier: GPL-2.0-only
config NEURON_CH_HAVEN
tristate "Shared memory channel drivers for Haven guests"
depends on OF && HAVEN_DRIVERS
help
This option enables receive and send Neuron channel drivers that use
a shared memory buffer and a pair of notification objects to
communicate with another VM. This is the Haven version.
If unsure, say Y.

3
net/neuron/channel/Makefile Normal file
View file

@ -0,0 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_NEURON_CH_HAVEN) += ch_haven_recv.o
obj-$(CONFIG_NEURON_CH_HAVEN) += ch_haven_send.o

461
net/neuron/channel/ch_haven_recv.c Normal file
View file

@ -0,0 +1,461 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/version.h>
#include <linux/fs.h>
#include <linux/cdev.h>
#include <linux/of.h>
#include <linux/of_irq.h>
#include <linux/of_address.h>
#include <linux/ioport.h>
#include <linux/slab.h>
#include <linux/nospec.h>
#include <linux/kthread.h>
#include <linux/neuron.h>
#include <asm-generic/barrier.h>
#include <linux/haven/hh_dbl.h>
#include "ch_mq_shmem_common.h"
#define CHANNEL_VERSION NEURON_SHMEM_CHANNEL_V1
#define CH_DBL_MASK 0x1
/* Cache line size, 64 bytes is picked since it is the largest one currently */
#define CACHE_LINE_SIZE 64
/* Messages has to be aligned for performance's concern. */
#define ALIGNMENT_BYTES 16
static inline void shm_clear_header(struct neuron_mq_data_priv *priv)
{
struct neuron_shmem_channel_header *hdr = priv->base;
struct neuron_msg_queue *msgq = &priv->msgq;
/* clear space_for_next field */
hdr->space_for_next = 0;
/* clear tail offset */
*msgq->tailp = 0;
/* Make sure memory writing is finished */
mb();
/* Set tail_offset to correct value to indicate
* to the sender that we are synced.
*/
smp_store_release(&hdr->tail_offset, CACHE_LINE_SIZE);
}
/* Read data from the ring buffer */
static inline int ring_read_msg(struct neuron_msg_queue *msgq,
struct buffer_list dest, size_t n)
{
int ret;
void *src;
off_t offset = msgq->offset;
offset = round_up(offset + PACKET_HEADER_SIZE,
msgq->message_alignment);
if (offset >= msgq->ring_buffer_len)
offset -= msgq->ring_buffer_len;
src = msgq->ring_buffer_p + offset;
if (offset + n <= msgq->ring_buffer_len) {
ret = skb_store_bits(dest.head, dest.offset, src, n);
if (ret)
return ret;
} else {
size_t n_1 = msgq->ring_buffer_len - offset;
size_t n_2 = n - n_1;
ret = skb_store_bits(dest.head, dest.offset, src, n_1);
if (ret)
return ret;
ret = skb_store_bits(dest.head, dest.offset + n_1,
msgq->ring_buffer_p, n_2);
if (ret)
return ret;
}
offset = round_up(offset + n, msgq->message_alignment);
if (offset >= msgq->ring_buffer_len)
offset -= msgq->ring_buffer_len;
msgq->offset = offset;
return 0;
}
static inline int channel_hh_kick(struct neuron_mq_data_priv *priv)
{
hh_dbl_flags_t dbl_mask = CH_DBL_MASK;
int ret;
ret = hh_dbl_send(priv->tx_dbl, &dbl_mask);
if (ret)
pr_err("failed to raise virq to the sender %d\n", ret);
return ret;
}
static ssize_t channel_hh_receivev(struct neuron_channel *channel_dev,
struct buffer_list buf)
{
ssize_t ret;
size_t left_space, new_left_space, space_for_next, len;
off_t head, tail;
struct neuron_msg_hdr *msg;
struct neuron_mq_data_priv *priv = dev_get_drvdata(&channel_dev->dev);
struct neuron_msg_queue *msgq = &priv->msgq;
/* read shared variable from memory */
if (unlikely(!smp_load_acquire(&priv->synced)))
return -EAGAIN;
/* Get head offset from the shared header */
head = smp_load_acquire(msgq->headp);
tail = msgq->offset;
/* The sender rebooting has been detected. */
if (unlikely(head >= msgq->ring_buffer_len)) {
dev_warn(&channel_dev->dev,
"The sender rebooted. Start to sync again!\n");
return -ECONNRESET;
}
if (head == tail) {
/* Set notification flag so that the sender can wake me up. */
dev_dbg(&channel_dev->dev, "empty buffer!\n");
return -EAGAIN;
}
msg = (struct neuron_msg_hdr *)((char *)msgq->ring_buffer_p + tail);
len = msg->size + 1;
WARN_ON(len > S32_MAX);
if (len > buf.size) {
dev_dbg(&channel_dev->dev, "message too long to fill");
return -EMSGSIZE;
}
ret = ring_read_msg(msgq, buf, len);
if (ret)
return ret;
/* Publish to the shared header */
smp_store_release(msgq->tailp, msgq->offset);
/* Making sure memory writing finished */
mb();
/* Get space_for_next from the shared header */
space_for_next = smp_load_acquire(msgq->space_for_next_p);
head = READ_ONCE(*msgq->headp);
if (head < tail)
left_space = tail - head - 1;
else
left_space = msgq->ring_buffer_len + tail - head - 1;
if (head < msgq->offset)
new_left_space = msgq->offset - head - 1;
else
new_left_space = msgq->ring_buffer_len + msgq->offset -
head - 1;
/* Wake up the sender when
* 1. There is not enough space when the sender tries to send last time
* 2. There is enough space after this receiving
*/
if (left_space < space_for_next && new_left_space >= space_for_next) {
dev_dbg(&channel_dev->dev, "Waking the sender up");
/* wake up the sender */
channel_hh_kick(priv);
}
return (ssize_t)len;
}
static ssize_t channel_hh_receive(struct neuron_channel *channel_dev,
struct sk_buff *skb)
{
struct buffer_list buf = {
.head = skb,
.offset = 0,
.size = skb->len
};
return channel_hh_receivev(channel_dev, buf);
}
static void channel_hh_cb(int irq, void *data)
{
struct neuron_mq_data_priv *priv = data;
wake_up(&priv->wait_q);
neuron_channel_wakeup(priv->dev);
}
static int msgq_init(struct neuron_mq_data_priv *priv)
{
int ret;
struct neuron_shmem_channel_header *hdr = priv->base;
struct neuron_msg_queue *msgq = &priv->msgq;
struct neuron_channel *channel = priv->dev;
msgq->offset = 0;
msgq->message_alignment = ALIGNMENT_BYTES;
msgq->headp = &hdr->head;
msgq->tailp = (u32 *)((u8 *)hdr + CACHE_LINE_SIZE);
msgq->space_for_next_p = &hdr->space_for_next;
msgq->ring_buffer_p = (void *)round_up((uintptr_t)hdr +
2 * CACHE_LINE_SIZE, msgq->message_alignment);
msgq->ring_buffer_len = resource_size(&priv->buffer) -
2 * CACHE_LINE_SIZE;
ret = channel_set_limits(channel, msgq);
if (ret)
return ret;
hdr->version = CHANNEL_VERSION;
/* Make sure version is visible when the sender sees tail_offset */
mb();
/* Set tail_offset to -1 to indicate it is unsynced.
*/
hdr->tail_offset = -1;
/* Set message_alignment field. */
hdr->message_alignment = msgq->message_alignment;
/* Set ring_buffer_offset field as 2*CACHE_LINE_SIZE */
hdr->ring_buffer_offset = (uintptr_t)msgq->ring_buffer_p -
(uintptr_t)hdr;
/* Set ring_buffer_len field */
hdr->ring_buffer_len = msgq->ring_buffer_len;
/* Set max msg size field */
hdr->max_msg_size = channel->max_size;
/* Notify the sender that the channel has been reset. */
channel_hh_kick(priv);
return 0;
}
/* Thread to sync with the sender. Note: this thread might never finishes if
* it fails to sync with the peer.
*/
static int channel_sync_thread(void *data)
{
struct neuron_shmem_channel_header *hdr;
struct neuron_mq_data_priv *priv = (struct neuron_mq_data_priv *)data;
hdr = (struct neuron_shmem_channel_header *)priv->base;
/* Waiting for head being updated by the sender. */
wait_event_killable(priv->wait_q, smp_load_acquire(&hdr->head) == -1 ||
kthread_should_stop());
if (kthread_should_stop())
return 0;
/* If the version doesn't match (it could also be case the peer is of
* version without version field), quit the sync.
*/
if (READ_ONCE(hdr->version) != CHANNEL_VERSION) {
pr_err("Mismatched channel version: Me: %u, Peer: %u\n",
CHANNEL_VERSION, hdr->version);
return -EPROTO;
}
shm_clear_header(priv);
/* Notify the sender that shared memory header has been initialized. */
channel_hh_kick(priv);
/* Waiting for the sender's readiness. */
wait_event_killable(priv->wait_q, smp_load_acquire(&hdr->head) != -1 ||
kthread_should_stop());
if (kthread_should_stop())
return 0;
/* flush shared variable to memory */
smp_store_release(&priv->synced, 1);
neuron_channel_wakeup(priv->dev);
return 0;
}
static int channel_hh_map_memory(struct neuron_mq_data_priv *priv,
struct device *dev)
{
struct device_node *np;
resource_size_t size;
struct resource *r;
int ret;
np = of_parse_phandle(dev->of_node, "shared-buffer", 0);
if (!np) {
dev_err(dev, "shared-buffer node missing!\n");
return -EINVAL;
}
ret = of_address_to_resource(np, 0, &priv->buffer);
of_node_put(np);
if (ret) {
dev_err(dev, "of_address_to_resource failed!\n");
return -EINVAL;
}
size = resource_size(&priv->buffer);
/* buffer parameters checking */
if (!priv->buffer.start || size % PAGE_SIZE) {
dev_err(dev, "invalid memory region: start:%llx, size:%llx\n",
priv->buffer.start, size);
return -EINVAL;
}
r = devm_request_mem_region(dev, priv->buffer.start, size,
dev_name(dev));
if (!r) {
dev_err(dev, "request memory region failed!\n");
return -ENXIO;
}
priv->base = ioremap_cache(priv->buffer.start, size);
if (!priv->base) {
dev_err(dev, "ioremap failed!\n");
return -ENXIO;
}
if (of_property_read_bool(dev->of_node, "qcom,primary"))
memset(priv->base, 0,
sizeof(struct neuron_shmem_channel_header));
return 0;
}
static int channel_hh_probe(struct neuron_channel *cdev)
{
struct device_node *node = cdev->dev.of_node;
struct device *dev = &cdev->dev;
struct neuron_mq_data_priv *priv;
enum hh_dbl_label dbl_label;
int ret;
if (!node)
return -ENODEV;
priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
priv->dev = cdev;
ret = channel_hh_map_memory(priv, dev);
if (ret) {
dev_err(dev, "failed to map memory %d\n", ret);
return ret;
}
/* Get outgoing haven doorbell information */
ret = of_property_read_u32(node, "haven-label", &dbl_label);
if (ret) {
dev_err(dev, "failed to read label info %d\n", ret);
goto fail_tx_dbl;
}
priv->tx_dbl = hh_dbl_tx_register(dbl_label);
if (IS_ERR_OR_NULL(priv->tx_dbl)) {
ret = PTR_ERR(priv->tx_dbl);
dev_err(dev, "failed to get haven tx dbl %d\n", ret);
goto fail_tx_dbl;
}
priv->rx_dbl = hh_dbl_rx_register(dbl_label, channel_hh_cb, priv);
if (IS_ERR_OR_NULL(priv->rx_dbl)) {
ret = PTR_ERR(priv->rx_dbl);
dev_err(dev, "failed to get haven rx dbl %d\n", ret);
goto fail_rx_dbl;
}
/* Init the shared memory header and local message queue. */
ret = msgq_init(priv);
if (ret)
goto fail_mask;
init_waitqueue_head(&priv->wait_q);
/* Start the thread for syncing with the sender. */
priv->sync_thread = kthread_run(channel_sync_thread, priv,
"recv_sync_thread");
dev_set_drvdata(&cdev->dev, priv);
return 0;
fail_mask:
hh_dbl_rx_unregister(priv->rx_dbl);
fail_rx_dbl:
hh_dbl_tx_unregister(priv->tx_dbl);
fail_tx_dbl:
iounmap(priv->base);
return ret;
}
static void channel_hh_remove(struct neuron_channel *cdev)
{
struct neuron_mq_data_priv *priv = dev_get_drvdata(&cdev->dev);
/* Stop it anyway. */
kthread_stop(priv->sync_thread);
iounmap(priv->base);
devm_release_mem_region(&cdev->dev, priv->buffer.start,
resource_size(&priv->buffer));
hh_dbl_tx_unregister(priv->tx_dbl);
hh_dbl_rx_unregister(priv->rx_dbl);
}
static const struct of_device_id channel_hh_match[] = {
{ .compatible = "qcom,neuron-channel-haven-shmem" },
{},
};
MODULE_DEVICE_TABLE(of, channel_hh_match);
static struct neuron_channel_driver channel_hh_recv_driver = {
.driver = {
.name = "ch_haven_recv",
.of_match_table = channel_hh_match,
},
.type = NEURON_CHANNEL_MESSAGE_QUEUE,
.direction = NEURON_CHANNEL_RECEIVE,
.receive_msgv = channel_hh_receivev,
.receive_msg = channel_hh_receive,
.probe = channel_hh_probe,
.remove = channel_hh_remove,
};
static int __init channel_hh_init(void)
{
int ret;
ret = neuron_register_channel_driver(&channel_hh_recv_driver);
if (ret < 0) {
pr_err("Failed to register driver:%d\n", ret);
return ret;
}
return 0;
}
static void __exit channel_hh_exit(void)
{
neuron_unregister_channel_driver(&channel_hh_recv_driver);
}
module_init(channel_hh_init);
module_exit(channel_hh_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Neuron channel haven shared memory receiver driver");

440
net/neuron/channel/ch_haven_send.c Normal file
View file

@ -0,0 +1,440 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/version.h>
#include <linux/fs.h>
#include <linux/cdev.h>
#include <linux/of.h>
#include <linux/of_irq.h>
#include <linux/of_address.h>
#include <linux/ioport.h>
#include <linux/slab.h>
#include <linux/nospec.h>
#include <linux/kthread.h>
#include <linux/neuron.h>
#include <asm-generic/barrier.h>
#include <linux/haven/hh_dbl.h>
#include "ch_mq_shmem_common.h"
#define CHANNEL_VERSION NEURON_SHMEM_CHANNEL_V1
#define CH_DBL_MASK 0x1
/* Write data to the ring buffer */
static inline void ring_write_msg(struct neuron_msg_queue *msgq,
struct buffer_list src, size_t n)
{
off_t offset = msgq->offset;
struct neuron_msg_hdr *hdr;
void *dest;
hdr = (struct neuron_msg_hdr *)(msgq->ring_buffer_p + offset);
/* Size index is from 0. */
hdr->size = n - 1;
offset = round_up(offset + PACKET_HEADER_SIZE,
msgq->message_alignment);
if (offset >= msgq->ring_buffer_len)
offset -= msgq->ring_buffer_len;
dest = msgq->ring_buffer_p + offset;
if (offset + n <= msgq->ring_buffer_len) {
skb_copy_bits(src.head, src.offset, dest, n);
} else {
size_t n_1 = msgq->ring_buffer_len - offset;
size_t n_2 = n - n_1;
skb_copy_bits(src.head, src.offset, dest, n_1);
skb_copy_bits(src.head, src.offset + n_1, msgq->ring_buffer_p,
n_2);
}
offset = round_up(offset + n, msgq->message_alignment);
if (offset >= msgq->ring_buffer_len)
offset -= msgq->ring_buffer_len;
msgq->offset = offset;
}
static inline int channel_hh_kick(struct neuron_mq_data_priv *priv)
{
hh_dbl_flags_t dbl_mask = CH_DBL_MASK;
int ret;
ret = hh_dbl_send(priv->tx_dbl, &dbl_mask);
if (ret)
pr_err("failed to raise virq to the sender %d\n", ret);
return ret;
}
static int channel_hh_sendv(struct neuron_channel *channel_dev,
struct buffer_list buf)
{
size_t space, space_needed, len;
off_t tail, head;
int prev_empty = 0;
struct neuron_mq_data_priv *priv = dev_get_drvdata(&channel_dev->dev);
struct neuron_msg_queue *msgq = &priv->msgq;
/* read shared variable from memory */
if (unlikely(!smp_load_acquire(&priv->synced)))
return -EAGAIN;
len = buf.size;
/* Nothing to send */
if (len == 0) {
dev_err(&channel_dev->dev, "len is 0\n");
return -EINVAL;
}
if (len > channel_dev->max_size) {
dev_err(&channel_dev->dev,
"buffer is oversized - len is %zd and %lu is supported\n",
len, channel_dev->max_size);
return -EMSGSIZE;
}
space_needed =
round_up(PACKET_HEADER_SIZE, msgq->message_alignment) +
round_up(len, msgq->message_alignment);
WRITE_ONCE(*msgq->space_for_next_p, space_needed);
/* Order the space_for_next write before the tail read */
mb();
/* Get the tail from the shared memory */
tail = smp_load_acquire(msgq->tailp);
head = msgq->offset;
/* The receiver rebooting has been detected. */
if (unlikely(tail >= msgq->ring_buffer_len)) {
dev_warn(&channel_dev->dev,
"The receiver rebooted. Start to sync again!\n");
return -ECONNRESET;
}
if (head < tail)
space = tail - head - 1;
else
space = msgq->ring_buffer_len + tail - head - 1;
if (space < space_needed)
return -EAGAIN;
prev_empty = msgq->offset == head ? 1 : 0;
ring_write_msg(msgq, buf, len);
/* Publish to the shared header */
smp_store_release(msgq->headp, msgq->offset);
if (prev_empty)
channel_hh_kick(priv);
return 0;
}
static int channel_hh_send(struct neuron_channel *channel_dev,
struct sk_buff *skb)
{
struct buffer_list buf = {
.head = skb,
.offset = 0,
.size = skb->len
};
return channel_hh_sendv(channel_dev, buf);
}
static void channel_hh_cb(int irq, void *data)
{
struct neuron_mq_data_priv *priv = data;
wake_up(&priv->wait_q);
neuron_channel_wakeup(priv->dev);
}
static int read_config(struct neuron_mq_data_priv *priv)
{
int ret;
u32 offset;
struct neuron_shmem_channel_header *hdr = priv->base;
struct neuron_msg_queue *msgq = &priv->msgq;
struct neuron_channel *channel = priv->dev;
/* Clear local offset. */
msgq->offset = 0;
/* Get message_alignment value. */
msgq->message_alignment = hdr->message_alignment;
/* Get ring_buffer_len value. */
msgq->ring_buffer_len = hdr->ring_buffer_len;
/* Get tail_offset, making sure the value is valid. */
offset = hdr->tail_offset;
if (offset > resource_size(&priv->buffer))
return -ECONNRESET;
offset = array_index_nospec(offset, resource_size(&priv->buffer));
msgq->tailp = (u32 *)((u8 *)hdr + offset);
msgq->space_for_next_p = &hdr->space_for_next;
/* Get ring_buffer_offset, making sure the value is valid*/
offset = hdr->ring_buffer_offset;
if (offset > resource_size(&priv->buffer))
return -ECONNRESET;
offset = array_index_nospec(offset, resource_size(&priv->buffer));
msgq->ring_buffer_p = (void *)round_up((uintptr_t)hdr + offset,
msgq->message_alignment);
/* send channel max message size is:
* 1. From device tree definition if it is not bigger than the receiver
* max message size. Or
* 2. The receiver max message size if not defined.
*/
if (channel->max_size && channel->max_size > hdr->max_msg_size) {
dev_err(&channel->dev,
"send max message size is bigger than receiver max message size!\n");
return -EINVAL;
} else if (channel->max_size == 0) {
channel->max_size = hdr->max_msg_size;
}
ret = channel_set_limits(channel, msgq);
if (ret < 0)
return ret;
/* write shared variable to memory */
smp_store_release(&priv->synced, 1);
return 0;
}
/* Thread to sync with the receiver. Note: this thread might never finishes if
* it fails to sync with the peer.
*/
static int channel_sync_thread(void *data)
{
struct neuron_mq_data_priv *priv = (struct neuron_mq_data_priv *)data;
struct neuron_shmem_channel_header *hdr;
hdr = (struct neuron_shmem_channel_header *)priv->base;
hdr->version = CHANNEL_VERSION;
hdr->tail_offset = -1;
/* Make sure the memory writing is in order. */
mb();
hdr->head = -1;
channel_hh_kick(priv);
/* Waiting for tail_offset being updated by the send. */
wait_event_killable(priv->wait_q, kthread_should_stop() ||
/* read shared variable from memory */
smp_load_acquire(&hdr->tail_offset) != -1);
if (kthread_should_stop())
return 0;
/* If the version doesn't match (it could also be case the peer is of
* version without version field), quit the sync.
*/
if (smp_load_acquire(&hdr->version) != CHANNEL_VERSION) {
pr_err("Mismatched channel version: Me: %u, Peer: %u\n",
CHANNEL_VERSION, hdr->version);
return -EPROTO;
}
/* Clear head offset to 0 */
smp_store_release(&hdr->head, 0);
if (read_config(priv)) {
pr_err("config failed\n");
return -EINVAL;
}
channel_hh_kick(priv);
neuron_channel_wakeup(priv->dev);
return 0;
}
static void msgq_init(struct neuron_mq_data_priv *priv)
{
struct neuron_shmem_channel_header *hdr = priv->base;
struct neuron_msg_queue *msgq = &priv->msgq;
msgq->headp = &hdr->head;
msgq->space_for_next_p = &hdr->space_for_next;
msgq->offset = (u32)-1;
/* Set it to -1 as UNINITIALIZED */
smp_store_release(msgq->headp, (u32)-1);
}
static int channel_hh_map_memory(struct neuron_mq_data_priv *priv,
struct device *dev)
{
struct device_node *np;
resource_size_t size;
struct resource *r;
int ret;
np = of_parse_phandle(dev->of_node, "shared-buffer", 0);
if (!np) {
dev_err(dev, "shared-buffer node missing!\n");
return -EINVAL;
}
ret = of_address_to_resource(np, 0, &priv->buffer);
of_node_put(np);
if (ret) {
dev_err(dev, "of_address_to_resource failed!\n");
return -EINVAL;
}
size = resource_size(&priv->buffer);
/* buffer parameters checking */
if (!priv->buffer.start || size % PAGE_SIZE) {
dev_err(dev, "invalid memory region: start:%llx, size:%llx\n",
priv->buffer.start, size);
return -EINVAL;
}
r = devm_request_mem_region(dev, priv->buffer.start, size,
dev_name(dev));
if (!r) {
dev_err(dev, "request memory region failed!\n");
return -ENXIO;
}
priv->base = ioremap_cache(priv->buffer.start, size);
if (!priv->base) {
dev_err(dev, "ioremap failed!\n");
return -ENXIO;
}
if (of_property_read_bool(dev->of_node, "qcom,primary"))
memset(priv->base, 0,
sizeof(struct neuron_shmem_channel_header));
return 0;
}
static int channel_hh_probe(struct neuron_channel *cdev)
{
struct device_node *node = cdev->dev.of_node;
struct device *dev = &cdev->dev;
struct neuron_mq_data_priv *priv;
enum hh_dbl_label dbl_label;
int ret;
if (!node)
return -ENODEV;
priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
priv->dev = cdev;
ret = channel_hh_map_memory(priv, dev);
if (ret)
return ret;
/* Get outgoing haven doorbell information */
ret = of_property_read_u32(node, "haven-label", &dbl_label);
if (ret) {
dev_err(dev, "failed to read label info %d\n", ret);
goto fail_tx_dbl;
}
priv->tx_dbl = hh_dbl_tx_register(dbl_label);
if (IS_ERR_OR_NULL(priv->tx_dbl)) {
ret = PTR_ERR(priv->tx_dbl);
dev_err(dev, "failed to get haven tx dbl %d\n", ret);
goto fail_tx_dbl;
}
priv->rx_dbl = hh_dbl_rx_register(dbl_label, channel_hh_cb, priv);
if (IS_ERR_OR_NULL(priv->rx_dbl)) {
ret = PTR_ERR(priv->rx_dbl);
dev_err(dev, "failed to get haven rx dbl %d\n", ret);
goto fail_rx_dbl;
}
msgq_init(priv);
init_waitqueue_head(&priv->wait_q);
/* Start the thread for syncing with the receiver. */
priv->sync_thread = kthread_run(channel_sync_thread, priv,
"send_sync_thread");
dev_set_drvdata(dev, priv);
return 0;
fail_rx_dbl:
hh_dbl_tx_unregister(priv->tx_dbl);
fail_tx_dbl:
iounmap(priv->base);
return ret;
}
static void channel_hh_remove(struct neuron_channel *cdev)
{
struct neuron_mq_data_priv *priv = dev_get_drvdata(&cdev->dev);
/* Stop it anyway. */
kthread_stop(priv->sync_thread);
iounmap(priv->base);
devm_release_mem_region(&cdev->dev, priv->buffer.start,
resource_size(&priv->buffer));
hh_dbl_tx_unregister(priv->tx_dbl);
hh_dbl_rx_unregister(priv->rx_dbl);
}
static const struct of_device_id channel_hh_match[] = {
{ .compatible = "qcom,neuron-channel-haven-shmem" },
{},
};
MODULE_DEVICE_TABLE(of, channel_hh_match);
static struct neuron_channel_driver channel_hh_send_driver = {
.driver = {
.name = "ch_haven_send",
.of_match_table = channel_hh_match,
},
.type = NEURON_CHANNEL_MESSAGE_QUEUE,
.direction = NEURON_CHANNEL_SEND,
.send_msgv = channel_hh_sendv,
.send_msg = channel_hh_send,
.probe = channel_hh_probe,
.remove = channel_hh_remove,
};
static int __init channel_hh_init(void)
{
int ret;
ret = neuron_register_channel_driver(&channel_hh_send_driver);
if (ret < 0) {
pr_err("Failed to register driver: %d\n", ret);
return ret;
}
return 0;
}
static void __exit channel_hh_exit(void)
{
neuron_unregister_channel_driver(&channel_hh_send_driver);
}
module_init(channel_hh_init);
module_exit(channel_hh_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Neuron channel haven shared memory send driver");

142
net/neuron/channel/ch_mq_shmem_common.h Normal file
View file

@ -0,0 +1,142 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
/* Neuron Message Queue transport layer header file */
#ifndef __NEURON_MQ_H
#include <linux/sched.h>
#include <linux/wait.h>
/* Current version number */
#define NEURON_SHMEM_CHANNEL_V1 0xcafe0001
/* Shared Memory header definition.
* It locates on the beginning of the shared memory.
*/
struct neuron_shmem_channel_header {
/* channel header version */
u32 version;
/* field for keeping head value, which is an offset value
* against the ring buffer
*/
u32 head;
/* tail offset */
u32 tail_offset;
/* field for the expecting length of the next message.*/
u32 space_for_next;
/* Maximum message size */
u32 max_msg_size;
/* ring buffer offset */
u32 ring_buffer_offset;
/* ring buffer lenghth */
u32 ring_buffer_len;
/* message alignment bytes */
u16 message_alignment;
/* notification flags. Currently unused. */
u16 notification_flags;
} __packed;
/* Message header. The message payload is somewhere aligned with
* message_alignment field after this header.
*/
struct neuron_msg_hdr {
u32 size; // size
} __packed;
/* Packet header size*/
#define PACKET_HEADER_SIZE sizeof(struct neuron_msg_hdr)
/* Message Queue definition */
struct neuron_msg_queue {
/* Point to field head in shared memory header */
u32 *headp;
/* Point to tail offset field in shared memory header */
u32 *tailp;
/* Point to space_for_next field in shared memory header */
u32 *space_for_next_p;
/* ring buffer address */
void *ring_buffer_p;
/* ring buffer length */
u32 ring_buffer_len;
/* message alignment bytes */
u16 message_alignment;
/* current local offset. It is the head offset for the sender
* and the tail offset for the receiver.
*/
u32 offset;
};
/* A struct for driver private data */
struct neuron_mq_data_priv {
/* outgoing vIRQ */
u32 virq_line;
/* incoming vIRQ */
u32 virq;
/* unused at the moment */
atomic64_t virq_payload;
/* A counter to calculate the interrupt received. */
u32 interrupt_counter;
/* haven tx doorbell descriptor */
void *tx_dbl;
/* haven rx doorbell descriptor */
void *rx_dbl;
/* pointer to the device structure */
struct neuron_channel *dev;
/* shared memory mapped address */
void *base;
/* shared memory resource */
struct resource buffer;
/* Flag to show whether the channel is synced. */
int synced;
/* The pointer of thread doing the sync work. */
struct task_struct *sync_thread;
/* Waiting queue for sync thread */
wait_queue_head_t wait_q;
/* Message Queue */
struct neuron_msg_queue msgq;
};
/* Setting limits for each side. */
static inline int channel_set_limits(struct neuron_channel *channel,
struct neuron_msg_queue *msgq)
{
/* Default limit is one message filling the whole buffer */
if (!channel->max_size && !channel->queue_length)
channel->queue_length = 1;
/* Calculate max size based on given queue length */
if (!channel->max_size)
channel->max_size =
round_down((msgq->ring_buffer_len - 1) /
channel->queue_length,
msgq->message_alignment) -
round_up(PACKET_HEADER_SIZE,
msgq->message_alignment);
/* Calculate queue length based on given max size */
if (!channel->queue_length)
channel->queue_length = (msgq->ring_buffer_len - 1) /
(round_up(PACKET_HEADER_SIZE,
msgq->message_alignment) +
round_up(channel->max_size,
msgq->message_alignment));
/* Assert that max_size and queue_length are sane. */
if ((size_t)msgq->ring_buffer_len - 1 <
channel->queue_length *
(size_t)(round_up(channel->max_size,
msgq->message_alignment) +
round_up(PACKET_HEADER_SIZE,
msgq->message_alignment))) {
dev_err(&channel->dev,
"ring buf size %zu too small for %u * %zu messages\n",
(size_t)msgq->ring_buffer_len,
(unsigned int)channel->queue_length,
channel->max_size);
return -EINVAL;
}
return 0;
}
#endif

216
net/neuron/channel_bus.c Normal file
View file

@ -0,0 +1,216 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
/* Neuron channel bus type driver
*
* This driver creates a channel bus type device and registers a channel driver.
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/of_device.h>
#include <linux/version.h>
#include <linux/neuron.h>
static int channel_match(struct device *dev, struct device_driver *driver)
{
struct neuron_channel *channel = to_neuron_channel(dev);
struct neuron_channel_driver *channel_drv =
to_neuron_channel_driver(driver);
/* The channel's type must match the driver's. */
if (channel->type != channel_drv->type)
return 0;
/* The driver must be able to drive the channel in the required
* direction(s).
*/
if ((channel->direction & channel_drv->direction) !=
channel->direction)
return 0;
if (of_driver_match_device(dev, driver))
return 1;
return 0;
}
static int channel_probe(struct device *dev)
{
struct neuron_channel *channel_dev = to_neuron_channel(dev);
struct neuron_channel_driver *channel_drv =
to_neuron_channel_driver(dev->driver);
if (channel_drv->probe)
return channel_drv->probe(channel_dev);
else if (dev->driver->probe)
return dev->driver->probe(dev);
return 0;
}
static int channel_remove(struct device *dev)
{
struct neuron_channel *channel_dev = to_neuron_channel(dev);
struct neuron_channel_driver *channel_drv =
to_neuron_channel_driver(dev->driver);
if (channel_drv->remove)
channel_drv->remove(channel_dev);
else if (dev->driver->remove)
dev->driver->remove(dev);
return 0;
}
static struct bus_type channel_bus_type = {
.name = "neuron_channel",
.match = channel_match,
.probe = channel_probe,
.remove = channel_remove,
};
static void channel_dev_release(struct device *dev)
{
struct neuron_channel *channel_dev = to_neuron_channel(dev);
put_device(&channel_dev->protocol->dev);
kfree(channel_dev);
}
struct neuron_channel *neuron_channel_add(struct device_node *node,
struct device *parent)
{
struct neuron_channel *channel_dev;
const char *str;
int reg = 0;
int err;
channel_dev = kzalloc(sizeof(*channel_dev), GFP_KERNEL);
if (!channel_dev)
return ERR_PTR(-ENOMEM);
device_initialize(&channel_dev->dev);
channel_dev->dev.of_node = node;
channel_dev->dev.bus = &channel_bus_type;
channel_dev->dev.parent = parent;
channel_dev->dev.release = channel_dev_release;
err = of_property_read_u32(node, "reg", &reg);
if (err < 0) {
dev_err(parent, "channel %s has no reg property\n",
node->full_name);
goto fail_properties;
}
channel_dev->id = reg;
err = of_property_read_string(node, "direction", &str);
if (err < 0) {
dev_err(parent, "channel %d: channel direction is undefined\n",
reg);
goto fail_properties;
}
err = -EINVAL;
if (!strcmp(str, "send")) {
channel_dev->direction = NEURON_CHANNEL_SEND;
} else if (!strcmp(str, "receive")) {
channel_dev->direction = NEURON_CHANNEL_RECEIVE;
} else if (!strcmp(str, "both")) {
channel_dev->direction = NEURON_CHANNEL_BIDIRECTIONAL;
} else {
dev_err(parent, "channel %d: bad channel direction \"%s\"\n",
reg, str);
goto fail_properties;
}
err = of_property_read_string(node, "class", &str);
if (err < 0) {
dev_err(parent, "channel %d: channel type is undefined\n",
reg);
goto fail_properties;
}
err = -EINVAL;
if (!strcmp(str, "message-queue")) {
u64 max_size = 0;
u32 queue_length = 0;
channel_dev->type = NEURON_CHANNEL_MESSAGE_QUEUE;
/* If these reads fail, we let the driver decide */
of_property_read_u64(node, "max-size", &max_size);
channel_dev->max_size = (size_t)max_size;
of_property_read_u32(node, "queue-length", &queue_length);
channel_dev->queue_length = (unsigned int)queue_length;
} else if (!strcmp(str, "notification")) {
channel_dev->type = NEURON_CHANNEL_NOTIFICATION;
} else if (!strcmp(str, "shared-memory")) {
channel_dev->type = NEURON_CHANNEL_SHARED_MEMORY;
} else {
dev_err(parent, "channel %d: unknown channel type \"%s\"\n",
reg, str);
goto fail_properties;
}
dev_set_name(&channel_dev->dev, "%s:%s%d", dev_name(parent),
node->name, reg);
err = device_add(&channel_dev->dev);
if (err)
goto fail_device_add;
return channel_dev;
fail_device_add:
fail_properties:
put_device(&channel_dev->dev);
kfree(channel_dev);
return ERR_PTR(err);
}
EXPORT_SYMBOL(neuron_channel_add);
int neuron_register_channel_driver(struct neuron_channel_driver *drv)
{
int ret;
drv->driver.bus = &channel_bus_type;
ret = driver_register(&drv->driver);
if (ret)
return ret;
return 0;
}
EXPORT_SYMBOL(neuron_register_channel_driver);
void neuron_unregister_channel_driver(struct neuron_channel_driver *drv)
{
driver_unregister(&drv->driver);
}
EXPORT_SYMBOL(neuron_unregister_channel_driver);
static int __init channel_bus_init(void)
{
int ret;
ret = bus_register(&channel_bus_type);
if (ret < 0) {
pr_err("Unable to register bus\n");
return ret;
}
return 0;
}
static void channel_bus_exit(void)
{
bus_unregister(&channel_bus_type);
}
subsys_initcall(channel_bus_init);
module_exit(channel_bus_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Neuron channel bus module");

204
net/neuron/neuron_service.c Normal file
View file

@ -0,0 +1,204 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
/* Neuron service driver
*
* This driver parses DT description of a Neuron service and creates protocol,
* application, and channel devices based on the DT nodes.
*/
#include <linux/init.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/version.h>
#include <linux/fs.h>
#include <linux/cdev.h>
#include <linux/of.h>
#include <linux/slab.h>
#include "neuron_service.h"
#define DRIVER_NAME "neuron-service"
#define DEVICE_NAME "neuron-service"
#ifndef CONFIG_OF
#error "neuron service driver only supported on device tree kernels"
#endif
static const struct of_device_id neuron_service_match[] = {
{
.compatible = "qcom,neuron-service",
},
{},
};
MODULE_DEVICE_TABLE(of, neuron_service_match);
static int neuron_service_probe(struct platform_device *pdev)
{
struct neuron_service *neuron_serv;
static struct device_node *node;
static struct device_node *pnode;
int count = 0;
int err;
int i = 0;
node = NULL;
pnode = pdev->dev.of_node;
for_each_child_of_node(pnode, node)
if (node->name && (of_node_cmp(node->name, "channel") == 0))
count++;
if (!count) {
err = -ENODEV;
goto fail_find_channels;
}
neuron_serv = kzalloc(sizeof(*neuron_serv) +
sizeof(struct neuron_channel *) * count,
GFP_KERNEL);
if (!neuron_serv) {
err = -ENOMEM;
goto fail_alloc_neuron_serv;
}
neuron_serv->channel_count = count;
node = NULL;
for_each_child_of_node(pnode, node) {
if (node->name && (of_node_cmp(node->name, "channel") == 0)) {
neuron_serv->channels[i] =
neuron_channel_add(node, &pdev->dev);
if (IS_ERR(neuron_serv->channels[i])) {
err = PTR_ERR(neuron_serv->channels[i]);
goto fail_add_channel;
}
neuron_serv->channels[i]->id = i;
i++;
}
}
node = of_get_child_by_name(pnode, "application");
if (!node) {
err = -ENODEV;
goto fail_find_application;
}
neuron_serv->application =
neuron_app_add(node, &pdev->dev);
if (IS_ERR(neuron_serv->application)) {
err = PTR_ERR(neuron_serv->application);
goto fail_add_application;
}
node = of_get_child_by_name(pnode, "protocol");
if (!node) {
err = -ENODEV;
goto fail_find_protocol;
}
neuron_serv->protocol =
neuron_protocol_add(node, count, neuron_serv->channels,
&pdev->dev, neuron_serv->application);
if (IS_ERR(neuron_serv->protocol)) {
err = PTR_ERR(neuron_serv->protocol);
goto fail_add_protocol;
}
for (i = 0; i < neuron_serv->channel_count; i++) {
neuron_serv->channels[i]->protocol = neuron_serv->protocol;
get_device(&neuron_serv->protocol->dev);
}
/* Save protocol pointer */
neuron_serv->application->protocol = neuron_serv->protocol;
get_device(&neuron_serv->protocol->dev);
dev_set_drvdata(&pdev->dev, neuron_serv);
return 0;
fail_add_application:
fail_find_application:
device_unregister(&neuron_serv->protocol->dev);
put_device(&neuron_serv->protocol->dev);
fail_add_protocol:
fail_find_protocol:
for (i = 0; i < (neuron_serv->channel_count); i++) {
device_unregister(&neuron_serv->channels[i]->dev);
put_device(&neuron_serv->channels[i]->dev);
}
fail_add_channel:
dev_set_drvdata(&pdev->dev, NULL);
kfree(neuron_serv);
fail_alloc_neuron_serv:
fail_find_channels:
return err;
}
static int neuron_service_remove(struct platform_device *pdev)
{
struct neuron_service *neuron_serv;
int i;
neuron_serv = dev_get_drvdata(&pdev->dev);
/* Clearing all pointers to devices */
neuron_serv->application->protocol = NULL;
neuron_serv->protocol->application = NULL;
for (i = 0; i < (neuron_serv->channel_count); i++) {
neuron_serv->protocol->channels[i] = NULL;
neuron_serv->channels[i]->protocol = NULL;
}
device_unregister(&neuron_serv->application->dev);
put_device(&neuron_serv->application->dev);
device_unregister(&neuron_serv->protocol->dev);
put_device(&neuron_serv->protocol->dev);
for (i = 0; i < (neuron_serv->channel_count); i++) {
device_unregister(&neuron_serv->channels[i]->dev);
put_device(&neuron_serv->channels[i]->dev);
}
dev_set_drvdata(&pdev->dev, NULL);
kfree(neuron_serv);
return 0;
}
static struct platform_driver neuron_service_driver = {
.driver = {
.name = DRIVER_NAME,
.of_match_table = neuron_service_match,
},
.probe = neuron_service_probe,
.remove = neuron_service_remove,
};
static int __init neuron_service_init(void)
{
int ret;
ret = platform_driver_register(&neuron_service_driver);
if (ret < 0) {
pr_err("Failed to register driver\n");
return ret;
}
return 0;
}
static void __exit neuron_service_exit(void)
{
platform_driver_unregister(&neuron_service_driver);
}
module_init(neuron_service_init);
module_exit(neuron_service_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Neuron service - configuration layer");

12
net/neuron/neuron_service.h Normal file
View file

@ -0,0 +1,12 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
#include <linux/neuron.h>
/* Device's driver data */
struct neuron_service {
struct neuron_protocol *protocol;
struct neuron_application *application;
unsigned int channel_count;
struct neuron_channel *channels[];
};

19
net/neuron/protocol/Kconfig Normal file
View file

@ -0,0 +1,19 @@
# SPDX-License-Identifier: GPL-2.0-only
config NEURON_PROT_BLOCK_CLIENT
tristate "Protocol block client driver"
help
This option enables a Neuron block protocol client driver. It
contains the commands to virtualize a block device. This driver
will read from a block device being virtualized by a server.
If unsure, say N.
config NEURON_PROT_BLOCK_SERVER
tristate "Protocol block server driver"
help
This option enables a Neuron block protocol server driver. It
contains the commands to virtualize a block device. This driver
will send commands to virtualize a block device owned by this
virtual machine.
If unsure, say N.

3
net/neuron/protocol/Makefile Normal file
View file

@ -0,0 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_NEURON_PROT_BLOCK_CLIENT) += prot_block_client.o
obj-$(CONFIG_NEURON_PROT_BLOCK_SERVER) += prot_block_server.o

47
net/neuron/protocol/prot_block.h Normal file
View file

@ -0,0 +1,47 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
#ifndef __H_PROT_BLOCK_COMMON_H
#define __H_PROT_BLOCK_COMMON_H
/* Block Server Advertisement message */
struct neuron_block_advertise {
u32 logical_block_size;
u32 physical_block_size;
u64 num_device_sectors;
u32 flags;
u32 discard_granularity;
u64 discard_max_hw_sectors;
u64 discard_max_sectors;
u16 alignment_offset;
bool wc_flag;
bool fua_flag;
u8 uuid[16];
u8 label[];
} __packed;
/* Request message */
struct neuron_block_req {
u32 req_id;
u16 req_type;
u16 flags;
u64 start_sector;
u32 sectors;
} __packed;
/* Response message */
struct neuron_block_resp {
u32 resp_id;
u16 resp_status;
} __packed;
/* Bit position definition for flags field in struct neuron_block_req */
enum neuron_block_param_flags_bit {
NEURON_BLOCK_PARAM_FLAG_READONLY = 0,
NEURON_BLOCK_PARAM_FLAG_DISCARD_ZEROES,
};
#define NEURON_BLOCK_READONLY BIT(NEURON_BLOCK_PARAM_FLAG_READONLY)
#define NEURON_BLOCK_DISCARD_ZEROES BIT(NEURON_BLOCK_PARAM_FLAG_DISCARD_ZEROES)
#endif

618
net/neuron/protocol/prot_block_client.c Normal file
View file

@ -0,0 +1,618 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
/* Protocol block client drivers
*
* This driver receives data from the application layer and sends it to the
* channel layer and vice versa. Receives from channel and sends to app.
*/
#include <linux/init.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/version.h>
#include <linux/fs.h>
#include <linux/cdev.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/kthread.h>
#include <linux/neuron.h>
#include <linux/wait.h>
#include <linux/delay.h>
#include <linux/idr.h>
#include <linux/neuron_block.h>
#include "prot_block.h"
#define DRIVER_NAME_CLIENT "neuron-protocol-client-block"
#define C_IN 1
#define C_OUT 0
#define CHANNEL_COUNT 2
#define EV_REQUEST NEURON_BLOCK_CLIENT_EVENT_REQUEST
#define EVENT_COUNT NEURON_BLOCK_CLIENT_EVENT__COUNT
#define CHANNEL_BIT(n) BIT(n)
#define EVENT_BIT(n) BIT(CHANNEL_COUNT + (n))
#define MIN_NEURON_BLOCK_AD_LEN offsetof(struct neuron_block_advertise, label)
struct req_context {
enum neuron_block_resp_status status;
enum neuron_block_req_type req_type;
int req_id;
u16 flags;
u32 sectors;
void *opaque_id;
off_t offset;
u64 start_sector;
struct buffer_list buf;
};
struct client_data {
struct task_struct *thread;
wait_queue_head_t wait_q;
unsigned long wakeups;
u32 sector_size;
struct idr req_idr;
};
static const struct of_device_id protocol_block_client_match[] = {
{
.compatible = "qcom,neuron-protocol-block",
},
{},
};
MODULE_DEVICE_TABLE(of, protocol_block_client_match);
/* Acquire a req id and store its associated context.
* @return A req id if success, others for failure.
*/
static int acquire_req_id(struct client_data *kdata, struct req_context *ctx)
{
return idr_alloc(&kdata->req_idr, ctx, 1, INT_MAX, GFP_KERNEL);
}
/* Retrieve req id associated context. */
static int get_req_id_associated(struct client_data *kdata, int req_id,
struct req_context **ctx)
{
struct req_context *context;
context = (struct req_context *)idr_find(&kdata->req_idr, req_id);
if (!context)
return -ENODEV;
*ctx = context;
return 0;
}
static void release_req_id(struct client_data *kdata, int req_id)
{
idr_remove(&kdata->req_idr, req_id);
}
static int protocol_block_channel_wakeup(struct neuron_protocol *protocol,
unsigned int id)
{
struct client_data *kdata;
if (WARN_ON_ONCE(id >= CHANNEL_COUNT))
return -EINVAL;
kdata = dev_get_drvdata(&protocol->dev);
set_bit(id, &kdata->wakeups);
wake_up(&kdata->wait_q);
return 0;
}
static int protocol_block_app_wakeup(struct neuron_protocol *prot_dev,
unsigned int ev)
{
struct client_data *kdata;
if (WARN_ON_ONCE(ev >= EVENT_COUNT))
return -EINVAL;
kdata = dev_get_drvdata(&prot_dev->dev);
set_bit(CHANNEL_COUNT + ev, &kdata->wakeups);
wake_up(&kdata->wait_q);
return 0;
}
static int protocol_block_client_thread(void *data)
{
struct neuron_protocol *prot_dev = (struct neuron_protocol *)data;
struct client_data *kdata = dev_get_drvdata(&prot_dev->dev);
struct device *dev = &prot_dev->channels[C_OUT]->dev;
struct neuron_channel *channel_dev = to_neuron_channel(dev);
struct neuron_channel_driver *channel_drv =
to_neuron_channel_driver(dev->driver);
struct device *dev_r = &prot_dev->channels[C_IN]->dev;
struct neuron_channel *channel_dev_r = to_neuron_channel(dev_r);
struct neuron_channel_driver *channel_drv_r =
to_neuron_channel_driver(dev_r->driver);
struct device *dev_app = &prot_dev->application->dev;
struct neuron_application *app_dev = to_neuron_application(dev_app);
struct neuron_app_driver *app_drv =
to_neuron_app_driver(dev_app->driver);
struct neuron_block_app_client_driver *block_drv =
container_of(app_drv,
struct neuron_block_app_client_driver, base);
struct sk_buff *skb_r, *skb, *skb_resp, *skb_req;
struct req_context *req_ctx = NULL, *resp_ctx = NULL;
struct neuron_block_resp *resp;
struct neuron_block_req *req;
void *opaque_id;
u16 flags;
u64 start_sector;
u32 sectors;
enum neuron_block_req_type req_type;
int req_id;
u32 left;
u32 to_send;
struct neuron_block_advertise *ad;
struct neuron_block_param *param;
size_t label_size;
unsigned long wakeup_mask;
bool req_valid = false;
bool req_header_sent = false;
bool resp_valid = false;
bool resp_ready = false;
int ret = 0;
skb_resp = NULL;
skb_req = NULL;
/* Wait for the channels to start */
wakeup_mask = 0;
while (!kthread_should_stop()) {
if (!channel_dev->max_size || !channel_dev->queue_length)
wakeup_mask |= CHANNEL_BIT(C_IN);
if (!channel_dev_r->max_size || !channel_dev_r->queue_length)
wakeup_mask |= CHANNEL_BIT(C_OUT);
if (!wakeup_mask)
break;
wait_event_killable(kdata->wait_q,
kthread_should_stop() ||
(kdata->wakeups & wakeup_mask));
wakeup_mask &= ~xchg(&kdata->wakeups, 0);
/* flush shared variable to memory */
smp_mb__after_atomic();
}
if (kthread_should_stop())
return 0;
WARN_ON(channel_dev_r->max_size <=
sizeof(struct neuron_block_advertise));
skb_r = alloc_skb(channel_dev_r->max_size, GFP_KERNEL);
if (!skb_r)
return -ENOMEM;
skb_put(skb_r, channel_dev_r->max_size);
/* Wait and get block params */
do {
clear_bit(C_IN, &kdata->wakeups);
/* flush shared variable to memory */
smp_mb__after_atomic();
ret = channel_drv_r->receive_msg(channel_dev_r, skb_r);
if (ret == -EAGAIN) {
wait_event_killable(kdata->wait_q,
kthread_should_stop() ||
(kdata->wakeups &
CHANNEL_BIT(C_IN)));
}
} while (ret == -EAGAIN);
if (ret <= 0) {
pr_err("Receiving Block param failed with error %d\n", ret);
kfree_skb(skb_r);
return ret;
}
if (ret < MIN_NEURON_BLOCK_AD_LEN) {
pr_err("Invalid block param length\n");
kfree_skb(skb_r);
return -EINVAL;
}
label_size = ret - MIN_NEURON_BLOCK_AD_LEN;
ad = (struct neuron_block_advertise *)skb_r->data;
/* Allocate 1 more byte to hold '\0' in case label is not
* NULL terminated.
*/
param = kzalloc(sizeof(*param) + label_size + 1, GFP_KERNEL);
if (!param) {
consume_skb(skb_r);
return -ENOMEM;
}
/* Save logical block size */
kdata->sector_size = ad->logical_block_size;
param->logical_block_size = ad->logical_block_size;
WARN_ON(param->logical_block_size > channel_dev->max_size);
param->physical_block_size = ad->physical_block_size;
param->num_device_sectors = ad->num_device_sectors;
param->discard_max_hw_sectors = ad->discard_max_hw_sectors;
param->discard_max_sectors = ad->discard_max_sectors;
param->discard_granularity = ad->discard_granularity;
param->alignment_offset = ad->alignment_offset;
param->read_only = ad->flags & NEURON_BLOCK_READONLY;
param->discard_zeroes_data = ad->flags & NEURON_BLOCK_DISCARD_ZEROES;
param->wc_flag = ad->wc_flag;
param->fua_flag = ad->fua_flag;
memcpy(param->uuid.b, ad->uuid, sizeof(param->uuid.b));
strlcpy(param->label, ad->label, label_size);
/* Ensure it is NULL terminated always. */
param->label[label_size] = '\0';
consume_skb(skb_r);
/* Set block device params. */
block_drv->do_set_bd_params(app_dev, param);
/* Pre-allocate a request SKB. This will be reallocated after each
* successful send; we can't reuse it in case the channel driver has
* cloned it.
*/
skb_req = alloc_skb(sizeof(*req), GFP_KERNEL);
if (!skb_req)
return -ENOMEM;
req = (struct neuron_block_req *)skb_put(skb_req, sizeof(*req));
/* Pre-allocate the response SKB. This can be safely reused
* because it is never sent to any asynchronous function outside
* this driver.
*/
skb_resp = __alloc_skb(sizeof(*resp), GFP_KERNEL, SKB_ALLOC_RX,
NUMA_NO_NODE);
if (!skb_resp) {
kfree_skb(skb_req);
return -ENOMEM;
}
resp = (struct neuron_block_resp *)skb_put(skb_resp, sizeof(*resp));
/* Main request handling loop */
wakeup_mask = 0;
while (!kthread_should_stop()) {
unsigned long old_wakeups, consumed_wakeups;
/*Try to get request from client*/
if (!req_valid && !(wakeup_mask & EVENT_BIT(EV_REQUEST))) {
ret = block_drv->get_request(app_dev, &opaque_id,
&req_type, &flags,
&start_sector,
&sectors, &skb);
if (ret == -EAGAIN) {
wakeup_mask |= EVENT_BIT(EV_REQUEST);
continue;
} else if (ret < 0) {
dev_err(&prot_dev->dev,
"get_request returned %d\n", ret);
goto fail;
}
/* Create request context */
req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL);
if (!req_ctx) {
ret = -ENOMEM;
goto fail;
}
req_ctx->opaque_id = opaque_id;
req_ctx->req_type = req_type;
req_ctx->flags = flags;
req_ctx->start_sector = start_sector;
req_ctx->sectors = sectors;
req_ctx->buf.head = skb;
req_ctx->buf.offset = 0;
/* Acquire req id */
req_id = acquire_req_id(kdata, req_ctx);
if (req_id < 0) {
dev_err(&prot_dev->dev,
"Can't acquire req id: %d\n", req_id);
ret = req_id;
kfree(req_ctx);
goto fail;
}
req_ctx->req_id = req_id;
dev_dbg(&prot_dev->dev, "Getting request: req_id =%d, req_type =%d, sectors = %d\n",
req_id, req_type, sectors);
req_valid = true;
req_header_sent = false;
continue;
}
/* If we have a request header, try to send it. */
if (req_valid && !req_header_sent &&
!(wakeup_mask & CHANNEL_BIT(C_OUT))) {
WARN_ON(!req_ctx);
req->req_id = req_ctx->req_id;
req->req_type = req_ctx->req_type;
req->flags = req_ctx->flags;
req->start_sector = req_ctx->start_sector;
req->sectors = req_ctx->sectors;
ret = channel_drv->send_msg(channel_dev, skb_req);
if (ret == -EAGAIN) {
wakeup_mask |= CHANNEL_BIT(C_OUT);
continue;
} else if (ret < 0) {
dev_err(&prot_dev->dev,
"Can't send request: %d\n",
ret);
goto fail;
}
dev_dbg(&prot_dev->dev, "Sending req header for req_id = %d, req_type = %d\n",
req->req_id, req->req_type);
/* Consume and replace the request skb. */
consume_skb(skb_req);
skb_req = alloc_skb(sizeof(*req), GFP_KERNEL);
if (!skb_req) {
kfree_skb(skb_resp);
return -ENOMEM;
}
req = (struct neuron_block_req *)skb_put(skb_req,
sizeof(*req));
memset(req, 0, sizeof(*req));
/* Is there any data to send? */
if (req_ctx->sectors &&
(req_ctx->req_type == NEURON_BLOCK_REQUEST_WRITE ||
req_ctx->req_type ==
NEURON_BLOCK_REQUEST_WRITE_SAME))
req_header_sent = true;
else
req_valid = false;
continue;
}
/* If we have a request data, try to send it. */
if (req_valid && req_header_sent &&
!(wakeup_mask & CHANNEL_BIT(C_OUT))) {
left = req_ctx->buf.head->len - req_ctx->buf.offset;
WARN_ON(left % kdata->sector_size);
to_send = min_t(u32, left, channel_dev->max_size);
to_send = round_down(to_send, kdata->sector_size);
req_ctx->buf.size = to_send;
ret = channel_drv->send_msgv(channel_dev, req_ctx->buf);
if (ret == -EAGAIN) {
wakeup_mask |= CHANNEL_BIT(C_OUT);
continue;
} else if (ret < 0) {
dev_err(&prot_dev->dev,
"Sending request data failed %d\n",
ret);
goto fail;
}
req_ctx->buf.offset += to_send;
if (req_ctx->buf.offset == req_ctx->buf.head->len) {
req_valid = false;
req_header_sent = false;
}
dev_dbg(&prot_dev->dev, "Sending req data for req_id = %d with %d bytes\n",
req_ctx->req_id, to_send);
continue;
}
/* Try to obtain a response from the server. */
if (!resp_valid && !(wakeup_mask & CHANNEL_BIT(C_IN))) {
ret = channel_drv_r->receive_msg(channel_dev_r,
skb_resp);
if (ret == -EAGAIN) {
wakeup_mask |= CHANNEL_BIT(C_IN);
continue;
} else if (ret < 0) {
dev_err(&prot_dev->dev, "receive_msg for resp header returned %d\n",
ret);
goto fail;
} else if (skb_resp->len < sizeof(*resp)) {
dev_err(&prot_dev->dev, "Truncated response\n");
ret = -EBADMSG;
goto fail;
}
if (get_req_id_associated(kdata, resp->resp_id,
&resp_ctx)) {
dev_err(&prot_dev->dev, "Incorrect response id %d\n",
resp->resp_id);
ret = -EBADMSG;
goto fail;
}
resp_ctx->status = resp->resp_status;
release_req_id(kdata, resp->resp_id);
dev_dbg(&prot_dev->dev, "Receiving response for req_id %d\n",
resp->resp_id);
resp_valid = true;
if (!resp->resp_status &&
resp_ctx->req_type == NEURON_BLOCK_REQUEST_READ &&
resp_ctx->sectors) {
WARN_ON(!resp_ctx->buf.head);
resp_ctx->buf.offset = 0;
resp_ctx->buf.size = resp_ctx->sectors *
kdata->sector_size;
resp_ready = false;
} else {
resp_ready = true;
}
continue;
}
/* Try to receive response data if we are waiting for any. */
if (resp_valid && !resp_ready &&
!(wakeup_mask & CHANNEL_BIT(C_IN))) {
if (WARN_ON(!resp_ctx))
goto fail;
ret = channel_drv_r->receive_msgv(channel_dev_r,
resp_ctx->buf);
if (ret == -EAGAIN) {
wakeup_mask |= CHANNEL_BIT(C_IN);
continue;
} else if (ret < 0) {
dev_err(&prot_dev->dev,
"receive_msg for resp header returned %d\n",
ret);
goto fail;
}
dev_dbg(&prot_dev->dev,
"Receiving response data for req_id %d with %d bytes\n",
resp_ctx->req_id, ret);
resp_ctx->buf.offset += ret;
resp_ctx->buf.size -= ret;
if (resp_ctx->buf.size == 0)
resp_ready = true;
continue;
}
/* Deal with response if the response is ready. */
if (resp_ready) {
if (WARN_ON(!resp_ctx))
goto fail;
dev_dbg(&prot_dev->dev, "Response is done for req %d\n",
resp_ctx->req_id);
resp_ready = false;
resp_valid = false;
consume_skb(resp_ctx->buf.head);
block_drv->do_response(app_dev, resp_ctx->opaque_id,
resp_ctx->status);
kfree(resp_ctx);
continue;
}
if (WARN_ON_ONCE(!wakeup_mask))
continue;
dev_dbg(&prot_dev->dev, "wait for events %#lx (now: %#lx)\n",
wakeup_mask, kdata->wakeups);
wait_event_killable(kdata->wait_q,
kthread_should_stop() ||
(kdata->wakeups & wakeup_mask));
do {
old_wakeups = READ_ONCE(kdata->wakeups);
consumed_wakeups = old_wakeups & wakeup_mask;
} while (cmpxchg(&kdata->wakeups, old_wakeups,
old_wakeups & ~consumed_wakeups) !=
old_wakeups);
/* flush shared variable to memory */
smp_mb__after_atomic();
wakeup_mask &= ~consumed_wakeups;
}
consume_skb(skb_req);
consume_skb(skb_resp);
fail:
kfree_skb(skb_req);
kfree_skb(skb_resp);
return ret;
}
static int protocol_block_client_probe(struct neuron_protocol *prot_dev)
{
struct client_data *kdata;
kdata = kzalloc(sizeof(*kdata), GFP_KERNEL);
if (!kdata)
return -ENOMEM;
init_waitqueue_head(&kdata->wait_q);
idr_init(&kdata->req_idr);
dev_set_drvdata(&prot_dev->dev, kdata);
kdata->thread = kthread_run(protocol_block_client_thread, prot_dev,
"%s", prot_dev->dev.driver->name);
return 0;
}
static void protocol_block_client_remove(struct neuron_protocol *prot_dev)
{
struct client_data *kdata;
kdata = dev_get_drvdata(&prot_dev->dev);
kthread_stop(kdata->thread);
dev_set_drvdata(&prot_dev->dev, NULL);
kfree(kdata);
}
static const struct neuron_channel_match_table channels_client_block[] = {
{NEURON_CHANNEL_MESSAGE_QUEUE, NEURON_CHANNEL_SEND},
{NEURON_CHANNEL_MESSAGE_QUEUE, NEURON_CHANNEL_RECEIVE}
};
static const char * const processes_client_block[] = {"client"};
struct neuron_protocol_driver protocol_client_block_driver = {
.channel_count = 2,
.channels = channels_client_block,
.process_count = 1,
.processes = processes_client_block,
.driver = {
.name = DRIVER_NAME_CLIENT,
.owner = THIS_MODULE,
.of_match_table = protocol_block_client_match,
},
.probe = protocol_block_client_probe,
.remove = protocol_block_client_remove,
.channel_wakeup = protocol_block_channel_wakeup,
.app_wakeup = protocol_block_app_wakeup,
};
EXPORT_SYMBOL(protocol_client_block_driver);
static int __init protocol_client_block_init(void)
{
int ret;
ret = neuron_register_protocol_driver(&protocol_client_block_driver);
if (ret < 0) {
pr_err("Failed to register driver\n");
return ret;
}
return 0;
}
static void __exit protocol_client_block_exit(void)
{
neuron_unregister_protocol_driver(&protocol_client_block_driver);
}
module_init(protocol_client_block_init);
module_exit(protocol_client_block_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Protocol block client drivers");

606
net/neuron/protocol/prot_block_server.c Normal file
View file

@ -0,0 +1,606 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
/* Protocol block server drivers
*
* This driver receives data from the application layer and sends it to the
* channel layer and vice versa. Receives from channel and sends to app.
*/
#include <linux/init.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/version.h>
#include <linux/fs.h>
#include <linux/cdev.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/kthread.h>
#include <linux/neuron.h>
#include <linux/wait.h>
#include <linux/delay.h>
#include <linux/neuron_block.h>
#include "prot_block.h"
#define DRIVER_NAME_SERVER "neuron-protocol-server-block"
#define C_IN 0
#define C_OUT 1
#define CHANNEL_COUNT 2
#define EV_BD_PARAMS NEURON_BLOCK_SERVER_EVENT_BD_PARAMS
#define EV_RESPONSE NEURON_BLOCK_SERVER_EVENT_RESPONSE
#define EVENT_COUNT NEURON_BLOCK_SERVER_EVENT__COUNT
#define CHANNEL_BIT(n) BIT(n)
#define EVENT_BIT(n) BIT(CHANNEL_COUNT + (n))
struct server_data {
struct task_struct *thread;
wait_queue_head_t wait_q;
unsigned long wakeups;
};
static const struct of_device_id protocol_block_server_match[] = {
{
.compatible = "qcom,neuron-protocol-block",
},
{},
};
MODULE_DEVICE_TABLE(of, protocol_block_server_match);
static int protocol_block_channel_wakeup(struct neuron_protocol *prot,
unsigned int id)
{
struct server_data *kdata;
if (WARN_ON_ONCE(id >= CHANNEL_COUNT))
return -EINVAL;
kdata = dev_get_drvdata(&prot->dev);
set_bit(id, &kdata->wakeups);
dev_dbg(&prot->dev, "wake ch %d (now: %#lx)\n", id, kdata->wakeups);
wake_up(&kdata->wait_q);
return 0;
}
static int protocol_block_app_wakeup(struct neuron_protocol *prot,
unsigned int ev)
{
struct server_data *kdata;
if (WARN_ON_ONCE(ev >= EVENT_COUNT))
return -EINVAL;
kdata = dev_get_drvdata(&prot->dev);
set_bit(CHANNEL_COUNT + ev, &kdata->wakeups);
dev_dbg(&prot->dev, "wake ev %d (now: %#lx)\n", ev, kdata->wakeups);
wake_up(&kdata->wait_q);
return 0;
}
static int handle_request(struct neuron_block_app_server_driver *block_drv,
struct neuron_application *app_dev,
struct neuron_block_req *req,
struct sk_buff *skb_data)
{
switch (req->req_type) {
case NEURON_BLOCK_REQUEST_READ:
WARN_ON(skb_data);
return block_drv->do_read(app_dev, req->req_id,
req->start_sector, req->sectors,
req->flags);
case NEURON_BLOCK_REQUEST_WRITE:
return block_drv->do_write(app_dev, req->req_id,
req->start_sector, req->sectors,
req->flags, skb_data);
case NEURON_BLOCK_REQUEST_DISCARD:
WARN_ON(skb_data);
return block_drv->do_discard(app_dev, req->req_id,
req->start_sector, req->sectors,
req->flags);
case NEURON_BLOCK_REQUEST_SECURE_ERASE:
WARN_ON(skb_data);
return block_drv->do_secure_erase(app_dev, req->req_id,
req->start_sector,
req->sectors, req->flags);
case NEURON_BLOCK_REQUEST_WRITE_SAME:
return block_drv->do_write_same(app_dev, req->req_id,
req->start_sector,
req->sectors, req->flags,
skb_data);
/* fall through return */
case NEURON_BLOCK_REQUEST_WRITE_ZEROES:
WARN_ON(skb_data);
return block_drv->do_write_zeroes(app_dev, req->req_id,
req->start_sector,
req->sectors, req->flags);
default:
pr_err("Wrong req type: %d\n", req->req_type);
if (skb_data)
kfree_skb(skb_data);
return -EINVAL;
}
}
static int protocol_block_server_thread(void *data)
{
struct neuron_protocol *prot = (struct neuron_protocol *)data;
struct neuron_channel *c_in = prot->channels[C_IN];
struct neuron_channel_driver *c_in_drv =
to_neuron_channel_driver(c_in->dev.driver);
struct neuron_channel *c_out = prot->channels[C_OUT];
struct neuron_channel_driver *c_out_drv =
to_neuron_channel_driver(c_out->dev.driver);
struct neuron_application *app = prot->application;
struct neuron_app_driver *app_drv =
to_neuron_app_driver(app->dev.driver);
struct neuron_block_app_server_driver *block_drv =
container_of(app_drv,
struct neuron_block_app_server_driver,
base);
struct server_data *kdata = dev_get_drvdata(&prot->dev);
int ret;
size_t size;
struct sk_buff *skb_ad = NULL;
struct neuron_block_advertise *ad;
const struct neuron_block_param *param;
size_t sector_size;
unsigned long wakeup_mask;
bool req_valid = false;
bool req_ready = false;
struct neuron_block_req *req;
struct sk_buff *skb_req = NULL;
struct sk_buff *skb_req_data = NULL;
size_t req_data_pos = 0;
bool resp_valid = false;
bool resp_sent_header = false;
struct neuron_block_resp *resp;
struct sk_buff *skb_resp = NULL;
struct sk_buff *skb_resp_data = NULL;
size_t resp_data_pos = 0;
kdata->wakeups = 0;
/* Wait for the channels to start */
wakeup_mask = 0;
while (!kthread_should_stop()) {
if (!c_in->max_size || !c_in->queue_length)
wakeup_mask |= CHANNEL_BIT(C_IN);
if (!c_out->max_size || !c_out->queue_length)
wakeup_mask |= CHANNEL_BIT(C_OUT);
if (!wakeup_mask)
break;
wait_event_killable(kdata->wait_q,
kthread_should_stop() ||
(kdata->wakeups & wakeup_mask));
wakeup_mask &= ~xchg(&kdata->wakeups, 0);
/* flush shared variable to memory */
smp_mb__after_atomic();
}
if (kthread_should_stop())
return 0;
/* Wait for the block params to be ready */
while (!kthread_should_stop()) {
clear_bit(EVENT_BIT(EV_BD_PARAMS), &kdata->wakeups);
/* flush shared variable to memory */
smp_mb__after_atomic();
ret = block_drv->get_bd_params(app, &param);
if (ret == 0) {
break;
} else if (ret != -EAGAIN) {
dev_err(&prot->dev, "Failed to get params: %d\n", ret);
return ret;
}
wait_event_killable(kdata->wait_q,
kthread_should_stop() ||
(kdata->wakeups & EVENT_BIT(EV_BD_PARAMS)));
}
if (kthread_should_stop())
return 0;
/* Save the logical sector size; the protocol will need it later */
sector_size = param->logical_block_size;
size = sizeof(*ad) + strlen(param->label);
skb_ad = alloc_skb(ret, GFP_KERNEL);
ad = (struct neuron_block_advertise *)skb_put(skb_ad, sizeof(*ad));
memset(ad, 0, sizeof(*ad));
ad->logical_block_size = param->logical_block_size;
ad->physical_block_size = param->physical_block_size;
ad->num_device_sectors = param->num_device_sectors;
ad->discard_max_hw_sectors = param->discard_max_hw_sectors;
ad->discard_max_sectors = param->discard_max_sectors;
ad->discard_granularity = param->discard_granularity;
ad->alignment_offset = param->alignment_offset;
ad->wc_flag = param->wc_flag;
ad->fua_flag = param->fua_flag;
memcpy(ad->uuid, param->uuid.b, sizeof(param->uuid.b));
strlcpy(ad->label, param->label, strlen(param->label));
if (param->read_only)
ad->flags |= NEURON_BLOCK_READONLY;
if (param->discard_zeroes_data)
ad->flags |= NEURON_BLOCK_DISCARD_ZEROES;
if (param->logical_block_size > c_in->max_size)
return -EPROTO;
if (param->logical_block_size > c_out->max_size)
return -EPROTO;
kfree(param);
while (!kthread_should_stop()) {
clear_bit(CHANNEL_BIT(C_OUT), &kdata->wakeups);
/* flush shared variable to memory */
smp_mb__after_atomic();
ret = c_out_drv->send_msg(c_out, skb_ad);
if (ret == 0) {
break;
} else if (ret != -EAGAIN) {
dev_err(&prot->dev, "Failed to send params: %d\n", ret);
kfree_skb(skb_ad);
return ret;
}
wait_event_killable(kdata->wait_q,
kthread_should_stop() ||
(kdata->wakeups & CHANNEL_BIT(C_OUT)));
}
consume_skb(skb_ad);
if (kthread_should_stop())
return 0;
/* Pre-allocate the request SKB. This can be safely reused
* because it is never sent to any asynchronous function outside
* this driver.
*/
skb_req = __alloc_skb(sizeof(*req), GFP_KERNEL, SKB_ALLOC_RX,
NUMA_NO_NODE);
if (!skb_req) {
ret = -ENOMEM;
goto fail;
}
req = (struct neuron_block_req *)skb_put(skb_req, sizeof(*req));
/* Pre-allocate a response SKB. This will be reallocated after each
* successful send; we can't reuse it in case the channel driver has
* cloned it.
*/
skb_resp = alloc_skb(sizeof(*resp), GFP_KERNEL);
if (!skb_resp) {
ret = -ENOMEM;
goto fail;
}
resp = (struct neuron_block_resp *)skb_put(skb_resp, sizeof(*resp));
memset(resp, 0, sizeof(*resp));
/* Main request handling loop */
wakeup_mask = 0;
while (!kthread_should_stop()) {
unsigned long old_wakeups, consumed_wakeups;
/* Try to obtain a request if we don't have one */
if (!req_valid && !(wakeup_mask & CHANNEL_BIT(C_IN))) {
size_t size;
/* New request */
WARN_ON(skb_req_data);
ret = c_in_drv->receive_msg(c_in, skb_req);
if (ret == -EAGAIN) {
wakeup_mask |= CHANNEL_BIT(C_IN);
continue;
} else if (ret < 0) {
dev_err(&prot->dev,
"%d Recv error %d\n", __LINE__, ret);
goto fail;
} else if (ret < sizeof(*req)) {
dev_err(&prot->dev, "Truncated request\n");
ret = -EBADMSG;
goto fail;
}
req_valid = true;
if (req->req_type == NEURON_BLOCK_REQUEST_WRITE)
size = req->sectors * sector_size;
else if (req->req_type ==
NEURON_BLOCK_REQUEST_WRITE_SAME)
size = sector_size;
else
size = 0;
if (size > 0) {
WARN_ON(skb_req_data);
skb_req_data = neuron_alloc_pskb(size,
GFP_KERNEL);
if (IS_ERR(skb_req_data)) {
ret = PTR_ERR(skb_req_data);
goto fail;
}
req_data_pos = 0;
req_ready = false;
} else {
req_ready = true;
}
continue;
}
/* Try to receive request data if we are waiting for any */
if (req_valid && !req_ready &&
!(wakeup_mask & CHANNEL_BIT(C_IN))) {
struct buffer_list buf;
WARN_ON(!skb_req_data);
buf.head = skb_req_data;
buf.offset = req_data_pos;
buf.size = min_t(size_t, c_in->max_size,
skb_req_data->len - req_data_pos);
buf.size = round_down(buf.size, sector_size);
WARN_ON(!buf.size);
ret = c_in_drv->receive_msgv(c_in, buf);
if (ret == -EAGAIN) {
wakeup_mask |= CHANNEL_BIT(C_IN);
continue;
} else if (ret < 0) {
dev_err(&prot->dev,
"%d Recv error %d\n", __LINE__, ret);
goto fail;
} else if (!ret) {
dev_err(&prot->dev, "Empty payload\n");
ret = -EBADMSG;
goto fail;
} else if (ret % sector_size) {
dev_err(&prot->dev, "Bad payload length %d\n",
ret);
ret = -EBADMSG;
goto fail;
}
req_data_pos += ret;
if (req_data_pos == skb_req_data->len)
req_ready = true;
continue;
}
/* Try to handle a request if there is one ready */
if (req_valid && req_ready) {
ret = handle_request(block_drv, app, req, skb_req_data);
skb_req_data = NULL;
if (ret < 0) {
dev_err(&prot->dev,
"Request failed: %d\n", ret);
ret = -EBADMSG;
goto fail;
}
req_data_pos = 0;
req_ready = false;
req_valid = false;
continue;
}
/* Try to obtain a response from the application layer. */
if (!resp_valid && !(wakeup_mask & EVENT_BIT(EV_RESPONSE))) {
enum neuron_block_resp_status resp_status;
WARN_ON(skb_resp_data);
ret = block_drv->get_response(app, &resp->resp_id,
&resp_status, &skb_resp_data);
if (ret == -EAGAIN) {
wakeup_mask |= EVENT_BIT(EV_RESPONSE);
continue;
} else if (ret < 0) {
dev_err(&prot->dev, "Can't get response: %d\n",
ret);
goto fail;
}
resp->resp_status = (u16)resp_status;
resp_valid = true;
resp_sent_header = false;
resp_data_pos = 0;
continue;
}
/* If we have a response header, try to send it. */
if (resp_valid && !resp_sent_header &&
!(wakeup_mask & CHANNEL_BIT(C_OUT))) {
ret = c_out_drv->send_msg(c_out, skb_resp);
if (ret == -EAGAIN) {
wakeup_mask |= CHANNEL_BIT(C_OUT);
continue;
} else if (ret < 0) {
dev_err(&prot->dev, "Can't send response: %d\n",
ret);
goto fail;
}
/* Consume and replace the response skb */
consume_skb(skb_resp);
skb_resp = alloc_skb(sizeof(*resp), GFP_KERNEL);
if (!skb_resp) {
ret = -ENOMEM;
goto fail;
}
resp = (struct neuron_block_resp *)skb_put(skb_resp,
sizeof(*resp));
memset(resp, 0, sizeof(*resp));
/* Is there any data to send? */
if (skb_resp_data)
resp_sent_header = true;
else
resp_valid = false;
continue;
}
/* If we have response data, try to send it. */
if (resp_valid && resp_sent_header &&
!(wakeup_mask & CHANNEL_BIT(C_OUT))) {
struct buffer_list buf;
WARN_ON(!skb_resp_data);
buf.head = skb_resp_data;
buf.offset = resp_data_pos;
buf.size = min_t(size_t, c_out->max_size,
skb_resp_data->len - resp_data_pos);
buf.size = round_down(buf.size, sector_size);
WARN_ON(!buf.size);
ret = c_out_drv->send_msgv(c_out, buf);
if (ret == -EAGAIN) {
wakeup_mask |= CHANNEL_BIT(C_OUT);
continue;
} else if (ret < 0) {
dev_err(&prot->dev, "Send error %d\n", ret);
goto fail;
}
resp_data_pos += buf.size;
WARN_ON(resp_data_pos > skb_resp_data->len);
if (resp_data_pos == skb_resp_data->len) {
resp_valid = false;
consume_skb(skb_resp_data);
skb_resp_data = NULL;
}
continue;
}
if (WARN_ON_ONCE(!wakeup_mask))
continue;
dev_dbg(&prot->dev, "wait for events %#lx (now: %#lx)\n",
wakeup_mask, kdata->wakeups);
wait_event_killable(kdata->wait_q,
kthread_should_stop() ||
(kdata->wakeups & wakeup_mask));
do {
old_wakeups = READ_ONCE(kdata->wakeups);
consumed_wakeups = old_wakeups & wakeup_mask;
} while (cmpxchg(&kdata->wakeups, old_wakeups,
old_wakeups & ~consumed_wakeups) !=
old_wakeups);
/* flush shared variable to memory */
smp_mb__after_atomic();
wakeup_mask &= ~consumed_wakeups;
}
consume_skb(skb_req);
consume_skb(skb_resp);
return 0;
fail:
if (skb_req)
kfree_skb(skb_req);
if (skb_req_data)
kfree_skb(skb_req_data);
if (skb_resp)
kfree_skb(skb_resp);
if (skb_resp_data)
kfree_skb(skb_resp_data);
return ret;
}
static int protocol_block_server_probe(struct neuron_protocol *protocol_dev)
{
struct server_data *kdata;
kdata = kzalloc(sizeof(*kdata), GFP_KERNEL);
if (!kdata)
return -ENOMEM;
init_waitqueue_head(&kdata->wait_q);
kdata->wakeups = 0;
dev_set_drvdata(&protocol_dev->dev, kdata);
kdata->thread = kthread_run(protocol_block_server_thread, protocol_dev,
"%s", dev_name(&protocol_dev->dev));
return 0;
}
static void protocol_block_server_remove(struct neuron_protocol *protocol_dev)
{
struct server_data *kdata;
kdata = dev_get_drvdata(&protocol_dev->dev);
kthread_stop(kdata->thread);
dev_set_drvdata(&protocol_dev->dev, NULL);
kfree(kdata);
}
static const struct neuron_channel_match_table channels_server_block[] = {
[C_IN] = {NEURON_CHANNEL_MESSAGE_QUEUE, NEURON_CHANNEL_RECEIVE},
[C_OUT] = {NEURON_CHANNEL_MESSAGE_QUEUE, NEURON_CHANNEL_SEND},
};
static const char * const processes_server_block[] = {"server"};
struct neuron_protocol_driver protocol_server_block_driver = {
.channel_count = CHANNEL_COUNT,
.channels = channels_server_block,
.process_count = 1,
.processes = processes_server_block,
.driver = {
.name = DRIVER_NAME_SERVER,
.owner = THIS_MODULE,
.of_match_table = protocol_block_server_match,
},
.probe = protocol_block_server_probe,
.remove = protocol_block_server_remove,
.channel_wakeup = protocol_block_channel_wakeup,
.app_wakeup = protocol_block_app_wakeup,
};
EXPORT_SYMBOL(protocol_server_block_driver);
static int __init protocol_server_block_init(void)
{
int ret;
ret = neuron_register_protocol_driver(&protocol_server_block_driver);
if (ret < 0) {
pr_err("Failed to register driver\n");
return ret;
}
return 0;
}
static void __exit protocol_server_block_exit(void)
{
neuron_unregister_protocol_driver(&protocol_server_block_driver);
}
module_init(protocol_server_block_init);
module_exit(protocol_server_block_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Neuron block server protocol driver");

262
net/neuron/protocol_bus.c Normal file
View file

@ -0,0 +1,262 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 The Linux Foundation. All rights reserved. */
/* Neuron protocol bus type driver
*
* This driver creates a protocol bus type device and registers a protocol
* driver.
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/of_device.h>
#include <linux/version.h>
#include <linux/neuron.h>
static int protocol_match(struct device *dev, struct device_driver *driver)
{
struct neuron_protocol *protocol_dev = to_neuron_protocol(dev);
struct neuron_protocol_driver *protocol_drv =
to_neuron_protocol_driver(driver);
int i;
/* The list of channel types and directions must exactly match those
* expected by the driver.
*/
if (protocol_dev->channel_count != protocol_drv->channel_count)
return 0;
for (i = 0; i < protocol_drv->channel_count; i++) {
if (protocol_dev->channels[i]->type !=
protocol_drv->channels[i].type)
return 0;
if (protocol_dev->channels[i]->direction !=
protocol_drv->channels[i].direction)
return 0;
}
/* The list of processes must exactly match those implemented by the
* driver. Typically this will just be "client" or "server", but some
* protocols might have multiple processes.
*/
if (protocol_dev->process_count != protocol_drv->process_count)
return 0;
for (i = 0; i < protocol_dev->process_count; i++)
if (strcmp(protocol_dev->processes[i],
protocol_drv->processes[i]))
return 0;
/* The protocol driver requires the application to be bound by a
* specific application driver, but that may not have happened yet so
* we can't check it here. It's checked in probe instead.
*
* It should not be possible to bind the wrong app driver anyway,
* unless the device tree has incorrect compatible strings. That is
* unlikely enough that we can live with the added overhead of not
* failing until probe.
*/
if (of_driver_match_device(dev, driver))
return 1;
return 0;
}
static int protocol_probe(struct device *dev)
{
int i, ret;
struct neuron_protocol *protocol_dev = to_neuron_protocol(dev);
struct neuron_protocol_driver *protocol_drv =
to_neuron_protocol_driver(dev->driver);
struct neuron_app_driver *app_drv =
to_neuron_app_driver(protocol_dev->application->dev.driver);
/* Fail if the application driver is not the expected one. */
if (protocol_drv != app_drv->protocol_driver)
return -EINVAL;
ret = 0;
if (protocol_drv->probe)
ret = protocol_drv->probe(protocol_dev);
else if (dev->driver->probe)
ret = dev->driver->probe(dev);
if (ret < 0)
return ret;
for (i = 0; i < protocol_dev->channel_count; i++)
rcu_assign_pointer(protocol_dev->channels[i]->protocol_drv,
protocol_drv);
if (ret >= 0)
rcu_assign_pointer(protocol_dev->application->protocol_drv,
protocol_drv);
if (app_drv->start)
app_drv->start(protocol_dev->application);
return ret;
}
static int protocol_remove(struct device *dev)
{
int i;
struct neuron_protocol *protocol_dev = to_neuron_protocol(dev);
struct neuron_protocol_driver *protocol_drv =
to_neuron_protocol_driver(dev->driver);
/* Disconnect the channels and application, and wait for them to
* notice
*/
for (i = 0; i < protocol_dev->channel_count; i++)
rcu_assign_pointer(protocol_dev->channels[i]->protocol_drv,
NULL);
rcu_assign_pointer(protocol_dev->application->protocol_drv, NULL);
synchronize_rcu();
if (protocol_drv->remove)
protocol_drv->remove(protocol_dev);
else if (dev->driver->remove)
dev->driver->remove(dev);
return 0;
}
static struct bus_type protocol_bus_type = {
.name = "neuron_protocol",
.match = protocol_match,
.probe = protocol_probe,
.remove = protocol_remove,
};
static void protocol_dev_release(struct device *dev)
{
int i;
struct neuron_protocol *protocol_dev = to_neuron_protocol(dev);
for (i = 0; i < protocol_dev->channel_count; i++)
put_device(&protocol_dev->channels[i]->dev);
kfree(protocol_dev->processes);
kfree(protocol_dev);
}
struct neuron_protocol *neuron_protocol_add(struct device_node *node,
unsigned int channel_count,
struct neuron_channel **channels,
struct device *parent,
struct neuron_application *app_dev)
{
struct neuron_protocol *protocol_dev;
const char *process;
struct property *prop;
int err;
int i;
protocol_dev = kzalloc(sizeof(*protocol_dev) +
(sizeof(struct neuron_channel *) * channel_count),
GFP_KERNEL);
if (!protocol_dev)
return ERR_PTR(-ENOMEM);
device_initialize(&protocol_dev->dev);
protocol_dev->dev.of_node = node;
protocol_dev->dev.bus = &protocol_bus_type;
protocol_dev->dev.parent = parent;
protocol_dev->dev.release = protocol_dev_release;
dev_set_name(&protocol_dev->dev, "%s:%s", dev_name(parent),
node->name);
/* Save process names */
err = of_property_count_strings(node, "processes");
if (err < 0)
goto fail_properties;
protocol_dev->process_count = err;
protocol_dev->processes = kcalloc(protocol_dev->process_count,
sizeof(const char *),
GFP_KERNEL);
if (!protocol_dev->processes)
goto fail_properties;
i = 0;
of_property_for_each_string(node, "processes", prop, process) {
protocol_dev->processes[i] = process;
i++;
}
/* Save channel pointers */
protocol_dev->channel_count = 0;
for (i = 0; i < channel_count; i++) {
protocol_dev->channels[i] = &(*channels[i]);
get_device(&(*channels[i]).dev);
protocol_dev->channel_count++;
}
/* Save application pointer in protocol */
protocol_dev->application = app_dev;
get_device(&app_dev->dev);
if (!device_link_add(&protocol_dev->dev, &app_dev->dev, 0)) {
err = -ENODEV;
goto fail_link;
}
err = device_add(&protocol_dev->dev);
if (err)
goto fail_device_add;
return protocol_dev;
fail_device_add:
fail_link:
fail_properties:
put_device(&protocol_dev->dev);
return ERR_PTR(err);
}
EXPORT_SYMBOL(neuron_protocol_add);
int neuron_register_protocol_driver(struct neuron_protocol_driver *drv)
{
int ret;
drv->driver.bus = &protocol_bus_type;
ret = driver_register(&drv->driver);
if (ret)
return ret;
return 0;
}
EXPORT_SYMBOL(neuron_register_protocol_driver);
void neuron_unregister_protocol_driver(struct neuron_protocol_driver *drv)
{
driver_unregister(&drv->driver);
}
EXPORT_SYMBOL(neuron_unregister_protocol_driver);
static int __init protocol_bus_init(void)
{
int ret;
ret = bus_register(&protocol_bus_type);
if (ret < 0) {
pr_err("Unable to register bus\n");
return ret;
}
return 0;
}
static void protocol_bus_exit(void)
{
bus_unregister(&protocol_bus_type);
}
subsys_initcall(protocol_bus_init);
module_exit(protocol_bus_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Neuron protocol bus module");