diff --git a/Documentation/ABI/testing/sysfs-fs-fuse b/Documentation/ABI/testing/sysfs-fs-fuse new file mode 100644 index 000000000000..b9956842b36f --- /dev/null +++ b/Documentation/ABI/testing/sysfs-fs-fuse @@ -0,0 +1,19 @@ +What: /sys/fs/fuse/features/fuse_bpf +Date: December 2022 +Contact: Paul Lawrence +Description: + Read-only file that contains the word 'supported' if fuse-bpf is + supported, does not exist otherwise + +What: /sys/fs/fuse/bpf_prog_type_fuse +Date: December 2022 +Contact: Paul Lawrence +Description: + bpf_prog_type_fuse defines the program type of bpf programs that + may be passed to fuse-bpf. For upstream bpf program types, this + is a constant defined in a contiguous array of constants. + bpf_prog_type_fuse is appended to the end of the list, so it may + change and therefore its value must be read from this file. + + Contents is ASCII decimal representation of bpf_prog_type_fuse + diff --git a/Documentation/filesystems/fuse.txt b/Documentation/filesystems/fuse.txt index 13af4a49e7db..d32457766947 100644 --- a/Documentation/filesystems/fuse.txt +++ b/Documentation/filesystems/fuse.txt @@ -46,10 +46,8 @@ non-privileged mounts. This opens up new possibilities for the use of filesystems. A good example is sshfs: a secure network filesystem using the sftp protocol. -The userspace library and utilities are available from the FUSE -homepage: - - http://fuse.sourceforge.net/ +The userspace library and utilities are available from the +`FUSE homepage: `_ Filesystem type ~~~~~~~~~~~~~~~ diff --git a/MAINTAINERS b/MAINTAINERS index 0bc36d8a3c63..7b73fc0469c8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6742,7 +6742,8 @@ F: drivers/platform/x86/fujitsu-tablet.c FUSE: FILESYSTEM IN USERSPACE M: Miklos Szeredi L: linux-fsdevel@vger.kernel.org -W: http://fuse.sourceforge.net/ +S: Maintained +W: https://github.com/libfuse/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git S: Maintained F: fs/fuse/ diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 7b1714aaf30d..64beb2922cf9 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -513,6 +513,7 @@ CONFIG_FS_VERITY_BUILTIN_SIGNATURES=y CONFIG_QUOTA=y CONFIG_QFMT_V2=y CONFIG_FUSE_FS=y +CONFIG_FUSE_BPF=y CONFIG_OVERLAY_FS=y CONFIG_MSDOS_FS=y CONFIG_VFAT_FS=y diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index 5f88798bb496..ebab1f1901d8 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -442,6 +442,7 @@ CONFIG_FS_VERITY_BUILTIN_SIGNATURES=y CONFIG_QUOTA=y CONFIG_QFMT_V2=y CONFIG_FUSE_FS=y +CONFIG_FUSE_BPF=y CONFIG_OVERLAY_FS=y CONFIG_MSDOS_FS=y CONFIG_VFAT_FS=y diff --git a/block/blk-core.c b/block/blk-core.c index 74f7a2a4f295..c565130ddf75 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -504,8 +504,6 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) if (!q->stats) goto fail_stats; - q->backing_dev_info->ra_pages = VM_READAHEAD_PAGES; - q->backing_dev_info->io_pages = VM_READAHEAD_PAGES; q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK; q->backing_dev_info->name = "block"; q->node = node_id; diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 5419d6e0e560..cfe7a2aa026c 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -6923,7 +6923,7 @@ const struct file_operations binder_fops = { .owner = THIS_MODULE, .poll = binder_poll, .unlocked_ioctl = binder_ioctl, - .compat_ioctl = binder_ioctl, + .compat_ioctl = compat_ptr_ioctl, .mmap = binder_mmap, .open = binder_open, .flush = binder_flush, diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index abc7a7f64d64..ef0e482ee04f 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -68,7 +68,7 @@ static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg); static const struct file_operations adf_ctl_ops = { .owner = THIS_MODULE, .unlocked_ioctl = adf_ctl_ioctl, - .compat_ioctl = adf_ctl_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; struct adf_ctl_drv_info { diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 5005b92b197d..c54bb58de20c 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -348,6 +348,26 @@ size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, } EXPORT_SYMBOL_GPL(dax_copy_to_iter); +int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, + size_t nr_pages) +{ + if (!dax_alive(dax_dev)) + return -ENXIO; + + if (!dax_dev->ops->zero_page_range) + return -EOPNOTSUPP; + /* + * There are no callers that want to zero more than one page as of now. + * Once users are there, this check can be removed after the + * device mapper code has been updated to split ranges across targets. + */ + if (nr_pages != 1) + return -EIO; + + return dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages); +} +EXPORT_SYMBOL_GPL(dax_zero_page_range); + #ifdef CONFIG_ARCH_HAS_PMEM_API void arch_wb_cache_pmem(void *addr, size_t size); void dax_flush(struct dax_device *dax_dev, void *addr, size_t size) diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 9b774277b75e..6541c28ede73 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -468,9 +468,7 @@ static const struct file_operations dma_buf_fops = { .llseek = dma_buf_llseek, .poll = dma_buf_poll, .unlocked_ioctl = dma_buf_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = dma_buf_ioctl, -#endif + .compat_ioctl = compat_ptr_ioctl, .show_fdinfo = dma_buf_show_fdinfo, }; diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c index 7e7356970d5f..7f5ed1aa7a9f 100644 --- a/drivers/dma-buf/sw_sync.c +++ b/drivers/dma-buf/sw_sync.c @@ -408,5 +408,5 @@ const struct file_operations sw_sync_debugfs_fops = { .open = sw_sync_debugfs_open, .release = sw_sync_debugfs_release, .unlocked_ioctl = sw_sync_ioctl, - .compat_ioctl = sw_sync_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c index 992c15c016dd..f0c822952c20 100644 --- a/drivers/dma-buf/sync_file.c +++ b/drivers/dma-buf/sync_file.c @@ -481,5 +481,5 @@ static const struct file_operations sync_file_fops = { .release = sync_file_release, .poll = sync_file_poll, .unlocked_ioctl = sync_file_ioctl, - .compat_ioctl = sync_file_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 548cd062b9cd..fb5fca3d2149 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -49,7 +49,7 @@ static const char kfd_dev_name[] = "kfd"; static const struct file_operations kfd_fops = { .owner = THIS_MODULE, .unlocked_ioctl = kfd_ioctl, - .compat_ioctl = kfd_ioctl, + .compat_ioctl = compat_ptr_ioctl, .open = kfd_open, .mmap = kfd_mmap, }; diff --git a/drivers/hid/hidraw.c b/drivers/hid/hidraw.c index f8ef6268f3f2..97a4eb4ca1fb 100644 --- a/drivers/hid/hidraw.c +++ b/drivers/hid/hidraw.c @@ -472,9 +472,7 @@ static const struct file_operations hidraw_ops = { .release = hidraw_release, .unlocked_ioctl = hidraw_ioctl, .fasync = hidraw_fasync, -#ifdef CONFIG_COMPAT - .compat_ioctl = hidraw_ioctl, -#endif + .compat_ioctl = compat_ptr_ioctl, .llseek = noop_llseek, }; diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index 485b2e6748c5..243a1600096a 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -1612,7 +1612,7 @@ static const struct file_operations iio_buffer_fileops = { .owner = THIS_MODULE, .llseek = noop_llseek, .unlocked_ioctl = iio_ioctl, - .compat_ioctl = iio_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static int iio_check_unique_scan_index(struct iio_dev *indio_dev) diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index af6dedabd6a5..d46ea235c216 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -1135,7 +1135,7 @@ static const struct file_operations uverbs_fops = { .release = ib_uverbs_close, .llseek = no_llseek, .unlocked_ioctl = ib_uverbs_ioctl, - .compat_ioctl = ib_uverbs_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static const struct file_operations uverbs_mmap_fops = { @@ -1146,7 +1146,7 @@ static const struct file_operations uverbs_mmap_fops = { .release = ib_uverbs_close, .llseek = no_llseek, .unlocked_ioctl = ib_uverbs_ioctl, - .compat_ioctl = ib_uverbs_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static int ib_uverbs_get_nl_info(struct ib_device *ibdev, void *client_data, diff --git a/drivers/media/rc/lirc_dev.c b/drivers/media/rc/lirc_dev.c index aeff7d7155fd..1d554bba567e 100644 --- a/drivers/media/rc/lirc_dev.c +++ b/drivers/media/rc/lirc_dev.c @@ -724,9 +724,7 @@ static const struct file_operations lirc_fops = { .owner = THIS_MODULE, .write = ir_lirc_transmit_ir, .unlocked_ioctl = ir_lirc_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = ir_lirc_ioctl, -#endif + .compat_ioctl = compat_ptr_ioctl, .read = ir_lirc_read, .poll = ir_lirc_poll, .open = ir_lirc_open, diff --git a/drivers/misc/vmw_vmci/vmci_host.c b/drivers/misc/vmw_vmci/vmci_host.c index 6eddb805642e..c7c98d50d9fe 100644 --- a/drivers/misc/vmw_vmci/vmci_host.c +++ b/drivers/misc/vmw_vmci/vmci_host.c @@ -968,7 +968,7 @@ static const struct file_operations vmuser_fops = { .release = vmci_host_close, .poll = vmci_host_poll, .unlocked_ioctl = vmci_host_unlocked_ioctl, - .compat_ioctl = vmci_host_unlocked_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static struct miscdevice vmci_host_miscdev = { diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index 83012d74dcd5..bccbde286233 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -1948,6 +1948,8 @@ static struct backing_dev_info * __init mtd_bdi_init(char *name) bdi = bdi_alloc(GFP_KERNEL); if (!bdi) return ERR_PTR(-ENOMEM); + bdi->ra_pages = 0; + bdi->io_pages = 0; bdi->name = name; /* diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index 56f189c3129a..4600f80fb250 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -1229,7 +1229,7 @@ static const struct file_operations nvdimm_bus_fops = { .owner = THIS_MODULE, .open = nd_open, .unlocked_ioctl = bus_ioctl, - .compat_ioctl = bus_ioctl, + .compat_ioctl = compat_ptr_ioctl, .llseek = noop_llseek, }; @@ -1237,7 +1237,7 @@ static const struct file_operations nvdimm_fops = { .owner = THIS_MODULE, .open = nd_open, .unlocked_ioctl = dimm_ioctl, - .compat_ioctl = dimm_ioctl, + .compat_ioctl = compat_ptr_ioctl, .llseek = noop_llseek, }; diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 7e65306b2bf2..0f94658af4e7 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -137,9 +137,25 @@ static blk_status_t read_pmem(struct page *page, unsigned int off, return BLK_STS_OK; } -static blk_status_t pmem_do_bvec(struct pmem_device *pmem, struct page *page, - unsigned int len, unsigned int off, unsigned int op, - sector_t sector) +static blk_status_t pmem_do_read(struct pmem_device *pmem, + struct page *page, unsigned int page_off, + sector_t sector, unsigned int len) +{ + blk_status_t rc; + phys_addr_t pmem_off = sector * 512 + pmem->data_offset; + void *pmem_addr = pmem->virt_addr + pmem_off; + + if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) + return BLK_STS_IOERR; + + rc = read_pmem(page, page_off, pmem_addr, len); + flush_dcache_page(page); + return rc; +} + +static blk_status_t pmem_do_write(struct pmem_device *pmem, + struct page *page, unsigned int page_off, + sector_t sector, unsigned int len) { blk_status_t rc = BLK_STS_OK; bool bad_pmem = false; @@ -149,34 +165,25 @@ static blk_status_t pmem_do_bvec(struct pmem_device *pmem, struct page *page, if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) bad_pmem = true; - if (!op_is_write(op)) { - if (unlikely(bad_pmem)) - rc = BLK_STS_IOERR; - else { - rc = read_pmem(page, off, pmem_addr, len); - flush_dcache_page(page); - } - } else { - /* - * Note that we write the data both before and after - * clearing poison. The write before clear poison - * handles situations where the latest written data is - * preserved and the clear poison operation simply marks - * the address range as valid without changing the data. - * In this case application software can assume that an - * interrupted write will either return the new good - * data or an error. - * - * However, if pmem_clear_poison() leaves the data in an - * indeterminate state we need to perform the write - * after clear poison. - */ - flush_dcache_page(page); - write_pmem(pmem_addr, page, off, len); - if (unlikely(bad_pmem)) { - rc = pmem_clear_poison(pmem, pmem_off, len); - write_pmem(pmem_addr, page, off, len); - } + /* + * Note that we write the data both before and after + * clearing poison. The write before clear poison + * handles situations where the latest written data is + * preserved and the clear poison operation simply marks + * the address range as valid without changing the data. + * In this case application software can assume that an + * interrupted write will either return the new good + * data or an error. + * + * However, if pmem_clear_poison() leaves the data in an + * indeterminate state we need to perform the write + * after clear poison. + */ + flush_dcache_page(page); + write_pmem(pmem_addr, page, page_off, len); + if (unlikely(bad_pmem)) { + rc = pmem_clear_poison(pmem, pmem_off, len); + write_pmem(pmem_addr, page, page_off, len); } return rc; @@ -198,8 +205,12 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) do_acct = nd_iostat_start(bio, &start); bio_for_each_segment(bvec, bio, iter) { - rc = pmem_do_bvec(pmem, bvec.bv_page, bvec.bv_len, - bvec.bv_offset, bio_op(bio), iter.bi_sector); + if (op_is_write(bio_op(bio))) + rc = pmem_do_write(pmem, bvec.bv_page, bvec.bv_offset, + iter.bi_sector, bvec.bv_len); + else + rc = pmem_do_read(pmem, bvec.bv_page, bvec.bv_offset, + iter.bi_sector, bvec.bv_len); if (rc) { bio->bi_status = rc; break; @@ -224,9 +235,12 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector, struct pmem_device *pmem = bdev->bd_queue->queuedata; blk_status_t rc; - rc = pmem_do_bvec(pmem, page, hpage_nr_pages(page) * PAGE_SIZE, - 0, op, sector); - + if (op_is_write(op)) + rc = pmem_do_write(pmem, page, 0, sector, + hpage_nr_pages(page) * PAGE_SIZE); + else + rc = pmem_do_read(pmem, page, 0, sector, + hpage_nr_pages(page) * PAGE_SIZE); /* * The ->rw_page interface is subtle and tricky. The core * retries on any error, so we can only invoke page_endio() in @@ -269,6 +283,16 @@ static const struct block_device_operations pmem_fops = { .revalidate_disk = nvdimm_revalidate_disk, }; +static int pmem_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, + size_t nr_pages) +{ + struct pmem_device *pmem = dax_get_private(dax_dev); + + return blk_status_to_errno(pmem_do_write(pmem, ZERO_PAGE(0), 0, + PFN_PHYS(pgoff) >> SECTOR_SHIFT, + PAGE_SIZE)); +} + static long pmem_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn) { @@ -300,6 +324,7 @@ static const struct dax_operations pmem_dax_ops = { .dax_supported = generic_fsdax_supported, .copy_from_iter = pmem_copy_from_iter, .copy_to_iter = pmem_copy_to_iter, + .zero_page_range = pmem_dax_zero_page_range, }; static const struct attribute_group *pmem_attribute_groups[] = { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 94c8ef4a54d3..d1a369c619dc 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3062,7 +3062,7 @@ static const struct file_operations nvme_dev_fops = { .open = nvme_dev_open, .release = nvme_dev_release, .unlocked_ioctl = nvme_dev_ioctl, - .compat_ioctl = nvme_dev_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static ssize_t nvme_sysfs_reset(struct device *dev, diff --git a/drivers/pci/switch/switchtec.c b/drivers/pci/switch/switchtec.c index 0b8325235852..1b30d08e1867 100644 --- a/drivers/pci/switch/switchtec.c +++ b/drivers/pci/switch/switchtec.c @@ -1025,7 +1025,7 @@ static const struct file_operations switchtec_fops = { .read = switchtec_dev_read, .poll = switchtec_dev_poll, .unlocked_ioctl = switchtec_dev_ioctl, - .compat_ioctl = switchtec_dev_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static void link_event_work(struct work_struct *work) diff --git a/drivers/platform/x86/wmi.c b/drivers/platform/x86/wmi.c index 66cfc35e4e3d..2acdbf93088d 100644 --- a/drivers/platform/x86/wmi.c +++ b/drivers/platform/x86/wmi.c @@ -912,7 +912,7 @@ static const struct file_operations wmi_fops = { .read = wmi_char_read, .open = wmi_char_open, .unlocked_ioctl = wmi_ioctl, - .compat_ioctl = wmi_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static int wmi_dev_probe(struct device *dev) diff --git a/drivers/rpmsg/rpmsg_char.c b/drivers/rpmsg/rpmsg_char.c index ac50ed757765..c503829fb307 100644 --- a/drivers/rpmsg/rpmsg_char.c +++ b/drivers/rpmsg/rpmsg_char.c @@ -292,7 +292,7 @@ static const struct file_operations rpmsg_eptdev_fops = { .write_iter = rpmsg_eptdev_write_iter, .poll = rpmsg_eptdev_poll, .unlocked_ioctl = rpmsg_eptdev_ioctl, - .compat_ioctl = rpmsg_eptdev_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static ssize_t name_show(struct device *dev, struct device_attribute *attr, @@ -446,7 +446,7 @@ static const struct file_operations rpmsg_ctrldev_fops = { .open = rpmsg_ctrldev_open, .release = rpmsg_ctrldev_release, .unlocked_ioctl = rpmsg_ctrldev_ioctl, - .compat_ioctl = rpmsg_ctrldev_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static void rpmsg_ctrldev_release_device(struct device *dev) diff --git a/drivers/sbus/char/display7seg.c b/drivers/sbus/char/display7seg.c index 971fe074d7c9..fad936eb845f 100644 --- a/drivers/sbus/char/display7seg.c +++ b/drivers/sbus/char/display7seg.c @@ -156,7 +156,7 @@ static long d7s_ioctl(struct file *file, unsigned int cmd, unsigned long arg) static const struct file_operations d7s_fops = { .owner = THIS_MODULE, .unlocked_ioctl = d7s_ioctl, - .compat_ioctl = d7s_ioctl, + .compat_ioctl = compat_ptr_ioctl, .open = d7s_open, .release = d7s_release, .llseek = noop_llseek, diff --git a/drivers/sbus/char/envctrl.c b/drivers/sbus/char/envctrl.c index a63d5e402ff2..12d66aa61ede 100644 --- a/drivers/sbus/char/envctrl.c +++ b/drivers/sbus/char/envctrl.c @@ -715,9 +715,7 @@ static const struct file_operations envctrl_fops = { .owner = THIS_MODULE, .read = envctrl_read, .unlocked_ioctl = envctrl_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = envctrl_ioctl, -#endif + .compat_ioctl = compat_ptr_ioctl, .open = envctrl_open, .release = envctrl_release, .llseek = noop_llseek, diff --git a/drivers/scsi/3w-xxxx.c b/drivers/scsi/3w-xxxx.c index 75290aabd543..211a25351e7d 100644 --- a/drivers/scsi/3w-xxxx.c +++ b/drivers/scsi/3w-xxxx.c @@ -1049,9 +1049,7 @@ static int tw_chrdev_open(struct inode *inode, struct file *file) static const struct file_operations tw_fops = { .owner = THIS_MODULE, .unlocked_ioctl = tw_chrdev_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = tw_chrdev_ioctl, -#endif + .compat_ioctl = compat_ptr_ioctl, .open = tw_chrdev_open, .release = NULL, .llseek = noop_llseek, diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c index 67d681c53c29..6f45074ee895 100644 --- a/drivers/scsi/cxlflash/main.c +++ b/drivers/scsi/cxlflash/main.c @@ -3593,7 +3593,7 @@ static const struct file_operations cxlflash_chr_fops = { .owner = THIS_MODULE, .open = cxlflash_chr_open, .unlocked_ioctl = cxlflash_chr_ioctl, - .compat_ioctl = cxlflash_chr_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; /** diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c index fdbda5c05aa0..80c5a235d193 100644 --- a/drivers/scsi/esas2r/esas2r_main.c +++ b/drivers/scsi/esas2r/esas2r_main.c @@ -613,7 +613,7 @@ static int __init esas2r_init(void) /* Handle ioctl calls to "/proc/scsi/esas2r/ATTOnode" */ static const struct file_operations esas2r_proc_fops = { - .compat_ioctl = esas2r_proc_ioctl, + .compat_ioctl = compat_ptr_ioctl, .unlocked_ioctl = esas2r_proc_ioctl, }; diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c index f95a970db8fd..4dc316ca5d28 100644 --- a/drivers/scsi/pmcraid.c +++ b/drivers/scsi/pmcraid.c @@ -3973,9 +3973,7 @@ static const struct file_operations pmcraid_fops = { .open = pmcraid_chr_open, .fasync = pmcraid_chr_fasync, .unlocked_ioctl = pmcraid_chr_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = pmcraid_chr_ioctl, -#endif + .compat_ioctl = compat_ptr_ioctl, .llseek = noop_llseek, }; diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c index ce974bdfc7d0..343957103791 100644 --- a/drivers/staging/android/ion/ion.c +++ b/drivers/staging/android/ion/ion.c @@ -221,9 +221,7 @@ static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) static const struct file_operations ion_fops = { .owner = THIS_MODULE, .unlocked_ioctl = ion_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = ion_ioctl, -#endif + .compat_ioctl = compat_ptr_ioctl, }; static int debug_shrink_set(void *data, u64 val) diff --git a/drivers/staging/vme/devices/vme_user.c b/drivers/staging/vme/devices/vme_user.c index 6a33aaa1a49f..fd0ea4dbcb91 100644 --- a/drivers/staging/vme/devices/vme_user.c +++ b/drivers/staging/vme/devices/vme_user.c @@ -494,7 +494,7 @@ static const struct file_operations vme_user_fops = { .write = vme_user_write, .llseek = vme_user_llseek, .unlocked_ioctl = vme_user_unlocked_ioctl, - .compat_ioctl = vme_user_unlocked_ioctl, + .compat_ioctl = compat_ptr_ioctl, .mmap = vme_user_mmap, }; diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index 28cbe4613ed9..faa6ac8719cc 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -683,7 +683,7 @@ static const struct file_operations tee_fops = { .open = tee_open, .release = tee_release, .unlocked_ioctl = tee_ioctl, - .compat_ioctl = tee_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static void tee_release_device(struct device *dev) diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c index 6afb941dd267..d2bbc9a43e7b 100644 --- a/drivers/usb/class/cdc-wdm.c +++ b/drivers/usb/class/cdc-wdm.c @@ -790,7 +790,7 @@ static const struct file_operations wdm_fops = { .release = wdm_release, .poll = wdm_poll, .unlocked_ioctl = wdm_ioctl, - .compat_ioctl = wdm_ioctl, + .compat_ioctl = compat_ptr_ioctl, .llseek = noop_llseek, }; diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c index d8ed205e6b43..7ad47637f2a7 100644 --- a/drivers/usb/class/usbtmc.c +++ b/drivers/usb/class/usbtmc.c @@ -2241,9 +2241,7 @@ static const struct file_operations fops = { .release = usbtmc_release, .flush = usbtmc_flush, .unlocked_ioctl = usbtmc_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = usbtmc_ioctl, -#endif + .compat_ioctl = compat_ptr_ioctl, .fasync = usbtmc_fasync, .poll = usbtmc_poll, .llseek = default_llseek, diff --git a/drivers/virt/fsl_hypervisor.c b/drivers/virt/fsl_hypervisor.c index fb292f9cf29d..46ee0a0998b6 100644 --- a/drivers/virt/fsl_hypervisor.c +++ b/drivers/virt/fsl_hypervisor.c @@ -705,7 +705,7 @@ static const struct file_operations fsl_hv_fops = { .poll = fsl_hv_poll, .read = fsl_hv_read, .unlocked_ioctl = fsl_hv_ioctl, - .compat_ioctl = fsl_hv_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static struct miscdevice fsl_hv_misc_dev = { diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index ffce168296bd..4446e83aea7a 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -455,7 +455,7 @@ static int v9fs_test_inode(struct inode *inode, void *data) umode = p9mode2unixmode(v9ses, st, &rdev); /* don't match inode of different type */ - if ((inode->i_mode & S_IFMT) != (umode & S_IFMT)) + if (inode_wrong_type(inode, umode)) return 0; /* compare qid details */ @@ -1412,7 +1412,7 @@ int v9fs_refresh_inode(struct p9_fid *fid, struct inode *inode) * Don't update inode if the file type is different */ umode = p9mode2unixmode(v9ses, st, &rdev); - if ((inode->i_mode & S_IFMT) != (umode & S_IFMT)) + if (inode_wrong_type(inode, umode)) goto out; /* diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index 9795fe0f1ba2..614f23a9d2bc 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -59,7 +59,7 @@ static int v9fs_test_inode_dotl(struct inode *inode, void *data) struct p9_stat_dotl *st = (struct p9_stat_dotl *)data; /* don't match inode of different type */ - if ((inode->i_mode & S_IFMT) != (st->st_mode & S_IFMT)) + if (inode_wrong_type(inode, st->st_mode)) return 0; if (inode->i_generation != st->st_gen) @@ -924,7 +924,7 @@ int v9fs_refresh_inode_dotl(struct p9_fid *fid, struct inode *inode) /* * Don't update inode if the file type is different */ - if ((inode->i_mode & S_IFMT) != (st->st_mode & S_IFMT)) + if (inode_wrong_type(inode, st->st_mode)) goto out; /* diff --git a/fs/9p/vfs_super.c b/fs/9p/vfs_super.c index 46e58fdf9ba5..c226d876383d 100644 --- a/fs/9p/vfs_super.c +++ b/fs/9p/vfs_super.c @@ -80,8 +80,10 @@ v9fs_fill_super(struct super_block *sb, struct v9fs_session_info *v9ses, if (ret) return ret; - if (v9ses->cache) - sb->s_bdi->ra_pages = VM_READAHEAD_PAGES; + if (!v9ses->cache) { + sb->s_bdi->ra_pages = 0; + sb->s_bdi->io_pages = 0; + } sb->s_flags |= SB_ACTIVE | SB_DIRSYNC; if (!v9ses->cache) diff --git a/fs/afs/super.c b/fs/afs/super.c index 554119068ea4..f432713872e3 100644 --- a/fs/afs/super.c +++ b/fs/afs/super.c @@ -449,7 +449,6 @@ static int afs_fill_super(struct super_block *sb, struct afs_fs_context *ctx) ret = super_setup_bdi(sb); if (ret) return ret; - sb->s_bdi->ra_pages = VM_READAHEAD_PAGES; /* allocate the root inode and dentry */ if (as->dyn_root) { diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 660544fc0781..176ba981f055 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -3008,7 +3008,6 @@ int open_ctree(struct super_block *sb, sb->s_bdi->congested_fn = btrfs_congested_fn; sb->s_bdi->congested_data = fs_info; sb->s_bdi->capabilities |= BDI_CAP_CGROUP_WRITEBACK; - sb->s_bdi->ra_pages = VM_READAHEAD_PAGES; sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super); sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE); diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index e271f1f8c601..daa0c0467991 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -2322,7 +2322,7 @@ static const struct super_operations btrfs_super_ops = { static const struct file_operations btrfs_ctl_fops = { .open = btrfs_control_open, .unlocked_ioctl = btrfs_control_ioctl, - .compat_ioctl = btrfs_control_ioctl, + .compat_ioctl = compat_ptr_ioctl, .owner = THIS_MODULE, .llseek = noop_llseek, }; diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index af0980c720c7..932aa65182b3 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -420,8 +420,7 @@ int cifs_get_inode_info_unix(struct inode **pinode, } /* if filetype is different, return error */ - if (unlikely(((*pinode)->i_mode & S_IFMT) != - (fattr.cf_mode & S_IFMT))) { + if (unlikely(inode_wrong_type(*pinode, fattr.cf_mode))) { CIFS_I(*pinode)->time = 0; /* force reval */ rc = -ESTALE; goto cgiiu_exit; @@ -980,7 +979,7 @@ cifs_find_inode(struct inode *inode, void *opaque) return 0; /* don't match inode of different type */ - if ((inode->i_mode & S_IFMT) != (fattr->cf_mode & S_IFMT)) + if (inode_wrong_type(inode, fattr->cf_mode)) return 0; /* if it's not a directory or has no dentries, then flag it */ diff --git a/fs/fuse/Kconfig b/fs/fuse/Kconfig index 0635cba19971..ab3428492b61 100644 --- a/fs/fuse/Kconfig +++ b/fs/fuse/Kconfig @@ -8,7 +8,7 @@ config FUSE_FS There's also a companion library: libfuse2. This library is available from the FUSE homepage: - + although chances are your distribution already has that library installed if you've installed the "fuse" package itself. @@ -34,7 +34,29 @@ config VIRTIO_FS select VIRTIO help The Virtio Filesystem allows guests to mount file systems from the - host. + host. If you want to share files between guests or with the host, answer Y - or M. + or M. + +config FUSE_DAX + bool "Virtio Filesystem Direct Host Memory Access support" + default y + select INTERVAL_TREE + depends on VIRTIO_FS + depends on FS_DAX + depends on DAX_DRIVER + help + This allows bypassing guest page cache and allows mapping host page + cache directly in guest address space. + + If you want to allow mounting a Virtio Filesystem with the "dax" + option, answer Y. + +config FUSE_BPF + bool "Adds BPF to fuse" + depends on FUSE_FS + depends on BPF + help + Extends FUSE by adding BPF to prefilter calls and potentially pass to a + backing file system diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile index 6971454a2bdf..7bd922ff8604 100644 --- a/fs/fuse/Makefile +++ b/fs/fuse/Makefile @@ -7,6 +7,8 @@ obj-$(CONFIG_FUSE_FS) += fuse.o obj-$(CONFIG_CUSE) += cuse.o obj-$(CONFIG_VIRTIO_FS) += virtiofs.o -fuse-objs := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o -fuse-objs += passthrough.o -virtiofs-y += virtio_fs.o +fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o passthrough.o +fuse-$(CONFIG_FUSE_DAX) += dax.o +fuse-$(CONFIG_FUSE_BPF) += backing.o + +virtiofs-y := virtio_fs.o diff --git a/fs/fuse/backing.c b/fs/fuse/backing.c new file mode 100644 index 000000000000..013a03259f0c --- /dev/null +++ b/fs/fuse/backing.c @@ -0,0 +1,2546 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE-BPF: Filesystem in Userspace with BPF + * Copyright (c) 2021 Google LLC + */ + +#include "fuse_i.h" + +#include +#include +#include +#include + +#include "../internal.h" + +#define FUSE_BPF_IOCB_MASK (IOCB_APPEND | IOCB_DSYNC | IOCB_HIPRI | IOCB_NOWAIT | IOCB_SYNC) + +struct fuse_bpf_aio_req { + struct kiocb iocb; + refcount_t ref; + struct kiocb *iocb_orig; +}; + +static struct kmem_cache *fuse_bpf_aio_request_cachep; + +static void fuse_stat_to_attr(struct fuse_conn *fc, struct inode *inode, + struct kstat *stat, struct fuse_attr *attr); + +static void fuse_file_accessed(struct file *dst_file, struct file *src_file) +{ + struct inode *dst_inode; + struct inode *src_inode; + + if (dst_file->f_flags & O_NOATIME) + return; + + dst_inode = file_inode(dst_file); + src_inode = file_inode(src_file); + + if ((!timespec64_equal(&dst_inode->i_mtime, &src_inode->i_mtime) || + !timespec64_equal(&dst_inode->i_ctime, &src_inode->i_ctime))) { + dst_inode->i_mtime = src_inode->i_mtime; + dst_inode->i_ctime = src_inode->i_ctime; + } + + touch_atime(&dst_file->f_path); +} + +struct bpf_prog *fuse_get_bpf_prog(struct file *file) +{ + struct bpf_prog *bpf_prog = ERR_PTR(-EINVAL); + + if (!file || IS_ERR(file)) + return bpf_prog; + /** + * Two ways of getting a bpf prog from another task's fd, since + * bpf_prog_get_type_dev only works with an fd + * + * 1) Duplicate a little of the needed code. Requires access to + * bpf_prog_fops for validation, which is not exported for modules + * 2) Insert the bpf_file object into a fd from the current task + * Stupidly complex, but I think OK, as security checks are not run + * during the existence of the handle + * + * Best would be to upstream 1) into kernel/bpf/syscall.c and export it + * for use here. Failing that, we have to use 2, since fuse must be + * compilable as a module. + */ +#if 0 + if (file->f_op != &bpf_prog_fops) + goto out; + + bpf_prog = file->private_data; + if (bpf_prog->type == BPF_PROG_TYPE_FUSE) + bpf_prog_inc(bpf_prog); + else + bpf_prog = ERR_PTR(-EINVAL); + +#else + { + int task_fd = get_unused_fd_flags(file->f_flags); + + if (task_fd < 0) + goto out; + + fd_install(task_fd, file); + + bpf_prog = bpf_prog_get_type_dev(task_fd, BPF_PROG_TYPE_FUSE, + false); + + /* Close the fd, which also closes the file */ + __close_fd(current->files, task_fd); + file = NULL; + } +#endif + +out: + if (file) + fput(file); + return bpf_prog; +} + +int fuse_open_initialize(struct fuse_bpf_args *fa, struct fuse_open_io *foio, + struct inode *inode, struct file *file, bool isdir) +{ + foio->foi = (struct fuse_open_in) { + .flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY), + }; + + foio->foo = (struct fuse_open_out) {0}; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_fuse_inode(inode)->nodeid, + .opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN, + .in_numargs = 1, + .out_numargs = 1, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(foio->foi), + .value = &foio->foi, + }, + .out_args[0] = (struct fuse_bpf_arg) { + .size = sizeof(foio->foo), + .value = &foio->foo, + }, + }; + + return 0; +} + +int fuse_open_backing(struct fuse_bpf_args *fa, + struct inode *inode, struct file *file, bool isdir) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + const struct fuse_open_in *foi = fa->in_args[0].value; + struct fuse_file *ff; + int retval; + int mask; + struct fuse_dentry *fd = get_fuse_dentry(file->f_path.dentry); + struct file *backing_file; + + ff = fuse_file_alloc(fm); + if (!ff) + return -ENOMEM; + file->private_data = ff; + + switch (foi->flags & O_ACCMODE) { + case O_RDONLY: + mask = MAY_READ; + break; + + case O_WRONLY: + mask = MAY_WRITE; + break; + + case O_RDWR: + mask = MAY_READ | MAY_WRITE; + break; + + default: + return -EINVAL; + } + + retval = inode_permission(get_fuse_inode(inode)->backing_inode, mask); + if (retval) + return retval; + + backing_file = dentry_open(&fd->backing_path, + foi->flags, + current_cred()); + + if (IS_ERR(backing_file)) { + fuse_file_free(ff); + file->private_data = NULL; + return PTR_ERR(backing_file); + } + ff->backing_file = backing_file; + + return 0; +} + +void *fuse_open_finalize(struct fuse_bpf_args *fa, + struct inode *inode, struct file *file, bool isdir) +{ + struct fuse_file *ff = file->private_data; + struct fuse_open_out *foo = fa->out_args[0].value; + + if (ff) { + ff->fh = foo->fh; + ff->nodeid = get_fuse_inode(inode)->nodeid; + } + return 0; +} + +int fuse_create_open_initialize( + struct fuse_bpf_args *fa, struct fuse_create_open_io *fcoio, + struct inode *dir, struct dentry *entry, + struct file *file, unsigned int flags, umode_t mode) +{ + fcoio->fci = (struct fuse_create_in) { + .flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY), + .mode = mode, + }; + + fcoio->feo = (struct fuse_entry_out) {0}; + fcoio->foo = (struct fuse_open_out) {0}; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(dir), + .opcode = FUSE_CREATE, + .in_numargs = 2, + .out_numargs = 2, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(fcoio->fci), + .value = &fcoio->fci, + }, + .in_args[1] = (struct fuse_bpf_in_arg) { + .size = entry->d_name.len + 1, + .value = entry->d_name.name, + }, + .out_args[0] = (struct fuse_bpf_arg) { + .size = sizeof(fcoio->feo), + .value = &fcoio->feo, + }, + .out_args[1] = (struct fuse_bpf_arg) { + .size = sizeof(fcoio->foo), + .value = &fcoio->foo, + }, + }; + + return 0; +} + +static int fuse_open_file_backing(struct inode *inode, struct file *file) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + struct dentry *entry = file->f_path.dentry; + struct fuse_dentry *fuse_dentry = get_fuse_dentry(entry); + struct fuse_file *fuse_file; + struct file *backing_file; + + fuse_file = fuse_file_alloc(fm); + if (!fuse_file) + return -ENOMEM; + file->private_data = fuse_file; + + backing_file = dentry_open(&fuse_dentry->backing_path, file->f_flags, + current_cred()); + if (IS_ERR(backing_file)) { + fuse_file_free(fuse_file); + file->private_data = NULL; + return PTR_ERR(backing_file); + } + fuse_file->backing_file = backing_file; + + return 0; +} + +int fuse_create_open_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, + struct file *file, unsigned int flags, umode_t mode) +{ + struct fuse_inode *dir_fuse_inode = get_fuse_inode(dir); + struct fuse_dentry *fuse_entry = get_fuse_dentry(entry); + struct fuse_dentry *dir_fuse_dentry = get_fuse_dentry(entry->d_parent); + struct dentry *backing_dentry = NULL; + struct inode *inode = NULL; + struct dentry *newent; + int err = 0; + const struct fuse_create_in *fci = fa->in_args[0].value; + struct inode *d_inode = entry->d_inode; + u64 target_nodeid = 0; + + if (!dir_fuse_inode || !dir_fuse_dentry) + return -EIO; + + inode_lock_nested(dir_fuse_inode->backing_inode, I_MUTEX_PARENT); + backing_dentry = lookup_one_len(fa->in_args[1].value, + dir_fuse_dentry->backing_path.dentry, + strlen(fa->in_args[1].value)); + inode_unlock(dir_fuse_inode->backing_inode); + + if (IS_ERR(backing_dentry)) + return PTR_ERR(backing_dentry); + + if (d_really_is_positive(backing_dentry)) { + err = -EIO; + goto out; + } + + err = vfs_create(dir_fuse_inode->backing_inode, backing_dentry, + fci->mode, true); + if (err) + goto out; + + if (fuse_entry->backing_path.dentry) + path_put(&fuse_entry->backing_path); + fuse_entry->backing_path = (struct path) { + .mnt = dir_fuse_dentry->backing_path.mnt, + .dentry = backing_dentry, + }; + path_get(&fuse_entry->backing_path); + + if (d_inode) + target_nodeid = get_fuse_inode(d_inode)->nodeid; + + inode = fuse_iget_backing(dir->i_sb, target_nodeid, + fuse_entry->backing_path.dentry->d_inode); + if (!inode) { + err = -EIO; + goto out; + } + + if (get_fuse_inode(inode)->bpf) + bpf_prog_put(get_fuse_inode(inode)->bpf); + get_fuse_inode(inode)->bpf = fuse_entry->bpf; + fuse_entry->bpf = NULL; + + newent = d_splice_alias(inode, entry); + if (IS_ERR(newent)) { + err = PTR_ERR(newent); + goto out; + } + + inode = NULL; + entry = newent ? newent : entry; + err = finish_open(file, entry, fuse_open_file_backing); + +out: + iput(inode); + dput(backing_dentry); + return err; +} + +void *fuse_create_open_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, + struct file *file, unsigned int flags, umode_t mode) +{ + struct fuse_file *ff = file->private_data; + struct fuse_inode *fi = get_fuse_inode(file->f_inode); + struct fuse_entry_out *feo = fa->out_args[0].value; + struct fuse_open_out *foo = fa->out_args[1].value; + + if (fi) + fi->nodeid = feo->nodeid; + if (ff) + ff->fh = foo->fh; + return 0; +} + +int fuse_release_initialize(struct fuse_bpf_args *fa, struct fuse_release_in *fri, + struct inode *inode, struct fuse_file *ff) +{ + /* Always put backing file whatever bpf/userspace says */ + if (ff->backing_file) { + fput(ff->backing_file); + } + + *fri = (struct fuse_release_in) { + .fh = ff->fh, + }; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_fuse_inode(inode)->nodeid, + .opcode = S_ISDIR(inode->i_mode) ? FUSE_RELEASEDIR + : FUSE_RELEASE, + .in_numargs = 1, + .in_args[0].size = sizeof(*fri), + .in_args[0].value = fri, + }; + + return 0; +} + +int fuse_release_backing(struct fuse_bpf_args *fa, + struct inode *inode, struct fuse_file *ff) +{ + return 0; +} + +void *fuse_release_finalize(struct fuse_bpf_args *fa, + struct inode *inode, struct fuse_file *ff) +{ + return NULL; +} + +int fuse_flush_initialize(struct fuse_bpf_args *fa, struct fuse_flush_in *ffi, + struct file *file, fl_owner_t id) +{ + struct fuse_file *fuse_file = file->private_data; + + *ffi = (struct fuse_flush_in) { + .fh = fuse_file->fh, + }; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(file->f_inode), + .opcode = FUSE_FLUSH, + .in_numargs = 1, + .in_args[0].size = sizeof(*ffi), + .in_args[0].value = ffi, + .flags = FUSE_BPF_FORCE, + }; + + return 0; +} + +int fuse_flush_backing(struct fuse_bpf_args *fa, struct file *file, fl_owner_t id) +{ + struct fuse_file *fuse_file = file->private_data; + struct file *backing_file = fuse_file->backing_file; + + if (backing_file->f_op->flush) + return backing_file->f_op->flush(backing_file, id); + return 0; +} + +void *fuse_flush_finalize(struct fuse_bpf_args *fa, struct file *file, fl_owner_t id) +{ + return NULL; +} + +int fuse_lseek_initialize(struct fuse_bpf_args *fa, struct fuse_lseek_io *flio, + struct file *file, loff_t offset, int whence) +{ + struct fuse_file *fuse_file = file->private_data; + + flio->fli = (struct fuse_lseek_in) { + .fh = fuse_file->fh, + .offset = offset, + .whence = whence, + }; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(file->f_inode), + .opcode = FUSE_LSEEK, + .in_numargs = 1, + .in_args[0].size = sizeof(flio->fli), + .in_args[0].value = &flio->fli, + .out_numargs = 1, + .out_args[0].size = sizeof(flio->flo), + .out_args[0].value = &flio->flo, + }; + + return 0; +} + +int fuse_lseek_backing(struct fuse_bpf_args *fa, struct file *file, loff_t offset, int whence) +{ + const struct fuse_lseek_in *fli = fa->in_args[0].value; + struct fuse_lseek_out *flo = fa->out_args[0].value; + struct fuse_file *fuse_file = file->private_data; + struct file *backing_file = fuse_file->backing_file; + loff_t ret; + + if (offset == 0) { + if (whence == SEEK_CUR) { + flo->offset = file->f_pos; + return 0; + } + + if (whence == SEEK_SET) { + flo->offset = vfs_setpos(file, 0, 0); + return 0; + } + } + + inode_lock(file->f_inode); + backing_file->f_pos = file->f_pos; + ret = vfs_llseek(backing_file, fli->offset, fli->whence); + + if (!IS_ERR(ERR_PTR(ret))) { + flo->offset = ret; + ret = 0; + } + inode_unlock(file->f_inode); + return ret; +} + +void *fuse_lseek_finalize(struct fuse_bpf_args *fa, struct file *file, loff_t offset, int whence) +{ + struct fuse_lseek_out *flo = fa->out_args[0].value; + + if (!fa->error_in) + file->f_pos = flo->offset; + return ERR_PTR(flo->offset); +} + +int fuse_copy_file_range_initialize(struct fuse_bpf_args *fa, struct fuse_copy_file_range_io *fcf, + struct file *file_in, loff_t pos_in, struct file *file_out, + loff_t pos_out, size_t len, unsigned int flags) +{ + struct fuse_file *fuse_file_in = file_in->private_data; + struct fuse_file *fuse_file_out = file_out->private_data; + + + fcf->fci = (struct fuse_copy_file_range_in) { + .fh_in = fuse_file_in->fh, + .off_in = pos_in, + .nodeid_out = fuse_file_out->nodeid, + .fh_out = fuse_file_out->fh, + .off_out = pos_out, + .len = len, + .flags = flags, + }; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(file_in->f_inode), + .opcode = FUSE_COPY_FILE_RANGE, + .in_numargs = 1, + .in_args[0].size = sizeof(fcf->fci), + .in_args[0].value = &fcf->fci, + .out_numargs = 1, + .out_args[0].size = sizeof(fcf->fwo), + .out_args[0].value = &fcf->fwo, + }; + + return 0; +} + +int fuse_copy_file_range_backing(struct fuse_bpf_args *fa, struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, size_t len, + unsigned int flags) +{ + const struct fuse_copy_file_range_in *fci = fa->in_args[0].value; + struct fuse_file *fuse_file_in = file_in->private_data; + struct file *backing_file_in = fuse_file_in->backing_file; + struct fuse_file *fuse_file_out = file_out->private_data; + struct file *backing_file_out = fuse_file_out->backing_file; + + /* TODO: Handle changing of in/out files */ + if (backing_file_out) + return vfs_copy_file_range(backing_file_in, fci->off_in, backing_file_out, + fci->off_out, fci->len, fci->flags); + else + return generic_copy_file_range(file_in, pos_in, file_out, pos_out, len, + flags); +} + +void *fuse_copy_file_range_finalize(struct fuse_bpf_args *fa, struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, size_t len, + unsigned int flags) +{ + return NULL; +} + +int fuse_fsync_initialize(struct fuse_bpf_args *fa, struct fuse_fsync_in *ffi, + struct file *file, loff_t start, loff_t end, int datasync) +{ + struct fuse_file *fuse_file = file->private_data; + + *ffi = (struct fuse_fsync_in) { + .fh = fuse_file->fh, + .fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0, + }; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_fuse_inode(file->f_inode)->nodeid, + .opcode = FUSE_FSYNC, + .in_numargs = 1, + .in_args[0].size = sizeof(*ffi), + .in_args[0].value = ffi, + .flags = FUSE_BPF_FORCE, + }; + + return 0; +} + +int fuse_fsync_backing(struct fuse_bpf_args *fa, + struct file *file, loff_t start, loff_t end, int datasync) +{ + struct fuse_file *fuse_file = file->private_data; + struct file *backing_file = fuse_file->backing_file; + const struct fuse_fsync_in *ffi = fa->in_args[0].value; + int new_datasync = (ffi->fsync_flags & FUSE_FSYNC_FDATASYNC) ? 1 : 0; + + return vfs_fsync(backing_file, new_datasync); +} + +void *fuse_fsync_finalize(struct fuse_bpf_args *fa, + struct file *file, loff_t start, loff_t end, int datasync) +{ + return NULL; +} + +int fuse_dir_fsync_initialize(struct fuse_bpf_args *fa, struct fuse_fsync_in *ffi, + struct file *file, loff_t start, loff_t end, int datasync) +{ + struct fuse_file *fuse_file = file->private_data; + + *ffi = (struct fuse_fsync_in) { + .fh = fuse_file->fh, + .fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0, + }; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_fuse_inode(file->f_inode)->nodeid, + .opcode = FUSE_FSYNCDIR, + .in_numargs = 1, + .in_args[0].size = sizeof(*ffi), + .in_args[0].value = ffi, + .flags = FUSE_BPF_FORCE, + }; + + return 0; +} + +int fuse_getxattr_initialize(struct fuse_bpf_args *fa, + struct fuse_getxattr_io *fgio, + struct dentry *dentry, const char *name, void *value, + size_t size) +{ + *fgio = (struct fuse_getxattr_io) { + .fgi.size = size, + }; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_fuse_inode(dentry->d_inode)->nodeid, + .opcode = FUSE_GETXATTR, + .in_numargs = 2, + .out_numargs = 1, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(fgio->fgi), + .value = &fgio->fgi, + }, + .in_args[1] = (struct fuse_bpf_in_arg) { + .size = strlen(name) + 1, + .value = name, + }, + .flags = size ? FUSE_BPF_OUT_ARGVAR : 0, + .out_args[0].size = size ? size : sizeof(fgio->fgo), + .out_args[0].value = size ? value : &fgio->fgo, + }; + return 0; +} + +int fuse_getxattr_backing(struct fuse_bpf_args *fa, + struct dentry *dentry, const char *name, void *value, + size_t size) +{ + ssize_t ret = vfs_getxattr(get_fuse_dentry(dentry)->backing_path.dentry, + fa->in_args[1].value, value, size); + + if (fa->flags & FUSE_BPF_OUT_ARGVAR) + fa->out_args[0].size = ret; + else + ((struct fuse_getxattr_out *)fa->out_args[0].value)->size = ret; + + return 0; +} + +void *fuse_getxattr_finalize(struct fuse_bpf_args *fa, + struct dentry *dentry, const char *name, void *value, + size_t size) +{ + struct fuse_getxattr_out *fgo; + + if (fa->flags & FUSE_BPF_OUT_ARGVAR) + return ERR_PTR(fa->out_args[0].size); + + fgo = fa->out_args[0].value; + + return ERR_PTR(fgo->size); + +} + +int fuse_listxattr_initialize(struct fuse_bpf_args *fa, + struct fuse_getxattr_io *fgio, + struct dentry *dentry, char *list, size_t size) +{ + *fgio = (struct fuse_getxattr_io){ + .fgi.size = size, + }; + + *fa = (struct fuse_bpf_args){ + .nodeid = get_fuse_inode(dentry->d_inode)->nodeid, + .opcode = FUSE_LISTXATTR, + .in_numargs = 1, + .out_numargs = 1, + .in_args[0] = + (struct fuse_bpf_in_arg){ + .size = sizeof(fgio->fgi), + .value = &fgio->fgi, + }, + .flags = size ? FUSE_BPF_OUT_ARGVAR : 0, + .out_args[0].size = size ? size : sizeof(fgio->fgo), + .out_args[0].value = size ? (void *)list : &fgio->fgo, + }; + + return 0; +} + +int fuse_listxattr_backing(struct fuse_bpf_args *fa, struct dentry *dentry, + char *list, size_t size) +{ + ssize_t ret = + vfs_listxattr(get_fuse_dentry(dentry)->backing_path.dentry, + list, size); + + if (ret < 0) + return ret; + + if (fa->flags & FUSE_BPF_OUT_ARGVAR) + fa->out_args[0].size = ret; + else + ((struct fuse_getxattr_out *)fa->out_args[0].value)->size = ret; + + return ret; +} + +void *fuse_listxattr_finalize(struct fuse_bpf_args *fa, struct dentry *dentry, + char *list, size_t size) +{ + struct fuse_getxattr_out *fgo; + + if (fa->error_in) + return NULL; + + if (fa->flags & FUSE_BPF_OUT_ARGVAR) + return ERR_PTR(fa->out_args[0].size); + + fgo = fa->out_args[0].value; + return ERR_PTR(fgo->size); +} + +int fuse_setxattr_initialize(struct fuse_bpf_args *fa, + struct fuse_setxattr_in *fsxi, + struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) +{ + *fsxi = (struct fuse_setxattr_in) { + .size = size, + .flags = flags, + }; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_fuse_inode(dentry->d_inode)->nodeid, + .opcode = FUSE_SETXATTR, + .in_numargs = 3, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(*fsxi), + .value = fsxi, + }, + .in_args[1] = (struct fuse_bpf_in_arg) { + .size = strlen(name) + 1, + .value = name, + }, + .in_args[2] = (struct fuse_bpf_in_arg) { + .size = size, + .value = value, + }, + }; + + return 0; +} + +int fuse_setxattr_backing(struct fuse_bpf_args *fa, struct dentry *dentry, + const char *name, const void *value, size_t size, + int flags) +{ + return vfs_setxattr(get_fuse_dentry(dentry)->backing_path.dentry, name, + value, size, flags); +} + +void *fuse_setxattr_finalize(struct fuse_bpf_args *fa, struct dentry *dentry, + const char *name, const void *value, size_t size, + int flags) +{ + return NULL; +} + +int fuse_removexattr_initialize(struct fuse_bpf_args *fa, + struct fuse_dummy_io *unused, + struct dentry *dentry, const char *name) +{ + *fa = (struct fuse_bpf_args) { + .nodeid = get_fuse_inode(dentry->d_inode)->nodeid, + .opcode = FUSE_REMOVEXATTR, + .in_numargs = 1, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = strlen(name) + 1, + .value = name, + }, + }; + + return 0; +} + +int fuse_removexattr_backing(struct fuse_bpf_args *fa, + struct dentry *dentry, const char *name) +{ + struct path *backing_path = + &get_fuse_dentry(dentry)->backing_path; + + /* TODO account for changes of the name by prefilter */ + return vfs_removexattr(backing_path->dentry, name); +} + +void *fuse_removexattr_finalize(struct fuse_bpf_args *fa, + struct dentry *dentry, const char *name) +{ + return NULL; +} + +static inline void fuse_bpf_aio_put(struct fuse_bpf_aio_req *aio_req) +{ + if (refcount_dec_and_test(&aio_req->ref)) + kmem_cache_free(fuse_bpf_aio_request_cachep, aio_req); +} + +static void fuse_bpf_aio_cleanup_handler(struct fuse_bpf_aio_req *aio_req) +{ + struct kiocb *iocb = &aio_req->iocb; + struct kiocb *iocb_orig = aio_req->iocb_orig; + + if (iocb->ki_flags & IOCB_WRITE) { + __sb_writers_acquired(file_inode(iocb->ki_filp)->i_sb, + SB_FREEZE_WRITE); + file_end_write(iocb->ki_filp); + fuse_copyattr(iocb_orig->ki_filp, iocb->ki_filp); + } + iocb_orig->ki_pos = iocb->ki_pos; + fuse_bpf_aio_put(aio_req); +} + +static void fuse_bpf_aio_rw_complete(struct kiocb *iocb, long res, long res2) +{ + struct fuse_bpf_aio_req *aio_req = + container_of(iocb, struct fuse_bpf_aio_req, iocb); + struct kiocb *iocb_orig = aio_req->iocb_orig; + + fuse_bpf_aio_cleanup_handler(aio_req); + iocb_orig->ki_complete(iocb_orig, res, res2); +} + + +int fuse_file_read_iter_initialize( + struct fuse_bpf_args *fa, struct fuse_file_read_iter_io *fri, + struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + + fri->fri = (struct fuse_read_in) { + .fh = ff->fh, + .offset = iocb->ki_pos, + .size = to->count, + }; + + fri->frio = (struct fuse_read_iter_out) { + .ret = fri->fri.size, + }; + + /* TODO we can't assume 'to' is a kvec */ + /* TODO we also can't assume the vector has only one component */ + *fa = (struct fuse_bpf_args) { + .opcode = FUSE_READ, + .nodeid = ff->nodeid, + .in_numargs = 1, + .in_args[0].size = sizeof(fri->fri), + .in_args[0].value = &fri->fri, + .out_numargs = 1, + .out_args[0].size = sizeof(fri->frio), + .out_args[0].value = &fri->frio, + /* + * TODO Design this properly. + * Possible approach: do not pass buf to bpf + * If going to userland, do a deep copy + * For extra credit, do that to/from the vector, rather than + * making an extra copy in the kernel + */ + }; + + return 0; +} + +int fuse_file_read_iter_backing(struct fuse_bpf_args *fa, + struct kiocb *iocb, struct iov_iter *to) +{ + struct fuse_read_iter_out *frio = fa->out_args[0].value; + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + ssize_t ret; + + if (!iov_iter_count(to)) + return 0; + + if ((iocb->ki_flags & IOCB_DIRECT) && + (!ff->backing_file->f_mapping->a_ops || + !ff->backing_file->f_mapping->a_ops->direct_IO)) + return -EINVAL; + + /* TODO This just plain ignores any change to fuse_read_in */ + if (is_sync_kiocb(iocb)) { + ret = vfs_iter_read(ff->backing_file, to, &iocb->ki_pos, + iocb_to_rw_flags(iocb->ki_flags, FUSE_BPF_IOCB_MASK)); + } else { + struct fuse_bpf_aio_req *aio_req; + + ret = -ENOMEM; + aio_req = kmem_cache_zalloc(fuse_bpf_aio_request_cachep, GFP_KERNEL); + if (!aio_req) + goto out; + + aio_req->iocb_orig = iocb; + kiocb_clone(&aio_req->iocb, iocb, ff->backing_file); + aio_req->iocb.ki_complete = fuse_bpf_aio_rw_complete; + refcount_set(&aio_req->ref, 2); + ret = vfs_iocb_iter_read(ff->backing_file, &aio_req->iocb, to); + fuse_bpf_aio_put(aio_req); + if (ret != -EIOCBQUEUED) + fuse_bpf_aio_cleanup_handler(aio_req); + } + + frio->ret = ret; + + /* TODO Need to point value at the buffer for post-modification */ + +out: + fuse_file_accessed(file, ff->backing_file); + + return ret; +} + +void *fuse_file_read_iter_finalize(struct fuse_bpf_args *fa, + struct kiocb *iocb, struct iov_iter *to) +{ + struct fuse_read_iter_out *frio = fa->out_args[0].value; + + return ERR_PTR(frio->ret); +} + +int fuse_file_write_iter_initialize( + struct fuse_bpf_args *fa, struct fuse_file_write_iter_io *fwio, + struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + + *fwio = (struct fuse_file_write_iter_io) { + .fwi.fh = ff->fh, + .fwi.offset = iocb->ki_pos, + .fwi.size = from->count, + }; + + /* TODO we can't assume 'from' is a kvec */ + *fa = (struct fuse_bpf_args) { + .opcode = FUSE_WRITE, + .nodeid = ff->nodeid, + .in_numargs = 2, + .in_args[0].size = sizeof(fwio->fwi), + .in_args[0].value = &fwio->fwi, + .in_args[1].size = fwio->fwi.size, + .in_args[1].value = from->kvec->iov_base, + .out_numargs = 1, + .out_args[0].size = sizeof(fwio->fwio), + .out_args[0].value = &fwio->fwio, + }; + + return 0; +} + +int fuse_file_write_iter_backing(struct fuse_bpf_args *fa, + struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct fuse_write_iter_out *fwio = fa->out_args[0].value; + ssize_t ret; + + if (!iov_iter_count(from)) + return 0; + + /* TODO This just plain ignores any change to fuse_write_in */ + /* TODO uint32_t seems smaller than ssize_t.... right? */ + inode_lock(file_inode(file)); + + fuse_copyattr(file, ff->backing_file); + + if (is_sync_kiocb(iocb)) { + file_start_write(ff->backing_file); + ret = vfs_iter_write(ff->backing_file, from, &iocb->ki_pos, + iocb_to_rw_flags(iocb->ki_flags, FUSE_BPF_IOCB_MASK)); + file_end_write(ff->backing_file); + + /* Must reflect change in size of backing file to upper file */ + if (ret > 0) + fuse_copyattr(file, ff->backing_file); + } else { + struct fuse_bpf_aio_req *aio_req; + + ret = -ENOMEM; + aio_req = kmem_cache_zalloc(fuse_bpf_aio_request_cachep, GFP_KERNEL); + if (!aio_req) + goto out; + + file_start_write(ff->backing_file); + __sb_writers_release(file_inode(ff->backing_file)->i_sb, SB_FREEZE_WRITE); + aio_req->iocb_orig = iocb; + kiocb_clone(&aio_req->iocb, iocb, ff->backing_file); + aio_req->iocb.ki_complete = fuse_bpf_aio_rw_complete; + refcount_set(&aio_req->ref, 2); + ret = vfs_iocb_iter_write(ff->backing_file, &aio_req->iocb, from); + fuse_bpf_aio_put(aio_req); + if (ret != -EIOCBQUEUED) + fuse_bpf_aio_cleanup_handler(aio_req); + } + +out: + inode_unlock(file_inode(file)); + fwio->ret = ret; + if (ret < 0) + return ret; + return 0; +} + +void *fuse_file_write_iter_finalize(struct fuse_bpf_args *fa, + struct kiocb *iocb, struct iov_iter *from) +{ + struct fuse_write_iter_out *fwio = fa->out_args[0].value; + + return ERR_PTR(fwio->ret); +} + +long fuse_backing_ioctl(struct file *file, unsigned int command, unsigned long arg, int flags) +{ + struct fuse_file *ff = file->private_data; + long ret; + + if (flags & FUSE_IOCTL_COMPAT) + ret = -ENOTTY; + else + ret = vfs_ioctl(ff->backing_file, command, arg); + + return ret; +} + +int fuse_file_flock_backing(struct file *file, int cmd, struct file_lock *fl) +{ + struct fuse_file *ff = file->private_data; + struct file *backing_file = ff->backing_file; + int error; + + fl->fl_file = backing_file; + if (backing_file->f_op->flock) + error = backing_file->f_op->flock(backing_file, cmd, fl); + else + error = locks_lock_file_wait(backing_file, fl); + return error; +} + +ssize_t fuse_backing_mmap(struct file *file, struct vm_area_struct *vma) +{ + int ret; + struct fuse_file *ff = file->private_data; + struct inode *fuse_inode = file_inode(file); + struct file *backing_file = ff->backing_file; + struct inode *backing_inode = file_inode(backing_file); + + if (!backing_file->f_op->mmap) + return -ENODEV; + + if (WARN_ON(file != vma->vm_file)) + return -EIO; + + get_file(backing_file); + swap(vma->vm_file, backing_file); + fput(backing_file); + ret = call_mmap(vma->vm_file, vma); + if (ret) + return ret; + + if (file->f_flags & O_NOATIME) + return ret; + + if ((!timespec64_equal(&fuse_inode->i_mtime, + &backing_inode->i_mtime) || + !timespec64_equal(&fuse_inode->i_ctime, + &backing_inode->i_ctime))) { + fuse_inode->i_mtime = backing_inode->i_mtime; + fuse_inode->i_ctime = backing_inode->i_ctime; + } + touch_atime(&file->f_path); + + return ret; +} + +int fuse_file_fallocate_initialize(struct fuse_bpf_args *fa, + struct fuse_fallocate_in *ffi, + struct file *file, int mode, loff_t offset, loff_t length) +{ + struct fuse_file *ff = file->private_data; + + *ffi = (struct fuse_fallocate_in) { + .fh = ff->fh, + .offset = offset, + .length = length, + .mode = mode + }; + + *fa = (struct fuse_bpf_args) { + .opcode = FUSE_FALLOCATE, + .nodeid = ff->nodeid, + .in_numargs = 1, + .in_args[0].size = sizeof(*ffi), + .in_args[0].value = ffi, + }; + + return 0; +} + +int fuse_file_fallocate_backing(struct fuse_bpf_args *fa, + struct file *file, int mode, loff_t offset, loff_t length) +{ + const struct fuse_fallocate_in *ffi = fa->in_args[0].value; + struct fuse_file *ff = file->private_data; + + return vfs_fallocate(ff->backing_file, ffi->mode, ffi->offset, + ffi->length); +} + +void *fuse_file_fallocate_finalize(struct fuse_bpf_args *fa, + struct file *file, int mode, loff_t offset, loff_t length) +{ + return NULL; +} + +/******************************************************************************* + * Directory operations after here * + ******************************************************************************/ + +int fuse_lookup_initialize(struct fuse_bpf_args *fa, struct fuse_lookup_io *fli, + struct inode *dir, struct dentry *entry, unsigned int flags) +{ + *fa = (struct fuse_bpf_args) { + .nodeid = get_fuse_inode(dir)->nodeid, + .opcode = FUSE_LOOKUP, + .in_numargs = 1, + .out_numargs = 2, + .flags = FUSE_BPF_OUT_ARGVAR, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = entry->d_name.len + 1, + .value = entry->d_name.name, + }, + .out_args[0] = (struct fuse_bpf_arg) { + .size = sizeof(fli->feo), + .value = &fli->feo, + }, + .out_args[1] = (struct fuse_bpf_arg) { + .size = sizeof(fli->feb.out), + .value = &fli->feb.out, + }, + }; + + return 0; +} + +int fuse_lookup_backing(struct fuse_bpf_args *fa, struct inode *dir, + struct dentry *entry, unsigned int flags) +{ + struct fuse_dentry *fuse_entry = get_fuse_dentry(entry); + struct fuse_dentry *dir_fuse_entry = get_fuse_dentry(entry->d_parent); + struct dentry *dir_backing_entry = dir_fuse_entry->backing_path.dentry; + struct inode *dir_backing_inode = dir_backing_entry->d_inode; + struct dentry *backing_entry; + struct fuse_entry_out *feo = (void *)fa->out_args[0].value; + struct kstat stat; + int err; + + inode_lock_nested(dir_backing_inode, I_MUTEX_PARENT); + backing_entry = lookup_one_len(entry->d_name.name, dir_backing_entry, + strlen(entry->d_name.name)); + inode_unlock(dir_backing_inode); + + if (IS_ERR(backing_entry)) + return PTR_ERR(backing_entry); + + fuse_entry->backing_path = (struct path) { + .dentry = backing_entry, + .mnt = mntget(dir_fuse_entry->backing_path.mnt), + }; + + if (d_is_negative(backing_entry)) { + fa->error_in = -ENOENT; + return 0; + } + + err = follow_down(&fuse_entry->backing_path); + if (err) + goto err_out; + + err = vfs_getattr(&fuse_entry->backing_path, &stat, + STATX_BASIC_STATS, 0); + if (err) + goto err_out; + + fuse_stat_to_attr(get_fuse_conn(dir), + backing_entry->d_inode, &stat, &feo->attr); + return 0; + +err_out: + path_put_init(&fuse_entry->backing_path); + return err; +} + +int fuse_handle_backing(struct fuse_entry_bpf *feb, struct inode **backing_inode, + struct path *backing_path) { + switch (feb->out.backing_action) { + case FUSE_ACTION_KEEP: + /* backing inode/path are added in fuse_lookup_backing */ + break; + + case FUSE_ACTION_REMOVE: + iput(*backing_inode); + *backing_inode = NULL; + path_put_init(backing_path); + break; + + case FUSE_ACTION_REPLACE: { + struct file *backing_file = feb->backing_file; + + if (!backing_file) + return -EINVAL; + if (IS_ERR(backing_file)) + return PTR_ERR(backing_file); + + if (backing_inode) + iput(*backing_inode); + *backing_inode = backing_file->f_inode; + ihold(*backing_inode); + + path_put(backing_path); + *backing_path = backing_file->f_path; + path_get(backing_path); + break; + } + + default: + return -EINVAL; + } + + return 0; +} + +int fuse_handle_bpf_prog(struct fuse_entry_bpf *feb, struct inode *parent, + struct bpf_prog **bpf) +{ + struct bpf_prog *new_bpf = NULL; + + switch (feb->out.bpf_action) { + case FUSE_ACTION_KEEP: { + /* Parent isn't presented, but we want to keep + * Don't touch bpf program at all in this case + */ + if (!parent) + return 0; + + new_bpf = get_fuse_inode(parent)->bpf; + if (new_bpf) + bpf_prog_inc(new_bpf); + break; + } + + case FUSE_ACTION_REMOVE: + break; + + case FUSE_ACTION_REPLACE: { + struct file *bpf_file = feb->bpf_file; + + if (!bpf_file) + return -EINVAL; + if (IS_ERR(bpf_file)) + return PTR_ERR(bpf_file); + + new_bpf = fuse_get_bpf_prog(bpf_file); + if (IS_ERR(new_bpf)) + return PTR_ERR(new_bpf); + break; + } + + default: + return -EINVAL; + } + + /* Cannot change existing program */ + if (*bpf) { + if (new_bpf) + bpf_prog_put(new_bpf); + return new_bpf == *bpf ? 0 : -EINVAL; + } + + *bpf = new_bpf; + return 0; +} + +struct dentry *fuse_lookup_finalize(struct fuse_bpf_args *fa, struct inode *dir, + struct dentry *entry, unsigned int flags) +{ + struct fuse_dentry *fuse_entry; + struct dentry *backing_entry; + struct inode *inode = NULL, *backing_inode; + struct inode *entry_inode = entry->d_inode; + struct fuse_entry_out *feo = fa->out_args[0].value; + struct fuse_entry_bpf_out *febo = fa->out_args[1].value; + struct fuse_entry_bpf *feb = container_of(febo, struct fuse_entry_bpf, + out); + int error = -1; + u64 target_nodeid = 0; + struct dentry *ret = NULL; + + fuse_entry = get_fuse_dentry(entry); + if (!fuse_entry) { + ret = ERR_PTR(-EIO); + goto out; + } + + backing_entry = fuse_entry->backing_path.dentry; + if (!backing_entry) { + ret = ERR_PTR(-ENOENT); + goto out; + } + + if (entry_inode) + target_nodeid = get_fuse_inode(entry_inode)->nodeid; + + backing_inode = backing_entry->d_inode; + if (backing_inode) + inode = fuse_iget_backing(dir->i_sb, target_nodeid, + backing_inode); + + error = inode ? + fuse_handle_bpf_prog(feb, dir, &get_fuse_inode(inode)->bpf) : + fuse_handle_bpf_prog(feb, dir, &fuse_entry->bpf); + if (error) { + ret = ERR_PTR(error); + goto out; + } + + if (inode) { + error = fuse_handle_backing(feb, + &get_fuse_inode(inode)->backing_inode, + &fuse_entry->backing_path); + if (error) { + ret = ERR_PTR(error); + goto out; + } + + get_fuse_inode(inode)->nodeid = feo->nodeid; + ret = d_splice_alias(inode, entry); + if (!IS_ERR(ret)) + inode = NULL; + } +out: + iput(inode); + if (feb->backing_file) + fput(feb->backing_file); + return ret; +} + +int fuse_revalidate_backing(struct dentry *entry, unsigned int flags) +{ + struct fuse_dentry *fuse_dentry = get_fuse_dentry(entry); + struct dentry *backing_entry = fuse_dentry->backing_path.dentry; + + spin_lock(&backing_entry->d_lock); + if (d_unhashed(backing_entry)) { + spin_unlock(&backing_entry->d_lock); + return 0; + } + spin_unlock(&backing_entry->d_lock); + + if (unlikely(backing_entry->d_flags & DCACHE_OP_REVALIDATE)) + return backing_entry->d_op->d_revalidate(backing_entry, flags); + return 1; +} + +int fuse_canonical_path_initialize(struct fuse_bpf_args *fa, + struct fuse_dummy_io *fdi, + const struct path *path, + struct path *canonical_path) +{ + fa->opcode = FUSE_CANONICAL_PATH; + return 0; +} + +int fuse_canonical_path_backing(struct fuse_bpf_args *fa, const struct path *path, + struct path *canonical_path) +{ + get_fuse_backing_path(path->dentry, canonical_path); + return 0; +} + +void *fuse_canonical_path_finalize(struct fuse_bpf_args *fa, + const struct path *path, + struct path *canonical_path) +{ + return NULL; +} + +int fuse_mknod_initialize( + struct fuse_bpf_args *fa, struct fuse_mknod_in *fmi, + struct inode *dir, struct dentry *entry, umode_t mode, dev_t rdev) +{ + *fmi = (struct fuse_mknod_in) { + .mode = mode, + .rdev = new_encode_dev(rdev), + .umask = current_umask(), + }; + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(dir), + .opcode = FUSE_MKNOD, + .in_numargs = 2, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(*fmi), + .value = fmi, + }, + .in_args[1] = (struct fuse_bpf_in_arg) { + .size = entry->d_name.len + 1, + .value = entry->d_name.name, + }, + }; + + return 0; +} + +int fuse_mknod_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, umode_t mode, dev_t rdev) +{ + int err = 0; + const struct fuse_mknod_in *fmi = fa->in_args[0].value; + struct fuse_inode *fuse_inode = get_fuse_inode(dir); + struct inode *backing_inode = fuse_inode->backing_inode; + struct path backing_path = {}; + struct inode *inode = NULL; + + //TODO Actually deal with changing the backing entry in mknod + get_fuse_backing_path(entry, &backing_path); + if (!backing_path.dentry) + return -EBADF; + + inode_lock_nested(backing_inode, I_MUTEX_PARENT); + mode = fmi->mode; + if (!IS_POSIXACL(backing_inode)) + mode &= ~fmi->umask; + err = vfs_mknod(backing_inode, backing_path.dentry, + mode, new_decode_dev(fmi->rdev)); + inode_unlock(backing_inode); + if (err) + goto out; + if (d_really_is_negative(backing_path.dentry) || + unlikely(d_unhashed(backing_path.dentry))) { + err = -EINVAL; + /** + * TODO: overlayfs responds to this situation with a + * lookupOneLen. Should we do that too? + */ + goto out; + } + inode = fuse_iget_backing(dir->i_sb, fuse_inode->nodeid, backing_inode); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out; + } + d_instantiate(entry, inode); +out: + path_put(&backing_path); + return err; +} + +void *fuse_mknod_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, umode_t mode, dev_t rdev) +{ + return NULL; +} + +int fuse_mkdir_initialize( + struct fuse_bpf_args *fa, struct fuse_mkdir_in *fmi, + struct inode *dir, struct dentry *entry, umode_t mode) +{ + *fmi = (struct fuse_mkdir_in) { + .mode = mode, + .umask = current_umask(), + }; + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(dir), + .opcode = FUSE_MKDIR, + .in_numargs = 2, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(*fmi), + .value = fmi, + }, + .in_args[1] = (struct fuse_bpf_in_arg) { + .size = entry->d_name.len + 1, + .value = entry->d_name.name, + }, + }; + + return 0; +} + +int fuse_mkdir_backing( + struct fuse_bpf_args *fa, + struct inode *dir_inode, struct dentry *entry, umode_t mode) +{ + int err = 0; + const struct fuse_mkdir_in *fmi = fa->in_args[0].value; + struct fuse_inode *dir_fuse_inode = get_fuse_inode(dir_inode); + struct inode *dir_backing_inode = dir_fuse_inode->backing_inode; + struct path backing_path = {}; + struct inode *inode = NULL; + + //TODO Actually deal with changing the backing entry in mkdir + get_fuse_backing_path(entry, &backing_path); + if (!backing_path.dentry) + return -EBADF; + + inode_lock_nested(dir_backing_inode, I_MUTEX_PARENT); + mode = fmi->mode; + if (!IS_POSIXACL(dir_backing_inode)) + mode &= ~fmi->umask; + err = vfs_mkdir(dir_backing_inode, backing_path.dentry, mode); + if (err) + goto out; + if (d_really_is_negative(backing_path.dentry) || + unlikely(d_unhashed(backing_path.dentry))) { + struct dentry *d = lookup_one_len(entry->d_name.name, + backing_path.dentry->d_parent, + entry->d_name.len); + + if (IS_ERR(d)) { + err = PTR_ERR(d); + goto out; + } + dput(backing_path.dentry); + backing_path.dentry = d; + } + inode = fuse_iget_backing(dir_inode->i_sb, 0, + backing_path.dentry->d_inode); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out; + } + d_instantiate(entry, inode); + if (get_fuse_inode(inode)->bpf) + bpf_prog_put(get_fuse_inode(inode)->bpf); + get_fuse_inode(inode)->bpf = get_fuse_dentry(entry)->bpf; + get_fuse_dentry(entry)->bpf = NULL; +out: + inode_unlock(dir_backing_inode); + path_put(&backing_path); + return err; +} + +void *fuse_mkdir_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, umode_t mode) +{ + return NULL; +} + +int fuse_rmdir_initialize( + struct fuse_bpf_args *fa, struct fuse_dummy_io *dummy, + struct inode *dir, struct dentry *entry) +{ + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(dir), + .opcode = FUSE_RMDIR, + .in_numargs = 1, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = entry->d_name.len + 1, + .value = entry->d_name.name, + }, + }; + + return 0; +} + +int fuse_rmdir_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry) +{ + int err = 0; + struct path backing_path = {}; + struct dentry *backing_parent_dentry; + struct inode *backing_inode; + + /* TODO Actually deal with changing the backing entry in rmdir */ + get_fuse_backing_path(entry, &backing_path); + if (!backing_path.dentry) + return -EBADF; + + /* TODO Not sure if we should reverify like overlayfs, or get inode from d_parent */ + backing_parent_dentry = dget_parent(backing_path.dentry); + backing_inode = d_inode(backing_parent_dentry); + + inode_lock_nested(backing_inode, I_MUTEX_PARENT); + err = vfs_rmdir(backing_inode, backing_path.dentry); + inode_unlock(backing_inode); + + dput(backing_parent_dentry); + if (!err) + d_drop(entry); + path_put(&backing_path); + return err; +} + +void *fuse_rmdir_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry) +{ + return NULL; +} + +static int fuse_rename_backing_common( + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent, + unsigned int flags) +{ + int err = 0; + struct path old_backing_path; + struct path new_backing_path; + struct dentry *old_backing_dir_dentry; + struct dentry *old_backing_dentry; + struct dentry *new_backing_dir_dentry; + struct dentry *new_backing_dentry; + struct dentry *trap = NULL; + struct inode *target_inode; + + //TODO Actually deal with changing anything that isn't a flag + get_fuse_backing_path(oldent, &old_backing_path); + if (!old_backing_path.dentry) + return -EBADF; + get_fuse_backing_path(newent, &new_backing_path); + if (!new_backing_path.dentry) { + /* + * TODO A file being moved from a backing path to another + * backing path which is not yet instrumented with FUSE-BPF. + * This may be slow and should be substituted with something + * more clever. + */ + err = -EXDEV; + goto put_old_path; + } + if (new_backing_path.mnt != old_backing_path.mnt) { + err = -EXDEV; + goto put_new_path; + } + old_backing_dentry = old_backing_path.dentry; + new_backing_dentry = new_backing_path.dentry; + old_backing_dir_dentry = dget_parent(old_backing_dentry); + new_backing_dir_dentry = dget_parent(new_backing_dentry); + target_inode = d_inode(newent); + + trap = lock_rename(old_backing_dir_dentry, new_backing_dir_dentry); + if (trap == old_backing_dentry) { + err = -EINVAL; + goto put_parents; + } + if (trap == new_backing_dentry) { + err = -ENOTEMPTY; + goto put_parents; + } + err = vfs_rename(d_inode(old_backing_dir_dentry), old_backing_dentry, + d_inode(new_backing_dir_dentry), new_backing_dentry, + NULL, flags); + if (err) + goto unlock; + if (target_inode) + fsstack_copy_attr_all(target_inode, + get_fuse_inode(target_inode)->backing_inode); + fsstack_copy_attr_all(d_inode(oldent), d_inode(old_backing_dentry)); +unlock: + unlock_rename(old_backing_dir_dentry, new_backing_dir_dentry); +put_parents: + dput(new_backing_dir_dentry); + dput(old_backing_dir_dentry); +put_new_path: + path_put(&new_backing_path); +put_old_path: + path_put(&old_backing_path); + return err; +} + +int fuse_rename2_initialize(struct fuse_bpf_args *fa, struct fuse_rename2_in *fri, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent, + unsigned int flags) +{ + *fri = (struct fuse_rename2_in) { + .newdir = get_node_id(newdir), + .flags = flags, + }; + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(olddir), + .opcode = FUSE_RENAME2, + .in_numargs = 3, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(*fri), + .value = fri, + }, + .in_args[1] = (struct fuse_bpf_in_arg) { + .size = oldent->d_name.len + 1, + .value = oldent->d_name.name, + }, + .in_args[2] = (struct fuse_bpf_in_arg) { + .size = newent->d_name.len + 1, + .value = newent->d_name.name, + }, + }; + + return 0; +} + +int fuse_rename2_backing(struct fuse_bpf_args *fa, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent, + unsigned int flags) +{ + const struct fuse_rename2_in *fri = fa->in_args[0].value; + + /* TODO: deal with changing dirs/ents */ + return fuse_rename_backing_common(olddir, oldent, newdir, newent, fri->flags); +} + +void *fuse_rename2_finalize(struct fuse_bpf_args *fa, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent, + unsigned int flags) +{ + return NULL; +} + +int fuse_rename_initialize(struct fuse_bpf_args *fa, struct fuse_rename_in *fri, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent) +{ + *fri = (struct fuse_rename_in) { + .newdir = get_node_id(newdir), + }; + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(olddir), + .opcode = FUSE_RENAME, + .in_numargs = 3, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(*fri), + .value = fri, + }, + .in_args[1] = (struct fuse_bpf_in_arg) { + .size = oldent->d_name.len + 1, + .value = oldent->d_name.name, + }, + .in_args[2] = (struct fuse_bpf_in_arg) { + .size = newent->d_name.len + 1, + .value = newent->d_name.name, + }, + }; + + return 0; +} + +int fuse_rename_backing(struct fuse_bpf_args *fa, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent) +{ + /* TODO: deal with changing dirs/ents */ + return fuse_rename_backing_common(olddir, oldent, newdir, newent, 0); +} + +void *fuse_rename_finalize(struct fuse_bpf_args *fa, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent) +{ + return NULL; +} + +int fuse_unlink_initialize( + struct fuse_bpf_args *fa, struct fuse_dummy_io *dummy, + struct inode *dir, struct dentry *entry) +{ + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(dir), + .opcode = FUSE_UNLINK, + .in_numargs = 1, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = entry->d_name.len + 1, + .value = entry->d_name.name, + }, + }; + + return 0; +} + +int fuse_unlink_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry) +{ + int err = 0; + struct path backing_path = {}; + struct dentry *backing_parent_dentry; + struct inode *backing_inode; + + /* TODO Actually deal with changing the backing entry in unlink */ + get_fuse_backing_path(entry, &backing_path); + if (!backing_path.dentry) + return -EBADF; + + /* TODO Not sure if we should reverify like overlayfs, or get inode from d_parent */ + backing_parent_dentry = dget_parent(backing_path.dentry); + backing_inode = d_inode(backing_parent_dentry); + + inode_lock_nested(backing_inode, I_MUTEX_PARENT); + err = vfs_unlink(backing_inode, backing_path.dentry, NULL); + inode_unlock(backing_inode); + + dput(backing_parent_dentry); + if (!err) + d_drop(entry); + path_put(&backing_path); + return err; +} + +void *fuse_unlink_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry) +{ + return NULL; +} + +int fuse_link_initialize(struct fuse_bpf_args *fa, struct fuse_link_in *fli, + struct dentry *entry, struct inode *dir, + struct dentry *newent) +{ + struct inode *src_inode = entry->d_inode; + + *fli = (struct fuse_link_in){ + .oldnodeid = get_node_id(src_inode), + }; + + fa->opcode = FUSE_LINK; + fa->in_numargs = 2; + fa->in_args[0].size = sizeof(*fli); + fa->in_args[0].value = fli; + fa->in_args[1].size = newent->d_name.len + 1; + fa->in_args[1].value = newent->d_name.name; + + return 0; +} + +int fuse_link_backing(struct fuse_bpf_args *fa, struct dentry *entry, + struct inode *dir, struct dentry *newent) +{ + int err = 0; + struct path backing_old_path = {}; + struct path backing_new_path = {}; + struct dentry *backing_dir_dentry; + struct inode *fuse_new_inode = NULL; + struct fuse_inode *fuse_dir_inode = get_fuse_inode(dir); + struct inode *backing_dir_inode = fuse_dir_inode->backing_inode; + + get_fuse_backing_path(entry, &backing_old_path); + if (!backing_old_path.dentry) + return -EBADF; + + get_fuse_backing_path(newent, &backing_new_path); + if (!backing_new_path.dentry) { + err = -EBADF; + goto err_dst_path; + } + + backing_dir_dentry = dget_parent(backing_new_path.dentry); + backing_dir_inode = d_inode(backing_dir_dentry); + + inode_lock_nested(backing_dir_inode, I_MUTEX_PARENT); + err = vfs_link(backing_old_path.dentry, backing_dir_inode, backing_new_path.dentry, NULL); + inode_unlock(backing_dir_inode); + if (err) + goto out; + + if (d_really_is_negative(backing_new_path.dentry) || + unlikely(d_unhashed(backing_new_path.dentry))) { + err = -EINVAL; + /** + * TODO: overlayfs responds to this situation with a + * lookupOneLen. Should we do that too? + */ + goto out; + } + + fuse_new_inode = fuse_iget_backing(dir->i_sb, fuse_dir_inode->nodeid, backing_dir_inode); + if (IS_ERR(fuse_new_inode)) { + err = PTR_ERR(fuse_new_inode); + goto out; + } + d_instantiate(newent, fuse_new_inode); + +out: + dput(backing_dir_dentry); + path_put(&backing_new_path); +err_dst_path: + path_put(&backing_old_path); + return err; +} + +void *fuse_link_finalize(struct fuse_bpf_args *fa, struct dentry *entry, + struct inode *dir, struct dentry *newent) +{ + return NULL; +} + +int fuse_getattr_initialize(struct fuse_bpf_args *fa, struct fuse_getattr_io *fgio, + const struct dentry *entry, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + fgio->fgi = (struct fuse_getattr_in) { + .getattr_flags = flags, + .fh = -1, /* TODO is this OK? */ + }; + + fgio->fao = (struct fuse_attr_out) {0}; + + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(entry->d_inode), + .opcode = FUSE_GETATTR, + .in_numargs = 1, + .out_numargs = 1, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(fgio->fgi), + .value = &fgio->fgi, + }, + .out_args[0] = (struct fuse_bpf_arg) { + .size = sizeof(fgio->fao), + .value = &fgio->fao, + }, + }; + + return 0; +} + +static void fuse_stat_to_attr(struct fuse_conn *fc, struct inode *inode, + struct kstat *stat, struct fuse_attr *attr) +{ + unsigned int blkbits; + + /* see the comment in fuse_change_attributes() */ + if (fc->writeback_cache && S_ISREG(inode->i_mode)) { + stat->size = i_size_read(inode); + stat->mtime.tv_sec = inode->i_mtime.tv_sec; + stat->mtime.tv_nsec = inode->i_mtime.tv_nsec; + stat->ctime.tv_sec = inode->i_ctime.tv_sec; + stat->ctime.tv_nsec = inode->i_ctime.tv_nsec; + } + + attr->ino = stat->ino; + attr->mode = (inode->i_mode & S_IFMT) | (stat->mode & 07777); + attr->nlink = stat->nlink; + attr->uid = from_kuid(fc->user_ns, stat->uid); + attr->gid = from_kgid(fc->user_ns, stat->gid); + attr->atime = stat->atime.tv_sec; + attr->atimensec = stat->atime.tv_nsec; + attr->mtime = stat->mtime.tv_sec; + attr->mtimensec = stat->mtime.tv_nsec; + attr->ctime = stat->ctime.tv_sec; + attr->ctimensec = stat->ctime.tv_nsec; + attr->size = stat->size; + attr->blocks = stat->blocks; + + if (stat->blksize != 0) + blkbits = ilog2(stat->blksize); + else + blkbits = inode->i_sb->s_blocksize_bits; + + attr->blksize = 1 << blkbits; +} + +int fuse_getattr_backing(struct fuse_bpf_args *fa, + const struct dentry *entry, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct path *backing_path = + &get_fuse_dentry(entry)->backing_path; + struct inode *backing_inode = backing_path->dentry->d_inode; + struct fuse_attr_out *fao = fa->out_args[0].value; + struct kstat tmp; + int err; + + if (!stat) + stat = &tmp; + + err = vfs_getattr(backing_path, stat, request_mask, flags); + + if (!err) + fuse_stat_to_attr(get_fuse_conn(entry->d_inode), + backing_inode, stat, &fao->attr); + + return err; +} + +void *fuse_getattr_finalize(struct fuse_bpf_args *fa, + const struct dentry *entry, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct fuse_attr_out *outarg = fa->out_args[0].value; + struct inode *inode = entry->d_inode; + u64 attr_version = fuse_get_attr_version(get_fuse_mount(inode)->fc); + int err = 0; + + /* TODO: Ensure this doesn't happen if we had an error getting attrs in + * backing. + */ + err = finalize_attr(inode, outarg, attr_version, stat); + return ERR_PTR(err); +} + +static void fattr_to_iattr(struct fuse_conn *fc, + const struct fuse_setattr_in *arg, + struct iattr *iattr) +{ + unsigned int fvalid = arg->valid; + + if (fvalid & FATTR_MODE) + iattr->ia_valid |= ATTR_MODE, iattr->ia_mode = arg->mode; + if (fvalid & FATTR_UID) { + iattr->ia_valid |= ATTR_UID; + iattr->ia_uid = make_kuid(fc->user_ns, arg->uid); + } + if (fvalid & FATTR_GID) { + iattr->ia_valid |= ATTR_GID; + iattr->ia_gid = make_kgid(fc->user_ns, arg->gid); + } + if (fvalid & FATTR_SIZE) + iattr->ia_valid |= ATTR_SIZE, iattr->ia_size = arg->size; + if (fvalid & FATTR_ATIME) { + iattr->ia_valid |= ATTR_ATIME; + iattr->ia_atime.tv_sec = arg->atime; + iattr->ia_atime.tv_nsec = arg->atimensec; + if (!(fvalid & FATTR_ATIME_NOW)) + iattr->ia_valid |= ATTR_ATIME_SET; + } + if (fvalid & FATTR_MTIME) { + iattr->ia_valid |= ATTR_MTIME; + iattr->ia_mtime.tv_sec = arg->mtime; + iattr->ia_mtime.tv_nsec = arg->mtimensec; + if (!(fvalid & FATTR_MTIME_NOW)) + iattr->ia_valid |= ATTR_MTIME_SET; + } + if (fvalid & FATTR_CTIME) { + iattr->ia_valid |= ATTR_CTIME; + iattr->ia_ctime.tv_sec = arg->ctime; + iattr->ia_ctime.tv_nsec = arg->ctimensec; + } +} + +int fuse_setattr_initialize(struct fuse_bpf_args *fa, struct fuse_setattr_io *fsio, + struct dentry *dentry, struct iattr *attr, struct file *file) +{ + struct fuse_conn *fc = get_fuse_conn(dentry->d_inode); + + *fsio = (struct fuse_setattr_io) {0}; + iattr_to_fattr(fc, attr, &fsio->fsi, true); + + *fa = (struct fuse_bpf_args) { + .opcode = FUSE_SETATTR, + .nodeid = get_node_id(dentry->d_inode), + .in_numargs = 1, + .in_args[0].size = sizeof(fsio->fsi), + .in_args[0].value = &fsio->fsi, + .out_numargs = 1, + .out_args[0].size = sizeof(fsio->fao), + .out_args[0].value = &fsio->fao, + }; + + return 0; +} + +int fuse_setattr_backing(struct fuse_bpf_args *fa, + struct dentry *dentry, struct iattr *attr, struct file *file) +{ + struct fuse_conn *fc = get_fuse_conn(dentry->d_inode); + const struct fuse_setattr_in *fsi = fa->in_args[0].value; + struct iattr new_attr = {0}; + struct path *backing_path = &get_fuse_dentry(dentry)->backing_path; + int res; + + fattr_to_iattr(fc, fsi, &new_attr); + /* TODO: Some info doesn't get saved by the attr->fattr->attr transition + * When we actually allow the bpf to change these, we may have to consider + * the extra flags more, or pass more info into the bpf. Until then we can + * keep everything except for ATTR_FILE, since we'd need a file on the + * lower fs. For what it's worth, neither f2fs nor ext4 make use of that + * even if it is present. + */ + new_attr.ia_valid = attr->ia_valid & ~ATTR_FILE; + inode_lock(d_inode(backing_path->dentry)); + res = notify_change(backing_path->dentry, &new_attr, NULL); + inode_unlock(d_inode(backing_path->dentry)); + + if (res == 0 && (new_attr.ia_valid & ATTR_SIZE)) + i_size_write(dentry->d_inode, new_attr.ia_size); + return res; +} + +void *fuse_setattr_finalize(struct fuse_bpf_args *fa, + struct dentry *dentry, struct iattr *attr, struct file *file) +{ + return NULL; +} + +int fuse_statfs_initialize( + struct fuse_bpf_args *fa, struct fuse_statfs_out *fso, + struct dentry *dentry, struct kstatfs *buf) +{ + *fso = (struct fuse_statfs_out) {0}; + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(d_inode(dentry)), + .opcode = FUSE_STATFS, + .out_numargs = 1, + .out_numargs = 1, + .out_args[0].size = sizeof(fso), + .out_args[0].value = fso, + }; + + return 0; +} + +int fuse_statfs_backing( + struct fuse_bpf_args *fa, + struct dentry *dentry, struct kstatfs *buf) +{ + int err = 0; + struct path backing_path; + struct fuse_statfs_out *fso = fa->out_args[0].value; + + get_fuse_backing_path(dentry, &backing_path); + if (!backing_path.dentry) + return -EBADF; + err = vfs_statfs(&backing_path, buf); + path_put(&backing_path); + buf->f_type = FUSE_SUPER_MAGIC; + + //TODO Provide postfilter opportunity to modify + if (!err) + convert_statfs_to_fuse(&fso->st, buf); + + return err; +} + +void *fuse_statfs_finalize( + struct fuse_bpf_args *fa, + struct dentry *dentry, struct kstatfs *buf) +{ + struct fuse_statfs_out *fso = fa->out_args[0].value; + + if (!fa->error_in) + convert_fuse_statfs(buf, &fso->st); + return NULL; +} + +int fuse_get_link_initialize(struct fuse_bpf_args *fa, struct fuse_dummy_io *unused, + struct inode *inode, struct dentry *dentry, + struct delayed_call *callback, const char **out) +{ + /* + * TODO + * If we want to handle changing these things, we'll need to copy + * the lower fs's data into our own buffer, and provide our own callback + * to free that buffer. + * + * Pre could change the name we're looking at + * postfilter can change the name we return + * + * We ought to only make that buffer if it's been requested, so leaving + * this unimplemented for the moment + */ + *fa = (struct fuse_bpf_args) { + .opcode = FUSE_READLINK, + .nodeid = get_node_id(inode), + .in_numargs = 1, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = dentry->d_name.len + 1, + .value = dentry->d_name.name, + }, + /* + * .out_argvar = 1, + * .out_numargs = 1, + * .out_args[0].size = , + * .out_args[0].value = , + */ + }; + + return 0; +} + +int fuse_get_link_backing(struct fuse_bpf_args *fa, + struct inode *inode, struct dentry *dentry, + struct delayed_call *callback, const char **out) +{ + struct path backing_path; + + if (!dentry) { + *out = ERR_PTR(-ECHILD); + return PTR_ERR(*out); + } + + get_fuse_backing_path(dentry, &backing_path); + if (!backing_path.dentry) { + *out = ERR_PTR(-ECHILD); + return PTR_ERR(*out); + } + + /* + * TODO: If we want to do our own thing, copy the data and then call the + * callback + */ + *out = vfs_get_link(backing_path.dentry, callback); + + path_put(&backing_path); + return 0; +} + +void *fuse_get_link_finalize(struct fuse_bpf_args *fa, + struct inode *inode, struct dentry *dentry, + struct delayed_call *callback, const char **out) +{ + return NULL; +} + +int fuse_symlink_initialize( + struct fuse_bpf_args *fa, struct fuse_dummy_io *unused, + struct inode *dir, struct dentry *entry, const char *link, int len) +{ + *fa = (struct fuse_bpf_args) { + .nodeid = get_node_id(dir), + .opcode = FUSE_SYMLINK, + .in_numargs = 2, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = entry->d_name.len + 1, + .value = entry->d_name.name, + }, + .in_args[1] = (struct fuse_bpf_in_arg) { + .size = len, + .value = link, + }, + }; + + return 0; +} + +int fuse_symlink_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, const char *link, int len) +{ + int err = 0; + struct fuse_inode *fuse_inode = get_fuse_inode(dir); + struct inode *backing_inode = fuse_inode->backing_inode; + struct path backing_path = {}; + struct inode *inode = NULL; + + //TODO Actually deal with changing the backing entry in symlink + get_fuse_backing_path(entry, &backing_path); + if (!backing_path.dentry) + return -EBADF; + + inode_lock_nested(backing_inode, I_MUTEX_PARENT); + err = vfs_symlink(backing_inode, backing_path.dentry, link); + inode_unlock(backing_inode); + if (err) + goto out; + if (d_really_is_negative(backing_path.dentry) || + unlikely(d_unhashed(backing_path.dentry))) { + err = -EINVAL; + /** + * TODO: overlayfs responds to this situation with a + * lookupOneLen. Should we do that too? + */ + goto out; + } + inode = fuse_iget_backing(dir->i_sb, fuse_inode->nodeid, backing_inode); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out; + } + d_instantiate(entry, inode); +out: + path_put(&backing_path); + return err; +} + +void *fuse_symlink_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, const char *link, int len) +{ + return NULL; +} + +int fuse_readdir_initialize(struct fuse_bpf_args *fa, struct fuse_read_io *frio, + struct file *file, struct dir_context *ctx, + bool *force_again, bool *allow_force, bool is_continued) +{ + struct fuse_file *ff = file->private_data; + u8 *page = (u8 *)__get_free_page(GFP_KERNEL); + + if (!page) + return -ENOMEM; + + *fa = (struct fuse_bpf_args) { + .nodeid = ff->nodeid, + .opcode = FUSE_READDIR, + .in_numargs = 1, + .flags = FUSE_BPF_OUT_ARGVAR, + .out_numargs = 2, + .in_args[0] = (struct fuse_bpf_in_arg) { + .size = sizeof(frio->fri), + .value = &frio->fri, + }, + .out_args[0] = (struct fuse_bpf_arg) { + .size = sizeof(frio->fro), + .value = &frio->fro, + }, + .out_args[1] = (struct fuse_bpf_arg) { + .size = PAGE_SIZE, + .value = page, + }, + }; + + frio->fri = (struct fuse_read_in) { + .fh = ff->fh, + .offset = ctx->pos, + .size = PAGE_SIZE, + }; + frio->fro = (struct fuse_read_out) { + .again = 0, + .offset = 0, + }; + *force_again = false; + *allow_force = true; + return 0; +} + +struct extfuse_ctx { + struct dir_context ctx; + u8 *addr; + size_t offset; +}; + +static int filldir(struct dir_context *ctx, const char *name, int namelen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct extfuse_ctx *ec = container_of(ctx, struct extfuse_ctx, ctx); + struct fuse_dirent *fd = (struct fuse_dirent *) (ec->addr + ec->offset); + + if (ec->offset + sizeof(struct fuse_dirent) + namelen > PAGE_SIZE) + return -ENOMEM; + + *fd = (struct fuse_dirent) { + .ino = ino, + .off = offset, + .namelen = namelen, + .type = d_type, + }; + + memcpy(fd->name, name, namelen); + ec->offset += FUSE_DIRENT_SIZE(fd); + + return 0; +} + +static int parse_dirfile(char *buf, size_t nbytes, struct dir_context *ctx) +{ + while (nbytes >= FUSE_NAME_OFFSET) { + struct fuse_dirent *dirent = (struct fuse_dirent *) buf; + size_t reclen = FUSE_DIRENT_SIZE(dirent); + + if (!dirent->namelen || dirent->namelen > FUSE_NAME_MAX) + return -EIO; + if (reclen > nbytes) + break; + if (memchr(dirent->name, '/', dirent->namelen) != NULL) + return -EIO; + + ctx->pos = dirent->off; + if (!dir_emit(ctx, dirent->name, dirent->namelen, dirent->ino, + dirent->type)) + break; + + buf += reclen; + nbytes -= reclen; + } + + return 0; +} + + +int fuse_readdir_backing(struct fuse_bpf_args *fa, + struct file *file, struct dir_context *ctx, + bool *force_again, bool *allow_force, bool is_continued) +{ + struct fuse_file *ff = file->private_data; + struct file *backing_dir = ff->backing_file; + struct fuse_read_out *fro = fa->out_args[0].value; + struct extfuse_ctx ec; + int err; + + ec = (struct extfuse_ctx) { + .ctx.actor = filldir, + .ctx.pos = ctx->pos, + .addr = fa->out_args[1].value, + }; + + if (!ec.addr) + return -ENOMEM; + + if (!is_continued) + backing_dir->f_pos = file->f_pos; + + err = iterate_dir(backing_dir, &ec.ctx); + if (ec.offset == 0) + *allow_force = false; + fa->out_args[1].size = ec.offset; + + fro->offset = ec.ctx.pos; + fro->again = false; + return err; +} + +void *fuse_readdir_finalize(struct fuse_bpf_args *fa, + struct file *file, struct dir_context *ctx, + bool *force_again, bool *allow_force, bool is_continued) +{ + struct fuse_read_out *fro = fa->out_args[0].value; + struct fuse_file *ff = file->private_data; + struct file *backing_dir = ff->backing_file; + int err = 0; + + err = parse_dirfile(fa->out_args[1].value, fa->out_args[1].size, ctx); + *force_again = !!fro->again; + if (*force_again && !*allow_force) + err = -EINVAL; + + ctx->pos = fro->offset; + backing_dir->f_pos = fro->offset; + + free_page((unsigned long) fa->out_args[1].value); + return ERR_PTR(err); +} + +int fuse_access_initialize(struct fuse_bpf_args *fa, struct fuse_access_in *fai, + struct inode *inode, int mask) +{ + *fai = (struct fuse_access_in) { + .mask = mask, + }; + + *fa = (struct fuse_bpf_args) { + .opcode = FUSE_ACCESS, + .nodeid = get_node_id(inode), + .in_numargs = 1, + .in_args[0].size = sizeof(*fai), + .in_args[0].value = fai, + }; + + return 0; +} + +int fuse_access_backing(struct fuse_bpf_args *fa, struct inode *inode, int mask) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + const struct fuse_access_in *fai = fa->in_args[0].value; + + return inode_permission(/* For mainline: init_user_ns,*/ + fi->backing_inode, fai->mask); +} + +void *fuse_access_finalize(struct fuse_bpf_args *fa, struct inode *inode, int mask) +{ + return NULL; +} + +int __init fuse_bpf_init(void) +{ + fuse_bpf_aio_request_cachep = kmem_cache_create("fuse_bpf_aio_req", + sizeof(struct fuse_bpf_aio_req), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!fuse_bpf_aio_request_cachep) + return -ENOMEM; + + return 0; +} + +void __exit fuse_bpf_cleanup(void) +{ + kmem_cache_destroy(fuse_bpf_aio_request_cachep); +} + +ssize_t fuse_bpf_simple_request(struct fuse_mount *fm, struct fuse_bpf_args *bpf_args) +{ + int i; + ssize_t res; + struct fuse_args args = { + .nodeid = bpf_args->nodeid, + .opcode = bpf_args->opcode, + .error_in = bpf_args->error_in, + .in_numargs = bpf_args->in_numargs, + .out_numargs = bpf_args->out_numargs, + .force = !!(bpf_args->flags & FUSE_BPF_FORCE), + .out_argvar = !!(bpf_args->flags & FUSE_BPF_OUT_ARGVAR), + }; + + for (i = 0; i < args.in_numargs; ++i) + args.in_args[i] = (struct fuse_in_arg) { + .size = bpf_args->in_args[i].size, + .value = bpf_args->in_args[i].value, + }; + for (i = 0; i < args.out_numargs; ++i) + args.out_args[i] = (struct fuse_arg) { + .size = bpf_args->out_args[i].size, + .value = bpf_args->out_args[i].value, + }; + + res = fuse_simple_request(fm, &args); + + *bpf_args = (struct fuse_bpf_args) { + .nodeid = args.nodeid, + .opcode = args.opcode, + .error_in = args.error_in, + .in_numargs = args.in_numargs, + .out_numargs = args.out_numargs, + }; + if (args.force) + bpf_args->flags |= FUSE_BPF_FORCE; + if (args.out_argvar) + bpf_args->flags |= FUSE_BPF_OUT_ARGVAR; + for (i = 0; i < args.in_numargs; ++i) + bpf_args->in_args[i] = (struct fuse_bpf_in_arg) { + .size = args.in_args[i].size, + .value = args.in_args[i].value, + }; + for (i = 0; i < args.out_numargs; ++i) + bpf_args->out_args[i] = (struct fuse_bpf_arg) { + .size = args.out_args[i].size, + .value = args.out_args[i].value, + }; + return res; +} diff --git a/fs/fuse/control.c b/fs/fuse/control.c index 7092958c43ed..4aae0e606ba4 100644 --- a/fs/fuse/control.c +++ b/fs/fuse/control.c @@ -164,6 +164,7 @@ static ssize_t fuse_conn_congestion_threshold_write(struct file *file, { unsigned val; struct fuse_conn *fc; + struct fuse_mount *fm; ssize_t ret; ret = fuse_conn_limit_write(file, buf, count, ppos, &val, @@ -174,18 +175,27 @@ static ssize_t fuse_conn_congestion_threshold_write(struct file *file, if (!fc) goto out; + down_read(&fc->killsb); spin_lock(&fc->bg_lock); fc->congestion_threshold = val; - if (fc->sb) { + + /* + * Get any fuse_mount belonging to this fuse_conn; s_bdi is + * shared between all of them + */ + + if (!list_empty(&fc->mounts)) { + fm = list_first_entry(&fc->mounts, struct fuse_mount, fc_entry); if (fc->num_background < fc->congestion_threshold) { - clear_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC); - clear_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC); + clear_bdi_congested(fm->sb->s_bdi, BLK_RW_SYNC); + clear_bdi_congested(fm->sb->s_bdi, BLK_RW_ASYNC); } else { - set_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC); - set_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC); + set_bdi_congested(fm->sb->s_bdi, BLK_RW_SYNC); + set_bdi_congested(fm->sb->s_bdi, BLK_RW_ASYNC); } } spin_unlock(&fc->bg_lock); + up_read(&fc->killsb); fuse_conn_put(fc); out: return ret; @@ -318,7 +328,7 @@ void fuse_ctl_remove_conn(struct fuse_conn *fc) drop_nlink(d_inode(fuse_control_sb->s_root)); } -static int fuse_ctl_fill_super(struct super_block *sb, struct fs_context *fctx) +static int fuse_ctl_fill_super(struct super_block *sb, struct fs_context *fsc) { static const struct tree_descr empty_descr = {""}; struct fuse_conn *fc; @@ -344,18 +354,18 @@ static int fuse_ctl_fill_super(struct super_block *sb, struct fs_context *fctx) return 0; } -static int fuse_ctl_get_tree(struct fs_context *fc) +static int fuse_ctl_get_tree(struct fs_context *fsc) { - return get_tree_single(fc, fuse_ctl_fill_super); + return get_tree_single(fsc, fuse_ctl_fill_super); } static const struct fs_context_operations fuse_ctl_context_ops = { .get_tree = fuse_ctl_get_tree, }; -static int fuse_ctl_init_fs_context(struct fs_context *fc) +static int fuse_ctl_init_fs_context(struct fs_context *fsc) { - fc->ops = &fuse_ctl_context_ops; + fsc->ops = &fuse_ctl_context_ops; return 0; } @@ -385,7 +395,7 @@ int __init fuse_ctl_init(void) return register_filesystem(&fuse_ctl_fs_type); } -void __exit fuse_ctl_cleanup(void) +void fuse_ctl_cleanup(void) { unregister_filesystem(&fuse_ctl_fs_type); } diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 072b68fbf8f4..a37528b51798 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -57,6 +57,7 @@ struct cuse_conn { struct list_head list; /* linked on cuse_conntbl */ + struct fuse_mount fm; /* Dummy mount referencing fc */ struct fuse_conn fc; /* fuse connection */ struct cdev *cdev; /* associated character device */ struct device *dev; /* device representing @cdev */ @@ -134,7 +135,7 @@ static int cuse_open(struct inode *inode, struct file *file) * Generic permission check is already done against the chrdev * file, proceed to open. */ - rc = fuse_do_open(&cc->fc, 0, file, 0); + rc = fuse_do_open(&cc->fm, 0, file, 0); if (rc) fuse_conn_put(&cc->fc); return rc; @@ -143,10 +144,10 @@ static int cuse_open(struct inode *inode, struct file *file) static int cuse_release(struct inode *inode, struct file *file) { struct fuse_file *ff = file->private_data; - struct fuse_conn *fc = ff->fc; + struct fuse_mount *fm = ff->fm; fuse_sync_release(NULL, ff, file->f_flags); - fuse_conn_put(fc); + fuse_conn_put(fm->fc); return 0; } @@ -155,7 +156,7 @@ static long cuse_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct fuse_file *ff = file->private_data; - struct cuse_conn *cc = fc_to_cc(ff->fc); + struct cuse_conn *cc = fc_to_cc(ff->fm->fc); unsigned int flags = 0; if (cc->unrestricted_ioctl) @@ -168,7 +169,7 @@ static long cuse_file_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct fuse_file *ff = file->private_data; - struct cuse_conn *cc = fc_to_cc(ff->fc); + struct cuse_conn *cc = fc_to_cc(ff->fm->fc); unsigned int flags = FUSE_IOCTL_COMPAT; if (cc->unrestricted_ioctl) @@ -313,9 +314,10 @@ struct cuse_init_args { * required data structures for it. Please read the comment at the * top of this file for high level overview. */ -static void cuse_process_init_reply(struct fuse_conn *fc, +static void cuse_process_init_reply(struct fuse_mount *fm, struct fuse_args *args, int error) { + struct fuse_conn *fc = fm->fc; struct cuse_init_args *ia = container_of(args, typeof(*ia), ap.args); struct fuse_args_pages *ap = &ia->ap; struct cuse_conn *cc = fc_to_cc(fc), *pos; @@ -424,7 +426,7 @@ static int cuse_send_init(struct cuse_conn *cc) { int rc; struct page *page; - struct fuse_conn *fc = &cc->fc; + struct fuse_mount *fm = &cc->fm; struct cuse_init_args *ia; struct fuse_args_pages *ap; @@ -451,8 +453,8 @@ static int cuse_send_init(struct cuse_conn *cc) ap->args.out_args[0].size = sizeof(ia->out); ap->args.out_args[0].value = &ia->out; ap->args.out_args[1].size = CUSE_INIT_INFO_MAX; - ap->args.out_argvar = 1; - ap->args.out_pages = 1; + ap->args.out_argvar = true; + ap->args.out_pages = true; ap->num_pages = 1; ap->pages = &ia->page; ap->descs = &ia->desc; @@ -460,7 +462,7 @@ static int cuse_send_init(struct cuse_conn *cc) ia->desc.length = ap->args.out_args[1].size; ap->args.end = cuse_process_init_reply; - rc = fuse_simple_background(fc, &ap->args, GFP_KERNEL); + rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL); if (rc) { kfree(ia); err_free_page: @@ -506,7 +508,8 @@ static int cuse_channel_open(struct inode *inode, struct file *file) * Limit the cuse channel to requests that can * be represented in file->f_cred->user_ns. */ - fuse_conn_init(&cc->fc, file->f_cred->user_ns, &fuse_dev_fiq_ops, NULL); + fuse_conn_init(&cc->fc, &cc->fm, file->f_cred->user_ns, + &fuse_dev_fiq_ops, NULL); fud = fuse_dev_alloc_install(&cc->fc); if (!fud) { diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c new file mode 100644 index 000000000000..2739e218df64 --- /dev/null +++ b/fs/fuse/dax.c @@ -0,0 +1,1366 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dax: direct host memory access + * Copyright (C) 2020 Red Hat, Inc. + */ + +#include "fuse_i.h" + +#include +#include +#include +#include +#include +#include + +/* + * Default memory range size. A power of 2 so it agrees with common FUSE_INIT + * map_alignment values 4KB and 64KB. + */ +#define FUSE_DAX_SHIFT 21 +#define FUSE_DAX_SZ (1 << FUSE_DAX_SHIFT) +#define FUSE_DAX_PAGES (FUSE_DAX_SZ / PAGE_SIZE) + +/* Number of ranges reclaimer will try to free in one invocation */ +#define FUSE_DAX_RECLAIM_CHUNK (10) + +/* + * Dax memory reclaim threshold in percetage of total ranges. When free + * number of free ranges drops below this threshold, reclaim can trigger + * Default is 20% + */ +#define FUSE_DAX_RECLAIM_THRESHOLD (20) + +/** Translation information for file offsets to DAX window offsets */ +struct fuse_dax_mapping { + /* Pointer to inode where this memory range is mapped */ + struct inode *inode; + + /* Will connect in fcd->free_ranges to keep track of free memory */ + struct list_head list; + + /* For interval tree in file/inode */ + struct interval_tree_node itn; + + /* Will connect in fc->busy_ranges to keep track busy memory */ + struct list_head busy_list; + + /** Position in DAX window */ + u64 window_offset; + + /** Length of mapping, in bytes */ + loff_t length; + + /* Is this mapping read-only or read-write */ + bool writable; + + /* reference count when the mapping is used by dax iomap. */ + refcount_t refcnt; +}; + +/* Per-inode dax map */ +struct fuse_inode_dax { + /* Semaphore to protect modifications to the dmap tree */ + struct rw_semaphore sem; + + /* Sorted rb tree of struct fuse_dax_mapping elements */ + struct rb_root_cached tree; + unsigned long nr; +}; + +struct fuse_conn_dax { + /* DAX device */ + struct dax_device *dev; + + /* Lock protecting accessess to members of this structure */ + spinlock_t lock; + + /* List of memory ranges which are busy */ + unsigned long nr_busy_ranges; + struct list_head busy_ranges; + + /* Worker to free up memory ranges */ + struct delayed_work free_work; + + /* Wait queue for a dax range to become free */ + wait_queue_head_t range_waitq; + + /* DAX Window Free Ranges */ + long nr_free_ranges; + struct list_head free_ranges; + + unsigned long nr_ranges; +}; + +static inline struct fuse_dax_mapping * +node_to_dmap(struct interval_tree_node *node) +{ + if (!node) + return NULL; + + return container_of(node, struct fuse_dax_mapping, itn); +} + +static struct fuse_dax_mapping * +alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode); + +static void +__kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms) +{ + unsigned long free_threshold; + + /* If number of free ranges are below threshold, start reclaim */ + free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100, + 1); + if (fcd->nr_free_ranges < free_threshold) + queue_delayed_work(system_long_wq, &fcd->free_work, + msecs_to_jiffies(delay_ms)); +} + +static void kick_dmap_free_worker(struct fuse_conn_dax *fcd, + unsigned long delay_ms) +{ + spin_lock(&fcd->lock); + __kick_dmap_free_worker(fcd, delay_ms); + spin_unlock(&fcd->lock); +} + +static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd) +{ + struct fuse_dax_mapping *dmap; + + spin_lock(&fcd->lock); + dmap = list_first_entry_or_null(&fcd->free_ranges, + struct fuse_dax_mapping, list); + if (dmap) { + list_del_init(&dmap->list); + WARN_ON(fcd->nr_free_ranges <= 0); + fcd->nr_free_ranges--; + } + spin_unlock(&fcd->lock); + + kick_dmap_free_worker(fcd, 0); + return dmap; +} + +/* This assumes fcd->lock is held */ +static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + list_del_init(&dmap->busy_list); + WARN_ON(fcd->nr_busy_ranges == 0); + fcd->nr_busy_ranges--; +} + +static void dmap_remove_busy_list(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + spin_lock(&fcd->lock); + __dmap_remove_busy_list(fcd, dmap); + spin_unlock(&fcd->lock); +} + +/* This assumes fcd->lock is held */ +static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + list_add_tail(&dmap->list, &fcd->free_ranges); + fcd->nr_free_ranges++; + wake_up(&fcd->range_waitq); +} + +static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + /* Return fuse_dax_mapping to free list */ + spin_lock(&fcd->lock); + __dmap_add_to_free_pool(fcd, dmap); + spin_unlock(&fcd->lock); +} + +static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx, + struct fuse_dax_mapping *dmap, bool writable, + bool upgrade) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_conn_dax *fcd = fm->fc->dax; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_setupmapping_in inarg; + loff_t offset = start_idx << FUSE_DAX_SHIFT; + FUSE_ARGS(args); + ssize_t err; + + WARN_ON(fcd->nr_free_ranges < 0); + + /* Ask fuse daemon to setup mapping */ + memset(&inarg, 0, sizeof(inarg)); + inarg.foffset = offset; + inarg.fh = -1; + inarg.moffset = dmap->window_offset; + inarg.len = FUSE_DAX_SZ; + inarg.flags |= FUSE_SETUPMAPPING_FLAG_READ; + if (writable) + inarg.flags |= FUSE_SETUPMAPPING_FLAG_WRITE; + args.opcode = FUSE_SETUPMAPPING; + args.nodeid = fi->nodeid; + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + err = fuse_simple_request(fm, &args); + if (err < 0) + return err; + dmap->writable = writable; + if (!upgrade) { + /* + * We don't take a refernce on inode. inode is valid right now + * and when inode is going away, cleanup logic should first + * cleanup dmap entries. + */ + dmap->inode = inode; + dmap->itn.start = dmap->itn.last = start_idx; + /* Protected by fi->dax->sem */ + interval_tree_insert(&dmap->itn, &fi->dax->tree); + fi->dax->nr++; + spin_lock(&fcd->lock); + list_add_tail(&dmap->busy_list, &fcd->busy_ranges); + fcd->nr_busy_ranges++; + spin_unlock(&fcd->lock); + } + return 0; +} + +static int fuse_send_removemapping(struct inode *inode, + struct fuse_removemapping_in *inargp, + struct fuse_removemapping_one *remove_one) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + + args.opcode = FUSE_REMOVEMAPPING; + args.nodeid = fi->nodeid; + args.in_numargs = 2; + args.in_args[0].size = sizeof(*inargp); + args.in_args[0].value = inargp; + args.in_args[1].size = inargp->count * sizeof(*remove_one); + args.in_args[1].value = remove_one; + return fuse_simple_request(fm, &args); +} + +static int dmap_removemapping_list(struct inode *inode, unsigned int num, + struct list_head *to_remove) +{ + struct fuse_removemapping_one *remove_one, *ptr; + struct fuse_removemapping_in inarg; + struct fuse_dax_mapping *dmap; + int ret, i = 0, nr_alloc; + + nr_alloc = min_t(unsigned int, num, FUSE_REMOVEMAPPING_MAX_ENTRY); + remove_one = kmalloc_array(nr_alloc, sizeof(*remove_one), GFP_NOFS); + if (!remove_one) + return -ENOMEM; + + ptr = remove_one; + list_for_each_entry(dmap, to_remove, list) { + ptr->moffset = dmap->window_offset; + ptr->len = dmap->length; + ptr++; + i++; + num--; + if (i >= nr_alloc || num == 0) { + memset(&inarg, 0, sizeof(inarg)); + inarg.count = i; + ret = fuse_send_removemapping(inode, &inarg, + remove_one); + if (ret) + goto out; + ptr = remove_one; + i = 0; + } + } +out: + kfree(remove_one); + return ret; +} + +/* + * Cleanup dmap entry and add back to free list. This should be called with + * fcd->lock held. + */ +static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + pr_debug("fuse: freeing memory range start_idx=0x%lx end_idx=0x%lx window_offset=0x%llx length=0x%llx\n", + dmap->itn.start, dmap->itn.last, dmap->window_offset, + dmap->length); + __dmap_remove_busy_list(fcd, dmap); + dmap->inode = NULL; + dmap->itn.start = dmap->itn.last = 0; + __dmap_add_to_free_pool(fcd, dmap); +} + +/* + * Free inode dmap entries whose range falls inside [start, end]. + * Does not take any locks. At this point of time it should only be + * called from evict_inode() path where we know all dmap entries can be + * reclaimed. + */ +static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd, + struct inode *inode, + loff_t start, loff_t end) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap, *n; + int err, num = 0; + LIST_HEAD(to_remove); + unsigned long start_idx = start >> FUSE_DAX_SHIFT; + unsigned long end_idx = end >> FUSE_DAX_SHIFT; + struct interval_tree_node *node; + + while (1) { + node = interval_tree_iter_first(&fi->dax->tree, start_idx, + end_idx); + if (!node) + break; + dmap = node_to_dmap(node); + /* inode is going away. There should not be any users of dmap */ + WARN_ON(refcount_read(&dmap->refcnt) > 1); + interval_tree_remove(&dmap->itn, &fi->dax->tree); + num++; + list_add(&dmap->list, &to_remove); + } + + /* Nothing to remove */ + if (list_empty(&to_remove)) + return; + + WARN_ON(fi->dax->nr < num); + fi->dax->nr -= num; + err = dmap_removemapping_list(inode, num, &to_remove); + if (err && err != -ENOTCONN) { + pr_warn("Failed to removemappings. start=0x%llx end=0x%llx\n", + start, end); + } + spin_lock(&fcd->lock); + list_for_each_entry_safe(dmap, n, &to_remove, list) { + list_del_init(&dmap->list); + dmap_reinit_add_to_free_pool(fcd, dmap); + } + spin_unlock(&fcd->lock); +} + +static int dmap_removemapping_one(struct inode *inode, + struct fuse_dax_mapping *dmap) +{ + struct fuse_removemapping_one forget_one; + struct fuse_removemapping_in inarg; + + memset(&inarg, 0, sizeof(inarg)); + inarg.count = 1; + memset(&forget_one, 0, sizeof(forget_one)); + forget_one.moffset = dmap->window_offset; + forget_one.len = dmap->length; + + return fuse_send_removemapping(inode, &inarg, &forget_one); +} + +/* + * It is called from evict_inode() and by that time inode is going away. So + * this function does not take any locks like fi->dax->sem for traversing + * that fuse inode interval tree. If that lock is taken then lock validator + * complains of deadlock situation w.r.t fs_reclaim lock. + */ +void fuse_dax_inode_cleanup(struct inode *inode) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + + /* + * fuse_evict_inode() has already called truncate_inode_pages_final() + * before we arrive here. So we should not have to worry about any + * pages/exception entries still associated with inode. + */ + inode_reclaim_dmap_range(fc->dax, inode, 0, -1); + WARN_ON(fi->dax->nr); +} + +static void fuse_fill_iomap_hole(struct iomap *iomap, loff_t length) +{ + iomap->addr = IOMAP_NULL_ADDR; + iomap->length = length; + iomap->type = IOMAP_HOLE; +} + +static void fuse_fill_iomap(struct inode *inode, loff_t pos, loff_t length, + struct iomap *iomap, struct fuse_dax_mapping *dmap, + unsigned int flags) +{ + loff_t offset, len; + loff_t i_size = i_size_read(inode); + + offset = pos - (dmap->itn.start << FUSE_DAX_SHIFT); + len = min(length, dmap->length - offset); + + /* If length is beyond end of file, truncate further */ + if (pos + len > i_size) + len = i_size - pos; + + if (len > 0) { + iomap->addr = dmap->window_offset + offset; + iomap->length = len; + if (flags & IOMAP_FAULT) + iomap->length = ALIGN(len, PAGE_SIZE); + iomap->type = IOMAP_MAPPED; + /* + * increace refcnt so that reclaim code knows this dmap is in + * use. This assumes fi->dax->sem mutex is held either + * shared/exclusive. + */ + refcount_inc(&dmap->refcnt); + + /* iomap->private should be NULL */ + WARN_ON_ONCE(iomap->private); + iomap->private = dmap; + } else { + /* Mapping beyond end of file is hole */ + fuse_fill_iomap_hole(iomap, length); + } +} + +static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos, + loff_t length, unsigned int flags, + struct iomap *iomap) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_conn_dax *fcd = fc->dax; + struct fuse_dax_mapping *dmap, *alloc_dmap = NULL; + int ret; + bool writable = flags & IOMAP_WRITE; + unsigned long start_idx = pos >> FUSE_DAX_SHIFT; + struct interval_tree_node *node; + + /* + * Can't do inline reclaim in fault path. We call + * dax_layout_busy_page() before we free a range. And + * fuse_wait_dax_page() drops fi->i_mmap_sem lock and requires it. + * In fault path we enter with fi->i_mmap_sem held and can't drop + * it. Also in fault path we hold fi->i_mmap_sem shared and not + * exclusive, so that creates further issues with fuse_wait_dax_page(). + * Hence return -EAGAIN and fuse_dax_fault() will wait for a memory + * range to become free and retry. + */ + if (flags & IOMAP_FAULT) { + alloc_dmap = alloc_dax_mapping(fcd); + if (!alloc_dmap) + return -EAGAIN; + } else { + alloc_dmap = alloc_dax_mapping_reclaim(fcd, inode); + if (IS_ERR(alloc_dmap)) + return PTR_ERR(alloc_dmap); + } + + /* If we are here, we should have memory allocated */ + if (WARN_ON(!alloc_dmap)) + return -EIO; + + /* + * Take write lock so that only one caller can try to setup mapping + * and other waits. + */ + down_write(&fi->dax->sem); + /* + * We dropped lock. Check again if somebody else setup + * mapping already. + */ + node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); + if (node) { + dmap = node_to_dmap(node); + fuse_fill_iomap(inode, pos, length, iomap, dmap, flags); + dmap_add_to_free_pool(fcd, alloc_dmap); + up_write(&fi->dax->sem); + return 0; + } + + /* Setup one mapping */ + ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, alloc_dmap, + writable, false); + if (ret < 0) { + dmap_add_to_free_pool(fcd, alloc_dmap); + up_write(&fi->dax->sem); + return ret; + } + fuse_fill_iomap(inode, pos, length, iomap, alloc_dmap, flags); + up_write(&fi->dax->sem); + return 0; +} + +static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos, + loff_t length, unsigned int flags, + struct iomap *iomap) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap; + int ret; + unsigned long idx = pos >> FUSE_DAX_SHIFT; + struct interval_tree_node *node; + + /* + * Take exclusive lock so that only one caller can try to setup + * mapping and others wait. + */ + down_write(&fi->dax->sem); + node = interval_tree_iter_first(&fi->dax->tree, idx, idx); + + /* We are holding either inode lock or i_mmap_sem, and that should + * ensure that dmap can't be truncated. We are holding a reference + * on dmap and that should make sure it can't be reclaimed. So dmap + * should still be there in tree despite the fact we dropped and + * re-acquired the fi->dax->sem lock. + */ + ret = -EIO; + if (WARN_ON(!node)) + goto out_err; + + dmap = node_to_dmap(node); + + /* We took an extra reference on dmap to make sure its not reclaimd. + * Now we hold fi->dax->sem lock and that reference is not needed + * anymore. Drop it. + */ + if (refcount_dec_and_test(&dmap->refcnt)) { + /* refcount should not hit 0. This object only goes + * away when fuse connection goes away + */ + WARN_ON_ONCE(1); + } + + /* Maybe another thread already upgraded mapping while we were not + * holding lock. + */ + if (dmap->writable) { + ret = 0; + goto out_fill_iomap; + } + + ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, dmap, true, + true); + if (ret < 0) + goto out_err; +out_fill_iomap: + fuse_fill_iomap(inode, pos, length, iomap, dmap, flags); +out_err: + up_write(&fi->dax->sem); + return ret; +} + +/* This is just for DAX and the mapping is ephemeral, do not use it for other + * purposes since there is no block device with a permanent mapping. + */ +static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length, + unsigned int flags, struct iomap *iomap, + struct iomap *srcmap) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_dax_mapping *dmap; + bool writable = flags & IOMAP_WRITE; + unsigned long start_idx = pos >> FUSE_DAX_SHIFT; + struct interval_tree_node *node; + + /* We don't support FIEMAP */ + if (WARN_ON(flags & IOMAP_REPORT)) + return -EIO; + + iomap->offset = pos; + iomap->flags = 0; + iomap->bdev = NULL; + iomap->dax_dev = fc->dax->dev; + + /* + * Both read/write and mmap path can race here. So we need something + * to make sure if we are setting up mapping, then other path waits + * + * For now, use a semaphore for this. It probably needs to be + * optimized later. + */ + down_read(&fi->dax->sem); + node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); + if (node) { + dmap = node_to_dmap(node); + if (writable && !dmap->writable) { + /* Upgrade read-only mapping to read-write. This will + * require exclusive fi->dax->sem lock as we don't want + * two threads to be trying to this simultaneously + * for same dmap. So drop shared lock and acquire + * exclusive lock. + * + * Before dropping fi->dax->sem lock, take reference + * on dmap so that its not freed by range reclaim. + */ + refcount_inc(&dmap->refcnt); + up_read(&fi->dax->sem); + pr_debug("%s: Upgrading mapping at offset 0x%llx length 0x%llx\n", + __func__, pos, length); + return fuse_upgrade_dax_mapping(inode, pos, length, + flags, iomap); + } else { + fuse_fill_iomap(inode, pos, length, iomap, dmap, flags); + up_read(&fi->dax->sem); + return 0; + } + } else { + up_read(&fi->dax->sem); + pr_debug("%s: no mapping at offset 0x%llx length 0x%llx\n", + __func__, pos, length); + if (pos >= i_size_read(inode)) + goto iomap_hole; + + return fuse_setup_new_dax_mapping(inode, pos, length, flags, + iomap); + } + + /* + * If read beyond end of file happnes, fs code seems to return + * it as hole + */ +iomap_hole: + fuse_fill_iomap_hole(iomap, length); + pr_debug("%s returning hole mapping. pos=0x%llx length_asked=0x%llx length_returned=0x%llx\n", + __func__, pos, length, iomap->length); + return 0; +} + +static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, + struct iomap *iomap) +{ + struct fuse_dax_mapping *dmap = iomap->private; + + if (dmap) { + if (refcount_dec_and_test(&dmap->refcnt)) { + /* refcount should not hit 0. This object only goes + * away when fuse connection goes away + */ + WARN_ON_ONCE(1); + } + } + + /* DAX writes beyond end-of-file aren't handled using iomap, so the + * file size is unchanged and there is nothing to do here. + */ + return 0; +} + +static const struct iomap_ops fuse_iomap_ops = { + .iomap_begin = fuse_iomap_begin, + .iomap_end = fuse_iomap_end, +}; + +static void fuse_wait_dax_page(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + up_write(&fi->i_mmap_sem); + schedule(); + down_write(&fi->i_mmap_sem); +} + +/* Should be called with fi->i_mmap_sem lock held exclusively */ +static int __fuse_dax_break_layouts(struct inode *inode, bool *retry, + loff_t start, loff_t end) +{ + struct page *page; + + page = dax_layout_busy_page_range(inode->i_mapping, start, end); + if (!page) + return 0; + + *retry = true; + return ___wait_var_event(&page->_refcount, + atomic_read(&page->_refcount) == 1, TASK_INTERRUPTIBLE, + 0, 0, fuse_wait_dax_page(inode)); +} + +/* dmap_end == 0 leads to unmapping of whole file */ +int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start, + u64 dmap_end) +{ + bool retry; + int ret; + + do { + retry = false; + ret = __fuse_dax_break_layouts(inode, &retry, dmap_start, + dmap_end); + } while (ret == 0 && retry); + + return ret; +} + +ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock_shared(inode)) + return -EAGAIN; + } else { + inode_lock_shared(inode); + } + + ret = dax_iomap_rw(iocb, to, &fuse_iomap_ops); + inode_unlock_shared(inode); + + /* TODO file_accessed(iocb->f_filp) */ + return ret; +} + +static bool file_extending_write(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + return (iov_iter_rw(from) == WRITE && + ((iocb->ki_pos) >= i_size_read(inode) || + (iocb->ki_pos + iov_iter_count(from) > i_size_read(inode)))); +} + +static ssize_t fuse_dax_direct_write(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + ssize_t ret; + + ret = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); + if (ret < 0) + return ret; + + fuse_invalidate_attr(inode); + fuse_write_update_size(inode, iocb->ki_pos); + return ret; +} + +ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock(inode)) + return -EAGAIN; + } else { + inode_lock(inode); + } + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + goto out; + + ret = file_remove_privs(iocb->ki_filp); + if (ret) + goto out; + /* TODO file_update_time() but we don't want metadata I/O */ + + /* Do not use dax for file extending writes as write and on + * disk i_size increase are not atomic otherwise. + */ + if (file_extending_write(iocb, from)) + ret = fuse_dax_direct_write(iocb, from); + else + ret = dax_iomap_rw(iocb, from, &fuse_iomap_ops); + +out: + inode_unlock(inode); + + if (ret > 0) + ret = generic_write_sync(iocb, ret); + return ret; +} + +static int fuse_dax_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + + struct inode *inode = mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); + + return dax_writeback_mapping_range(mapping, fc->dax->dev, wbc); +} + +static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, + enum page_entry_size pe_size, bool write) +{ + vm_fault_t ret; + struct inode *inode = file_inode(vmf->vma->vm_file); + struct super_block *sb = inode->i_sb; + pfn_t pfn; + int error = 0; + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_conn_dax *fcd = fc->dax; + bool retry = false; + + if (write) + sb_start_pagefault(sb); +retry: + if (retry && !(fcd->nr_free_ranges > 0)) + wait_event(fcd->range_waitq, (fcd->nr_free_ranges > 0)); + + /* + * We need to serialize against not only truncate but also against + * fuse dax memory range reclaim. While a range is being reclaimed, + * we do not want any read/write/mmap to make progress and try + * to populate page cache or access memory we are trying to free. + */ + down_read(&get_fuse_inode(inode)->i_mmap_sem); + ret = dax_iomap_fault(vmf, pe_size, &pfn, &error, &fuse_iomap_ops); + if ((ret & VM_FAULT_ERROR) && error == -EAGAIN) { + error = 0; + retry = true; + up_read(&get_fuse_inode(inode)->i_mmap_sem); + goto retry; + } + + if (ret & VM_FAULT_NEEDDSYNC) + ret = dax_finish_sync_fault(vmf, pe_size, pfn); + up_read(&get_fuse_inode(inode)->i_mmap_sem); + + if (write) + sb_end_pagefault(sb); + + return ret; +} + +static vm_fault_t fuse_dax_fault(struct vm_fault *vmf) +{ + return __fuse_dax_fault(vmf, PE_SIZE_PTE, + vmf->flags & FAULT_FLAG_WRITE); +} + +static vm_fault_t fuse_dax_huge_fault(struct vm_fault *vmf, + enum page_entry_size pe_size) +{ + return __fuse_dax_fault(vmf, pe_size, vmf->flags & FAULT_FLAG_WRITE); +} + +static vm_fault_t fuse_dax_page_mkwrite(struct vm_fault *vmf) +{ + return __fuse_dax_fault(vmf, PE_SIZE_PTE, true); +} + +static vm_fault_t fuse_dax_pfn_mkwrite(struct vm_fault *vmf) +{ + return __fuse_dax_fault(vmf, PE_SIZE_PTE, true); +} + +static const struct vm_operations_struct fuse_dax_vm_ops = { + .fault = fuse_dax_fault, + .huge_fault = fuse_dax_huge_fault, + .page_mkwrite = fuse_dax_page_mkwrite, + .pfn_mkwrite = fuse_dax_pfn_mkwrite, +}; + +int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma) +{ + file_accessed(file); + vma->vm_ops = &fuse_dax_vm_ops; + vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE; + return 0; +} + +static int dmap_writeback_invalidate(struct inode *inode, + struct fuse_dax_mapping *dmap) +{ + int ret; + loff_t start_pos = dmap->itn.start << FUSE_DAX_SHIFT; + loff_t end_pos = (start_pos + FUSE_DAX_SZ - 1); + + ret = filemap_fdatawrite_range(inode->i_mapping, start_pos, end_pos); + if (ret) { + pr_debug("fuse: filemap_fdatawrite_range() failed. err=%d start_pos=0x%llx, end_pos=0x%llx\n", + ret, start_pos, end_pos); + return ret; + } + + ret = invalidate_inode_pages2_range(inode->i_mapping, + start_pos >> PAGE_SHIFT, + end_pos >> PAGE_SHIFT); + if (ret) + pr_debug("fuse: invalidate_inode_pages2_range() failed err=%d\n", + ret); + + return ret; +} + +static int reclaim_one_dmap_locked(struct inode *inode, + struct fuse_dax_mapping *dmap) +{ + int ret; + struct fuse_inode *fi = get_fuse_inode(inode); + + /* + * igrab() was done to make sure inode won't go under us, and this + * further avoids the race with evict(). + */ + ret = dmap_writeback_invalidate(inode, dmap); + if (ret) + return ret; + + /* Remove dax mapping from inode interval tree now */ + interval_tree_remove(&dmap->itn, &fi->dax->tree); + fi->dax->nr--; + + /* It is possible that umount/shutdown has killed the fuse connection + * and worker thread is trying to reclaim memory in parallel. Don't + * warn in that case. + */ + ret = dmap_removemapping_one(inode, dmap); + if (ret && ret != -ENOTCONN) { + pr_warn("Failed to remove mapping. offset=0x%llx len=0x%llx ret=%d\n", + dmap->window_offset, dmap->length, ret); + } + return 0; +} + +/* Find first mapped dmap for an inode and return file offset. Caller needs + * to hold fi->dax->sem lock either shared or exclusive. + */ +static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap; + struct interval_tree_node *node; + + for (node = interval_tree_iter_first(&fi->dax->tree, 0, -1); node; + node = interval_tree_iter_next(node, 0, -1)) { + dmap = node_to_dmap(node); + /* still in use. */ + if (refcount_read(&dmap->refcnt) > 1) + continue; + + return dmap; + } + + return NULL; +} + +/* + * Find first mapping in the tree and free it and return it. Do not add + * it back to free pool. + */ +static struct fuse_dax_mapping * +inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode, + bool *retry) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap; + u64 dmap_start, dmap_end; + unsigned long start_idx; + int ret; + struct interval_tree_node *node; + + down_write(&fi->i_mmap_sem); + + /* Lookup a dmap and corresponding file offset to reclaim. */ + down_read(&fi->dax->sem); + dmap = inode_lookup_first_dmap(inode); + if (dmap) { + start_idx = dmap->itn.start; + dmap_start = start_idx << FUSE_DAX_SHIFT; + dmap_end = dmap_start + FUSE_DAX_SZ - 1; + } + up_read(&fi->dax->sem); + + if (!dmap) + goto out_mmap_sem; + /* + * Make sure there are no references to inode pages using + * get_user_pages() + */ + ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end); + if (ret) { + pr_debug("fuse: fuse_dax_break_layouts() failed. err=%d\n", + ret); + dmap = ERR_PTR(ret); + goto out_mmap_sem; + } + + down_write(&fi->dax->sem); + node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); + /* Range already got reclaimed by somebody else */ + if (!node) { + if (retry) + *retry = true; + goto out_write_dmap_sem; + } + + dmap = node_to_dmap(node); + /* still in use. */ + if (refcount_read(&dmap->refcnt) > 1) { + dmap = NULL; + if (retry) + *retry = true; + goto out_write_dmap_sem; + } + + ret = reclaim_one_dmap_locked(inode, dmap); + if (ret < 0) { + dmap = ERR_PTR(ret); + goto out_write_dmap_sem; + } + + /* Clean up dmap. Do not add back to free list */ + dmap_remove_busy_list(fcd, dmap); + dmap->inode = NULL; + dmap->itn.start = dmap->itn.last = 0; + + pr_debug("fuse: %s: inline reclaimed memory range. inode=%p, window_offset=0x%llx, length=0x%llx\n", + __func__, inode, dmap->window_offset, dmap->length); + +out_write_dmap_sem: + up_write(&fi->dax->sem); +out_mmap_sem: + up_write(&fi->i_mmap_sem); + return dmap; +} + +static struct fuse_dax_mapping * +alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode) +{ + struct fuse_dax_mapping *dmap; + struct fuse_inode *fi = get_fuse_inode(inode); + + while (1) { + bool retry = false; + + dmap = alloc_dax_mapping(fcd); + if (dmap) + return dmap; + + dmap = inode_inline_reclaim_one_dmap(fcd, inode, &retry); + /* + * Either we got a mapping or it is an error, return in both + * the cases. + */ + if (dmap) + return dmap; + + /* If we could not reclaim a mapping because it + * had a reference or some other temporary failure, + * Try again. We want to give up inline reclaim only + * if there is no range assigned to this node. Otherwise + * if a deadlock is possible if we sleep with fi->i_mmap_sem + * held and worker to free memory can't make progress due + * to unavailability of fi->i_mmap_sem lock. So sleep + * only if fi->dax->nr=0 + */ + if (retry) + continue; + /* + * There are no mappings which can be reclaimed. Wait for one. + * We are not holding fi->dax->sem. So it is possible + * that range gets added now. But as we are not holding + * fi->i_mmap_sem, worker should still be able to free up + * a range and wake us up. + */ + if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) { + if (wait_event_killable_exclusive(fcd->range_waitq, + (fcd->nr_free_ranges > 0))) { + return ERR_PTR(-EINTR); + } + } + } +} + +static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd, + struct inode *inode, + unsigned long start_idx) +{ + int ret; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap; + struct interval_tree_node *node; + + /* Find fuse dax mapping at file offset inode. */ + node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); + + /* Range already got cleaned up by somebody else */ + if (!node) + return 0; + dmap = node_to_dmap(node); + + /* still in use. */ + if (refcount_read(&dmap->refcnt) > 1) + return 0; + + ret = reclaim_one_dmap_locked(inode, dmap); + if (ret < 0) + return ret; + + /* Cleanup dmap entry and add back to free list */ + spin_lock(&fcd->lock); + dmap_reinit_add_to_free_pool(fcd, dmap); + spin_unlock(&fcd->lock); + return ret; +} + +/* + * Free a range of memory. + * Locking: + * 1. Take fi->i_mmap_sem to block dax faults. + * 2. Take fi->dax->sem to protect interval tree and also to make sure + * read/write can not reuse a dmap which we might be freeing. + */ +static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd, + struct inode *inode, + unsigned long start_idx, + unsigned long end_idx) +{ + int ret; + struct fuse_inode *fi = get_fuse_inode(inode); + loff_t dmap_start = start_idx << FUSE_DAX_SHIFT; + loff_t dmap_end = (dmap_start + FUSE_DAX_SZ) - 1; + + down_write(&fi->i_mmap_sem); + ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end); + if (ret) { + pr_debug("virtio_fs: fuse_dax_break_layouts() failed. err=%d\n", + ret); + goto out_mmap_sem; + } + + down_write(&fi->dax->sem); + ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx); + up_write(&fi->dax->sem); +out_mmap_sem: + up_write(&fi->i_mmap_sem); + return ret; +} + +static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd, + unsigned long nr_to_free) +{ + struct fuse_dax_mapping *dmap, *pos, *temp; + int ret, nr_freed = 0; + unsigned long start_idx = 0, end_idx = 0; + struct inode *inode = NULL; + + /* Pick first busy range and free it for now*/ + while (1) { + if (nr_freed >= nr_to_free) + break; + + dmap = NULL; + spin_lock(&fcd->lock); + + if (!fcd->nr_busy_ranges) { + spin_unlock(&fcd->lock); + return 0; + } + + list_for_each_entry_safe(pos, temp, &fcd->busy_ranges, + busy_list) { + /* skip this range if it's in use. */ + if (refcount_read(&pos->refcnt) > 1) + continue; + + inode = igrab(pos->inode); + /* + * This inode is going away. That will free + * up all the ranges anyway, continue to + * next range. + */ + if (!inode) + continue; + /* + * Take this element off list and add it tail. If + * this element can't be freed, it will help with + * selecting new element in next iteration of loop. + */ + dmap = pos; + list_move_tail(&dmap->busy_list, &fcd->busy_ranges); + start_idx = end_idx = dmap->itn.start; + break; + } + spin_unlock(&fcd->lock); + if (!dmap) + return 0; + + ret = lookup_and_reclaim_dmap(fcd, inode, start_idx, end_idx); + iput(inode); + if (ret) + return ret; + nr_freed++; + } + return 0; +} + +static void fuse_dax_free_mem_worker(struct work_struct *work) +{ + int ret; + struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax, + free_work.work); + ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK); + if (ret) { + pr_debug("fuse: try_to_free_dmap_chunks() failed with err=%d\n", + ret); + } + + /* If number of free ranges are still below threhold, requeue */ + kick_dmap_free_worker(fcd, 1); +} + +static void fuse_free_dax_mem_ranges(struct list_head *mem_list) +{ + struct fuse_dax_mapping *range, *temp; + + /* Free All allocated elements */ + list_for_each_entry_safe(range, temp, mem_list, list) { + list_del(&range->list); + if (!list_empty(&range->busy_list)) + list_del(&range->busy_list); + kfree(range); + } +} + +void fuse_dax_conn_free(struct fuse_conn *fc) +{ + if (fc->dax) { + fuse_free_dax_mem_ranges(&fc->dax->free_ranges); + kfree(fc->dax); + fc->dax = NULL; + } +} + +static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd) +{ + long nr_pages, nr_ranges; + void *kaddr; + pfn_t pfn; + struct fuse_dax_mapping *range; + int ret, id; + size_t dax_size = -1; + unsigned long i; + + init_waitqueue_head(&fcd->range_waitq); + INIT_LIST_HEAD(&fcd->free_ranges); + INIT_LIST_HEAD(&fcd->busy_ranges); + INIT_DELAYED_WORK(&fcd->free_work, fuse_dax_free_mem_worker); + + id = dax_read_lock(); + nr_pages = dax_direct_access(fcd->dev, 0, PHYS_PFN(dax_size), &kaddr, + &pfn); + dax_read_unlock(id); + if (nr_pages < 0) { + pr_debug("dax_direct_access() returned %ld\n", nr_pages); + return nr_pages; + } + + nr_ranges = nr_pages/FUSE_DAX_PAGES; + pr_debug("%s: dax mapped %ld pages. nr_ranges=%ld\n", + __func__, nr_pages, nr_ranges); + + for (i = 0; i < nr_ranges; i++) { + range = kzalloc(sizeof(struct fuse_dax_mapping), GFP_KERNEL); + ret = -ENOMEM; + if (!range) + goto out_err; + + /* TODO: This offset only works if virtio-fs driver is not + * having some memory hidden at the beginning. This needs + * better handling + */ + range->window_offset = i * FUSE_DAX_SZ; + range->length = FUSE_DAX_SZ; + INIT_LIST_HEAD(&range->busy_list); + refcount_set(&range->refcnt, 1); + list_add_tail(&range->list, &fcd->free_ranges); + } + + fcd->nr_free_ranges = nr_ranges; + fcd->nr_ranges = nr_ranges; + return 0; +out_err: + /* Free All allocated elements */ + fuse_free_dax_mem_ranges(&fcd->free_ranges); + return ret; +} + +int fuse_dax_conn_alloc(struct fuse_conn *fc, struct dax_device *dax_dev) +{ + struct fuse_conn_dax *fcd; + int err; + + if (!dax_dev) + return 0; + + fcd = kzalloc(sizeof(*fcd), GFP_KERNEL); + if (!fcd) + return -ENOMEM; + + spin_lock_init(&fcd->lock); + fcd->dev = dax_dev; + err = fuse_dax_mem_range_init(fcd); + if (err) { + kfree(fcd); + return err; + } + + fc->dax = fcd; + return 0; +} + +bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi) +{ + struct fuse_conn *fc = get_fuse_conn_super(sb); + + fi->dax = NULL; + if (fc->dax) { + fi->dax = kzalloc(sizeof(*fi->dax), GFP_KERNEL_ACCOUNT); + if (!fi->dax) + return false; + + init_rwsem(&fi->dax->sem); + fi->dax->tree = RB_ROOT_CACHED; + } + + return true; +} + +static const struct address_space_operations fuse_dax_file_aops = { + .writepages = fuse_dax_writepages, + .direct_IO = noop_direct_IO, + .set_page_dirty = noop_set_page_dirty, + .invalidatepage = noop_invalidatepage, +}; + +void fuse_dax_inode_init(struct inode *inode) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + + if (!fc->dax) + return; + + inode->i_flags |= S_DAX; + inode->i_data.a_ops = &fuse_dax_file_aops; +} + +bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment) +{ + if (fc->dax && (map_alignment > FUSE_DAX_SHIFT)) { + pr_warn("FUSE: map_alignment %u incompatible with dax mem range size %u\n", + map_alignment, FUSE_DAX_SZ); + return false; + } + return true; +} + +void fuse_dax_cancel_work(struct fuse_conn *fc) +{ + struct fuse_conn_dax *fcd = fc->dax; + + if (fcd) + cancel_delayed_work_sync(&fcd->free_work); + +} +EXPORT_SYMBOL_GPL(fuse_dax_cancel_work); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 90afaa7eeeeb..c908eb46cbd0 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -42,20 +42,21 @@ static struct fuse_dev *fuse_get_dev(struct file *file) return READ_ONCE(file->private_data); } -static void fuse_request_init(struct fuse_req *req) +static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req) { INIT_LIST_HEAD(&req->list); INIT_LIST_HEAD(&req->intr_entry); init_waitqueue_head(&req->waitq); refcount_set(&req->count, 1); __set_bit(FR_PENDING, &req->flags); + req->fm = fm; } -static struct fuse_req *fuse_request_alloc(gfp_t flags) +static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags) { struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags); if (req) - fuse_request_init(req); + fuse_request_init(fm, req); return req; } @@ -102,10 +103,11 @@ static void fuse_drop_waiting(struct fuse_conn *fc) } } -static void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req); +static void fuse_put_request(struct fuse_req *req); -static struct fuse_req *fuse_get_req(struct fuse_conn *fc, bool for_background) +static struct fuse_req *fuse_get_req(struct fuse_mount *fm, bool for_background) { + struct fuse_conn *fc = fm->fc; struct fuse_req *req; int err; atomic_inc(&fc->num_waiting); @@ -127,7 +129,7 @@ static struct fuse_req *fuse_get_req(struct fuse_conn *fc, bool for_background) if (fc->conn_error) goto out; - req = fuse_request_alloc(GFP_KERNEL); + req = fuse_request_alloc(fm, GFP_KERNEL); err = -ENOMEM; if (!req) { if (for_background) @@ -145,7 +147,7 @@ static struct fuse_req *fuse_get_req(struct fuse_conn *fc, bool for_background) if (unlikely(req->in.h.uid == ((uid_t)-1) || req->in.h.gid == ((gid_t)-1))) { - fuse_put_request(fc, req); + fuse_put_request(req); return ERR_PTR(-EOVERFLOW); } return req; @@ -155,8 +157,10 @@ static struct fuse_req *fuse_get_req(struct fuse_conn *fc, bool for_background) return ERR_PTR(err); } -static void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req) +static void fuse_put_request(struct fuse_req *req) { + struct fuse_conn *fc = req->fm->fc; + if (refcount_dec_and_test(&req->count)) { if (test_bit(FR_BACKGROUND, &req->flags)) { /* @@ -239,6 +243,11 @@ void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget, { struct fuse_iqueue *fiq = &fc->iq; + if (nodeid == 0) { + kfree(forget); + return; + } + forget->forget_one.nodeid = nodeid; forget->forget_one.nlookup = nlookup; @@ -278,8 +287,10 @@ static void flush_bg_queue(struct fuse_conn *fc) * the 'end' callback is called if given, else the reference to the * request is released */ -void fuse_request_end(struct fuse_conn *fc, struct fuse_req *req) +void fuse_request_end(struct fuse_req *req) { + struct fuse_mount *fm = req->fm; + struct fuse_conn *fc = fm->fc; struct fuse_iqueue *fiq = &fc->iq; if (test_and_set_bit(FR_FINISHED, &req->flags)) @@ -314,9 +325,9 @@ void fuse_request_end(struct fuse_conn *fc, struct fuse_req *req) wake_up(&fc->blocked_waitq); } - if (fc->num_background == fc->congestion_threshold && fc->sb) { - clear_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC); - clear_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC); + if (fc->num_background == fc->congestion_threshold && fm->sb) { + clear_bdi_congested(fm->sb->s_bdi, BLK_RW_SYNC); + clear_bdi_congested(fm->sb->s_bdi, BLK_RW_ASYNC); } fc->num_background--; fc->active_background--; @@ -328,14 +339,16 @@ void fuse_request_end(struct fuse_conn *fc, struct fuse_req *req) } if (test_bit(FR_ASYNC, &req->flags)) - req->args->end(fc, req->args, req->out.h.error); + req->args->end(fm, req->args, req->out.h.error); put_request: - fuse_put_request(fc, req); + fuse_put_request(req); } EXPORT_SYMBOL_GPL(fuse_request_end); -static int queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) +static int queue_interrupt(struct fuse_req *req) { + struct fuse_iqueue *fiq = &req->fm->fc->iq; + spin_lock(&fiq->lock); /* Check for we've sent request to interrupt this req */ if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags))) { @@ -347,7 +360,7 @@ static int queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) list_add_tail(&req->intr_entry, &fiq->interrupts); /* * Pairs with smp_mb() implied by test_and_set_bit() - * from request_end(). + * from fuse_request_end(). */ smp_mb(); if (test_bit(FR_FINISHED, &req->flags)) { @@ -362,8 +375,9 @@ static int queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) return 0; } -static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req) +static void request_wait_answer(struct fuse_req *req) { + struct fuse_conn *fc = req->fm->fc; struct fuse_iqueue *fiq = &fc->iq; int err; @@ -378,7 +392,7 @@ static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req) /* matches barrier in fuse_dev_do_read() */ smp_mb__after_atomic(); if (test_bit(FR_SENT, &req->flags)) - queue_interrupt(fiq, req); + queue_interrupt(req); } if (!test_bit(FR_FORCE, &req->flags)) { @@ -409,9 +423,9 @@ static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req) test_bit(FR_FINISHED, &req->flags)); } -static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req) +static void __fuse_request_send(struct fuse_req *req) { - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &req->fm->fc->iq; BUG_ON(test_bit(FR_BACKGROUND, &req->flags)); spin_lock(&fiq->lock); @@ -425,7 +439,7 @@ static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req) __fuse_get_request(req); queue_request_and_unlock(fiq, req, 1); - request_wait_answer(fc, req); + request_wait_answer(req); /* Pairs with smp_wmb() in fuse_request_end() */ smp_rmb(); } @@ -464,8 +478,10 @@ static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args) } } -static void fuse_force_creds(struct fuse_conn *fc, struct fuse_req *req) +static void fuse_force_creds(struct fuse_req *req) { + struct fuse_conn *fc = req->fm->fc; + req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid()); req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid()); req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns); @@ -475,28 +491,30 @@ static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) { req->in.h.opcode = args->opcode; req->in.h.nodeid = args->nodeid; + req->in.h.error_in = args->error_in; req->args = args; if (args->end) __set_bit(FR_ASYNC, &req->flags); } -ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args) +ssize_t fuse_simple_request(struct fuse_mount *fm, struct fuse_args *args) { + struct fuse_conn *fc = fm->fc; struct fuse_req *req; ssize_t ret; if (args->force) { atomic_inc(&fc->num_waiting); - req = fuse_request_alloc(GFP_KERNEL | __GFP_NOFAIL); + req = fuse_request_alloc(fm, GFP_KERNEL | __GFP_NOFAIL); if (!args->nocreds) - fuse_force_creds(fc, req); + fuse_force_creds(req); __set_bit(FR_WAITING, &req->flags); __set_bit(FR_FORCE, &req->flags); } else { WARN_ON(args->nocreds); - req = fuse_get_req(fc, false); + req = fuse_get_req(fm, false); if (IS_ERR(req)) return PTR_ERR(req); } @@ -507,20 +525,21 @@ ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args) if (!args->noreply) __set_bit(FR_ISREPLY, &req->flags); - __fuse_request_send(fc, req); + __fuse_request_send(req); ret = req->out.h.error; if (!ret && args->out_argvar) { BUG_ON(args->out_numargs == 0); ret = args->out_args[args->out_numargs - 1].size; } - fuse_put_request(fc, req); + fuse_put_request(req); return ret; } -static bool fuse_request_queue_background(struct fuse_conn *fc, - struct fuse_req *req) +static bool fuse_request_queue_background(struct fuse_req *req) { + struct fuse_mount *fm = req->fm; + struct fuse_conn *fc = fm->fc; bool queued = false; WARN_ON(!test_bit(FR_BACKGROUND, &req->flags)); @@ -534,9 +553,9 @@ static bool fuse_request_queue_background(struct fuse_conn *fc, fc->num_background++; if (fc->num_background == fc->max_background) fc->blocked = 1; - if (fc->num_background == fc->congestion_threshold && fc->sb) { - set_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC); - set_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC); + if (fc->num_background == fc->congestion_threshold && fm->sb) { + set_bdi_congested(fm->sb->s_bdi, BLK_RW_SYNC); + set_bdi_congested(fm->sb->s_bdi, BLK_RW_ASYNC); } list_add_tail(&req->list, &fc->bg_queue); flush_bg_queue(fc); @@ -547,28 +566,28 @@ static bool fuse_request_queue_background(struct fuse_conn *fc, return queued; } -int fuse_simple_background(struct fuse_conn *fc, struct fuse_args *args, +int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, gfp_t gfp_flags) { struct fuse_req *req; if (args->force) { WARN_ON(!args->nocreds); - req = fuse_request_alloc(gfp_flags); + req = fuse_request_alloc(fm, gfp_flags); if (!req) return -ENOMEM; __set_bit(FR_BACKGROUND, &req->flags); } else { WARN_ON(args->nocreds); - req = fuse_get_req(fc, true); + req = fuse_get_req(fm, true); if (IS_ERR(req)) return PTR_ERR(req); } fuse_args_to_req(req, args); - if (!fuse_request_queue_background(fc, req)) { - fuse_put_request(fc, req); + if (!fuse_request_queue_background(req)) { + fuse_put_request(req); return -ENOTCONN; } @@ -576,14 +595,14 @@ int fuse_simple_background(struct fuse_conn *fc, struct fuse_args *args, } EXPORT_SYMBOL_GPL(fuse_simple_background); -static int fuse_simple_notify_reply(struct fuse_conn *fc, +static int fuse_simple_notify_reply(struct fuse_mount *fm, struct fuse_args *args, u64 unique) { struct fuse_req *req; - struct fuse_iqueue *fiq = &fc->iq; + struct fuse_iqueue *fiq = &fm->fc->iq; int err = 0; - req = fuse_get_req(fc, false); + req = fuse_get_req(fm, false); if (IS_ERR(req)) return PTR_ERR(req); @@ -598,7 +617,7 @@ static int fuse_simple_notify_reply(struct fuse_conn *fc, } else { err = -ENODEV; spin_unlock(&fiq->lock); - fuse_put_request(fc, req); + fuse_put_request(req); } return err; @@ -677,6 +696,10 @@ static void fuse_copy_finish(struct fuse_copy_state *cs) flush_dcache_page(cs->pg); set_page_dirty_lock(cs->pg); } + /* + * The page could be GUP page(see iov_iter_get_pages in + * fuse_copy_fill) so use put_page to release it. + */ put_page(cs->pg); } cs->pg = NULL; @@ -780,8 +803,7 @@ static int fuse_check_page(struct page *page) 1 << PG_workingset | 1 << PG_reclaim | 1 << PG_waiters))) { - pr_warn("trying to steal weird page\n"); - pr_warn(" page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping); + dump_page(page, "fuse: trying to steal weird page"); return 1; } return 0; @@ -1283,7 +1305,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, /* SETXATTR is special, since it may contain too large data */ if (args->opcode == FUSE_SETXATTR) req->out.h.error = -E2BIG; - fuse_request_end(fc, req); + fuse_request_end(req); goto restart; } spin_lock(&fpq->lock); @@ -1326,8 +1348,8 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, /* matches barrier in request_wait_answer() */ smp_mb__after_atomic(); if (test_bit(FR_INTERRUPTED, &req->flags)) - queue_interrupt(fiq, req); - fuse_put_request(fc, req); + queue_interrupt(req); + fuse_put_request(req); return reqsize; @@ -1335,7 +1357,7 @@ out_end: if (!test_bit(FR_PRIVATE, &req->flags)) list_del_init(&req->list); spin_unlock(&fpq->lock); - fuse_request_end(fc, req); + fuse_request_end(req); return err; err_unlock: @@ -1458,11 +1480,8 @@ static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size, fuse_copy_finish(cs); down_read(&fc->killsb); - err = -ENOENT; - if (fc->sb) { - err = fuse_reverse_inval_inode(fc->sb, outarg.ino, - outarg.off, outarg.len); - } + err = fuse_reverse_inval_inode(fc, outarg.ino, + outarg.off, outarg.len); up_read(&fc->killsb); return err; @@ -1508,9 +1527,7 @@ static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size, buf[outarg.namelen] = 0; down_read(&fc->killsb); - err = -ENOENT; - if (fc->sb) - err = fuse_reverse_inval_entry(fc->sb, outarg.parent, 0, &name); + err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name); up_read(&fc->killsb); kfree(buf); return err; @@ -1558,10 +1575,7 @@ static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size, buf[outarg.namelen] = 0; down_read(&fc->killsb); - err = -ENOENT; - if (fc->sb) - err = fuse_reverse_inval_entry(fc->sb, outarg.parent, - outarg.child, &name); + err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name); up_read(&fc->killsb); kfree(buf); return err; @@ -1603,10 +1617,7 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, down_read(&fc->killsb); err = -ENOENT; - if (!fc->sb) - goto out_up_killsb; - - inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid); + inode = fuse_ilookup(fc, nodeid, NULL); if (!inode) goto out_up_killsb; @@ -1665,7 +1676,7 @@ struct fuse_retrieve_args { struct fuse_notify_retrieve_in inarg; }; -static void fuse_retrieve_end(struct fuse_conn *fc, struct fuse_args *args, +static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args, int error) { struct fuse_retrieve_args *ra = @@ -1675,7 +1686,7 @@ static void fuse_retrieve_end(struct fuse_conn *fc, struct fuse_args *args, kfree(ra); } -static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode, +static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, struct fuse_notify_retrieve_out *outarg) { int err; @@ -1686,6 +1697,7 @@ static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode, unsigned int offset; size_t total_len = 0; unsigned int num_pages; + struct fuse_conn *fc = fm->fc; struct fuse_retrieve_args *ra; size_t args_size = sizeof(*ra); struct fuse_args_pages *ap; @@ -1747,9 +1759,9 @@ static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode, args->in_args[0].value = &ra->inarg; args->in_args[1].size = total_len; - err = fuse_simple_notify_reply(fc, args, outarg->notify_unique); + err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); if (err) - fuse_retrieve_end(fc, args, err); + fuse_retrieve_end(fm, args, err); return err; } @@ -1758,7 +1770,9 @@ static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, struct fuse_copy_state *cs) { struct fuse_notify_retrieve_out outarg; + struct fuse_mount *fm; struct inode *inode; + u64 nodeid; int err; err = -EINVAL; @@ -1773,14 +1787,12 @@ static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, down_read(&fc->killsb); err = -ENOENT; - if (fc->sb) { - u64 nodeid = outarg.nodeid; + nodeid = outarg.nodeid; - inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid); - if (inode) { - err = fuse_retrieve(fc, inode, &outarg); - iput(inode); - } + inode = fuse_ilookup(fc, nodeid, &fm); + if (inode) { + err = fuse_retrieve(fm, inode, &outarg); + iput(inode); } up_read(&fc->killsb); @@ -1919,9 +1931,9 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, else if (oh.error == -ENOSYS) fc->no_interrupt = 1; else if (oh.error == -EAGAIN) - err = queue_interrupt(&fc->iq, req); + err = queue_interrupt(req); - fuse_put_request(fc, req); + fuse_put_request(req); goto copy_finish; } @@ -1947,6 +1959,19 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, req->out.h.error = kern_path(path, 0, req->args->canonical_path); } + if (!err && (req->in.h.opcode == FUSE_LOOKUP || + req->in.h.opcode == (FUSE_LOOKUP | FUSE_POSTFILTER)) && + req->args->out_args[1].size == sizeof(struct fuse_entry_bpf_out)) { + struct fuse_entry_bpf_out *febo = (struct fuse_entry_bpf_out *) + req->args->out_args[1].value; + struct fuse_entry_bpf *feb = container_of(febo, struct fuse_entry_bpf, out); + + if (febo->backing_action == FUSE_ACTION_REPLACE) + feb->backing_file = fget(febo->backing_fd); + if (febo->bpf_action == FUSE_ACTION_REPLACE) + feb->bpf_file = fget(febo->bpf_fd); + } + spin_lock(&fpq->lock); clear_bit(FR_LOCKED, &req->flags); if (!fpq->connected) @@ -1957,7 +1982,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, list_del_init(&req->list); spin_unlock(&fpq->lock); - fuse_request_end(fc, req); + fuse_request_end(req); out: return err ? err : nbytes; @@ -2093,7 +2118,7 @@ static __poll_t fuse_dev_poll(struct file *file, poll_table *wait) } /* Abort all requests on the given list (pending or processing) */ -static void end_requests(struct fuse_conn *fc, struct list_head *head) +static void end_requests(struct list_head *head) { while (!list_empty(head)) { struct fuse_req *req; @@ -2101,7 +2126,7 @@ static void end_requests(struct fuse_conn *fc, struct list_head *head) req->out.h.error = -ECONNABORTED; clear_bit(FR_SENT, &req->flags); list_del_init(&req->list); - fuse_request_end(fc, req); + fuse_request_end(req); } } @@ -2196,7 +2221,7 @@ void fuse_abort_conn(struct fuse_conn *fc) wake_up_all(&fc->blocked_waitq); spin_unlock(&fc->lock); - end_requests(fc, &to_end); + end_requests(&to_end); } else { spin_unlock(&fc->lock); } @@ -2226,7 +2251,7 @@ int fuse_dev_release(struct inode *inode, struct file *file) list_splice_init(&fpq->processing[i], &to_end); spin_unlock(&fpq->lock); - end_requests(fc, &to_end); + end_requests(&to_end); /* Are we the last open device? */ if (atomic_dec_and_test(&fc->dev_count)) { @@ -2328,7 +2353,7 @@ const struct file_operations fuse_dev_operations = { .release = fuse_dev_release, .fasync = fuse_dev_fasync, .unlocked_ioctl = fuse_dev_ioctl, - .compat_ioctl = fuse_dev_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; EXPORT_SYMBOL_GPL(fuse_dev_operations); diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 7cf55a7635fa..a55375ae2797 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -8,8 +8,11 @@ #include "fuse_i.h" +#include #include #include +#include +#include #include #include #include @@ -17,6 +20,8 @@ #include #include +#include "../internal.h" + static void fuse_advise_use_readdirplus(struct inode *dir) { struct fuse_inode *fi = get_fuse_inode(dir); @@ -24,7 +29,7 @@ static void fuse_advise_use_readdirplus(struct inode *dir) set_bit(FUSE_I_ADVISE_RDPLUS, &fi->state); } -#if BITS_PER_LONG >= 64 +#if BITS_PER_LONG >= 64 && !defined(CONFIG_FUSE_BPF) static inline void __fuse_dentry_settime(struct dentry *entry, u64 time) { entry->d_fsdata = (void *) time; @@ -36,19 +41,15 @@ static inline u64 fuse_dentry_time(const struct dentry *entry) } #else -union fuse_dentry { - u64 time; - struct rcu_head rcu; -}; static inline void __fuse_dentry_settime(struct dentry *dentry, u64 time) { - ((union fuse_dentry *) dentry->d_fsdata)->time = time; + ((struct fuse_dentry *) dentry->d_fsdata)->time = time; } static inline u64 fuse_dentry_time(const struct dentry *entry) { - return ((union fuse_dentry *) entry->d_fsdata)->time; + return ((struct fuse_dentry *) entry->d_fsdata)->time; } #endif @@ -73,26 +74,16 @@ static void fuse_dentry_settime(struct dentry *dentry, u64 time) __fuse_dentry_settime(dentry, time); } -/* - * FUSE caches dentries and attributes with separate timeout. The - * time in jiffies until the dentry/attributes are valid is stored in - * dentry->d_fsdata and fuse_inode->i_time respectively. - */ - -/* - * Calculate the time in jiffies until a dentry/attributes are valid - */ -static u64 time_to_jiffies(u64 sec, u32 nsec) +void fuse_init_dentry_root(struct dentry *root, struct file *backing_dir) { - if (sec || nsec) { - struct timespec64 ts = { - sec, - min_t(u32, nsec, NSEC_PER_SEC - 1) - }; +#ifdef CONFIG_FUSE_BPF + struct fuse_dentry *fuse_dentry = root->d_fsdata; - return get_jiffies_64() + timespec64_to_jiffies(&ts); - } else - return 0; + if (backing_dir) { + fuse_dentry->backing_path = backing_dir->f_path; + path_get(&fuse_dentry->backing_path); + } +#endif } /* @@ -105,11 +96,6 @@ void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o) time_to_jiffies(o->entry_valid, o->entry_valid_nsec)); } -static u64 attr_timeout(struct fuse_attr_out *o) -{ - return time_to_jiffies(o->attr_valid, o->attr_valid_nsec); -} - u64 entry_attr_timeout(struct fuse_entry_out *o) { return time_to_jiffies(o->attr_valid, o->attr_valid_nsec); @@ -170,7 +156,8 @@ static void fuse_invalidate_entry(struct dentry *entry) static void fuse_lookup_init(struct fuse_conn *fc, struct fuse_args *args, u64 nodeid, const struct qstr *name, - struct fuse_entry_out *outarg) + struct fuse_entry_out *outarg, + struct fuse_entry_bpf_out *bpf_outarg) { memset(outarg, 0, sizeof(struct fuse_entry_out)); args->opcode = FUSE_LOOKUP; @@ -178,11 +165,57 @@ static void fuse_lookup_init(struct fuse_conn *fc, struct fuse_args *args, args->in_numargs = 1; args->in_args[0].size = name->len + 1; args->in_args[0].value = name->name; - args->out_numargs = 1; + args->out_argvar = true; + args->out_numargs = 2; args->out_args[0].size = sizeof(struct fuse_entry_out); args->out_args[0].value = outarg; + args->out_args[1].size = sizeof(struct fuse_entry_bpf_out); + args->out_args[1].value = bpf_outarg; } +#ifdef CONFIG_FUSE_BPF +static bool backing_data_changed(struct fuse_inode *fi, struct dentry *entry, + struct fuse_entry_bpf *bpf_arg) +{ + struct path new_backing_path; + struct inode *new_backing_inode; + struct bpf_prog *bpf = NULL; + int err; + bool ret = true; + + if (!entry || !fi->backing_inode) { + ret = false; + goto put_backing_file; + } + + get_fuse_backing_path(entry, &new_backing_path); + new_backing_inode = fi->backing_inode; + ihold(new_backing_inode); + + err = fuse_handle_backing(bpf_arg, &new_backing_inode, &new_backing_path); + + if (err) + goto put_inode; + + err = fuse_handle_bpf_prog(bpf_arg, entry->d_parent->d_inode, &bpf); + if (err) + goto put_bpf; + + ret = (bpf != fi->bpf || fi->backing_inode != new_backing_inode || + !path_equal(&get_fuse_dentry(entry)->backing_path, &new_backing_path)); +put_bpf: + if (bpf) + bpf_prog_put(bpf); +put_inode: + iput(new_backing_inode); + path_put(&new_backing_path); +put_backing_file: + if (bpf_arg->backing_file) + fput(bpf_arg->backing_file); + return ret; +} +#endif + /* * Check whether the dentry is still valid * @@ -196,16 +229,29 @@ static int fuse_dentry_revalidate(struct dentry *entry, unsigned int flags) { struct inode *inode; struct dentry *parent; - struct fuse_conn *fc; + struct fuse_mount *fm; struct fuse_inode *fi; int ret; inode = d_inode_rcu(entry); if (inode && fuse_is_bad(inode)) goto invalid; - else if (time_before64(fuse_dentry_time(entry), get_jiffies_64()) || - (flags & LOOKUP_REVAL)) { +#ifdef CONFIG_FUSE_BPF + /* TODO: Do we need bpf support for revalidate? + * If the lower filesystem says the entry is invalid, FUSE probably shouldn't + * try to fix that without going through the normal lookup path... + */ + if (get_fuse_dentry(entry)->backing_path.dentry) { + ret = fuse_revalidate_backing(entry, flags); + if (ret <= 0) { + goto out; + } + } +#endif + if (time_before64(fuse_dentry_time(entry), get_jiffies_64()) || + (flags & (LOOKUP_EXCL | LOOKUP_REVAL | LOOKUP_RENAME_TARGET))) { struct fuse_entry_out outarg; + struct fuse_entry_bpf bpf_arg; FUSE_ARGS(args); struct fuse_forget_link *forget; u64 attr_version; @@ -217,28 +263,47 @@ static int fuse_dentry_revalidate(struct dentry *entry, unsigned int flags) ret = -ECHILD; if (flags & LOOKUP_RCU) goto out; - - fc = get_fuse_conn(inode); - - forget = fuse_alloc_forget(); - ret = -ENOMEM; - if (!forget) - goto out; - - attr_version = fuse_get_attr_version(fc); + fm = get_fuse_mount(inode); parent = dget_parent(entry); - fuse_lookup_init(fc, &args, get_node_id(d_inode(parent)), - &entry->d_name, &outarg); - ret = fuse_simple_request(fc, &args); + +#ifdef CONFIG_FUSE_BPF + /* TODO: Once we're handling timeouts for backing inodes, do a + * bpf based lookup_revalidate here. + */ + if (get_fuse_inode(parent->d_inode)->backing_inode) { + dput(parent); + ret = 1; + goto out; + } +#endif + forget = fuse_alloc_forget(); + ret = -ENOMEM; + if (!forget) { + dput(parent); + goto out; + } + + attr_version = fuse_get_attr_version(fm->fc); + + fuse_lookup_init(fm->fc, &args, get_node_id(d_inode(parent)), + &entry->d_name, &outarg, &bpf_arg.out); + ret = fuse_simple_request(fm, &args); dput(parent); + /* Zero nodeid is same as -ENOENT */ if (!ret && !outarg.nodeid) ret = -ENOENT; - if (!ret) { + if (!ret || ret == sizeof(bpf_arg.out)) { fi = get_fuse_inode(inode); - if (outarg.nodeid != get_node_id(inode)) { - fuse_queue_forget(fc, forget, outarg.nodeid, 1); + if (outarg.nodeid != get_node_id(inode) || +#ifdef CONFIG_FUSE_BPF + (ret == sizeof(bpf_arg.out) && + backing_data_changed(fi, entry, &bpf_arg)) || +#endif + (bool) IS_AUTOMOUNT(inode) != (bool) (outarg.attr.flags & FUSE_ATTR_SUBMOUNT)) { + fuse_queue_forget(fm->fc, forget, + outarg.nodeid, 1); goto invalid; } spin_lock(&fi->lock); @@ -249,7 +314,7 @@ static int fuse_dentry_revalidate(struct dentry *entry, unsigned int flags) if (ret == -ENOMEM || ret == -EINTR) goto out; if (ret || fuse_invalid_attr(&outarg.attr) || - (outarg.attr.mode ^ inode->i_mode) & S_IFMT) + fuse_stale_inode(inode, outarg.generation, &outarg.attr)) goto invalid; forget_all_cached_acls(inode); @@ -277,17 +342,25 @@ invalid: goto out; } -#if BITS_PER_LONG < 64 +#if BITS_PER_LONG < 64 || defined(CONFIG_FUSE_BPF) static int fuse_dentry_init(struct dentry *dentry) { - dentry->d_fsdata = kzalloc(sizeof(union fuse_dentry), + dentry->d_fsdata = kzalloc(sizeof(struct fuse_dentry), GFP_KERNEL_ACCOUNT | __GFP_RECLAIMABLE); return dentry->d_fsdata ? 0 : -ENOMEM; } static void fuse_dentry_release(struct dentry *dentry) { - union fuse_dentry *fd = dentry->d_fsdata; + struct fuse_dentry *fd = dentry->d_fsdata; + +#ifdef CONFIG_FUSE_BPF + if (fd && fd->backing_path.dentry) + path_put(&fd->backing_path); + + if (fd && fd->bpf) + bpf_prog_put(fd->bpf); +#endif kfree_rcu(fd, rcu); } @@ -306,11 +379,25 @@ static int fuse_dentry_delete(const struct dentry *dentry) */ static void fuse_dentry_canonical_path(const struct path *path, struct path *canonical_path) { struct inode *inode = d_inode(path->dentry); - struct fuse_conn *fc = get_fuse_conn(inode); + //struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount_super(path->mnt->mnt_sb); FUSE_ARGS(args); char *path_name; int err; + +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_dummy_io, + fuse_canonical_path_initialize, + fuse_canonical_path_backing, + fuse_canonical_path_finalize, path, + canonical_path); + if (fer.ret) + return; +#endif + path_name = (char *)get_zeroed_page(GFP_KERNEL); if (!path_name) goto default_path; @@ -324,7 +411,7 @@ static void fuse_dentry_canonical_path(const struct path *path, struct path *can args.canonical_path = canonical_path; args.out_argvar = 1; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); free_page((unsigned long)path_name); if (err > 0) return; @@ -334,18 +421,107 @@ default_path: path_get(canonical_path); } +/* + * Create a fuse_mount object with a new superblock (with path->dentry + * as the root), and return that mount so it can be auto-mounted on + * @path. + */ +static struct vfsmount *fuse_dentry_automount(struct path *path) +{ + struct fs_context *fsc; + struct fuse_mount *parent_fm = get_fuse_mount_super(path->mnt->mnt_sb); + struct fuse_conn *fc = parent_fm->fc; + struct fuse_mount *fm; + struct vfsmount *mnt; + struct fuse_inode *mp_fi = get_fuse_inode(d_inode(path->dentry)); + struct super_block *sb; + int err; + + fsc = fs_context_for_submount(path->mnt->mnt_sb->s_type, path->dentry); + if (IS_ERR(fsc)) { + err = PTR_ERR(fsc); + goto out; + } + + err = -ENOMEM; + fm = kzalloc(sizeof(struct fuse_mount), GFP_KERNEL); + if (!fm) + goto out_put_fsc; + + refcount_set(&fm->count, 1); + fsc->s_fs_info = fm; + sb = sget_fc(fsc, NULL, set_anon_super_fc); + if (IS_ERR(sb)) { + err = PTR_ERR(sb); + fuse_mount_put(fm); + goto out_put_fsc; + } + fm->fc = fuse_conn_get(fc); + + /* Initialize superblock, making @mp_fi its root */ + err = fuse_fill_super_submount(sb, mp_fi); + if (err) { + fuse_conn_put(fc); + kfree(fm); + sb->s_fs_info = NULL; + goto out_put_sb; + } + + down_write(&fc->killsb); + list_add_tail(&fm->fc_entry, &fc->mounts); + up_write(&fc->killsb); + + sb->s_flags |= SB_ACTIVE; + fsc->root = dget(sb->s_root); + + /* + * FIXME: setting SB_BORN requires a write barrier for + * super_cache_count(). We should actually come + * up with a proper ->get_tree() implementation + * for submounts and call vfs_get_tree() to take + * care of the write barrier. + */ + smp_wmb(); + sb->s_flags |= SB_BORN; + + /* We are done configuring the superblock, so unlock it */ + up_write(&sb->s_umount); + + /* Create the submount */ + mnt = vfs_create_mount(fsc); + if (IS_ERR(mnt)) { + err = PTR_ERR(mnt); + goto out_put_fsc; + } + mntget(mnt); + put_fs_context(fsc); + return mnt; + +out_put_sb: + /* + * Only jump here when fsc->root is NULL and sb is still locked + * (otherwise put_fs_context() will put the superblock) + */ + deactivate_locked_super(sb); +out_put_fsc: + put_fs_context(fsc); +out: + return ERR_PTR(err); +} + const struct dentry_operations fuse_dentry_operations = { .d_revalidate = fuse_dentry_revalidate, .d_delete = fuse_dentry_delete, -#if BITS_PER_LONG < 64 +#if BITS_PER_LONG < 64 || defined(CONFIG_FUSE_BPF) .d_init = fuse_dentry_init, .d_release = fuse_dentry_release, #endif + .d_automount = fuse_dentry_automount, .d_canonical_path = fuse_dentry_canonical_path, }; const struct dentry_operations fuse_root_dentry_operations = { -#if BITS_PER_LONG < 64 +#if BITS_PER_LONG < 64 || defined(CONFIG_FUSE_BPF) .d_init = fuse_dentry_init, .d_release = fuse_dentry_release, #endif @@ -364,10 +540,13 @@ bool fuse_invalid_attr(struct fuse_attr *attr) } int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name, - struct fuse_entry_out *outarg, struct inode **inode) + struct fuse_entry_out *outarg, + struct dentry *entry, + struct inode **inode) { - struct fuse_conn *fc = get_fuse_conn_super(sb); + struct fuse_mount *fm = get_fuse_mount_super(sb); FUSE_ARGS(args); + struct fuse_entry_bpf bpf_arg = {0}; struct fuse_forget_link *forget; u64 attr_version; int err; @@ -377,32 +556,77 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name if (name->len > FUSE_NAME_MAX) goto out; - forget = fuse_alloc_forget(); err = -ENOMEM; if (!forget) goto out; - attr_version = fuse_get_attr_version(fc); + attr_version = fuse_get_attr_version(fm->fc); - fuse_lookup_init(fc, &args, nodeid, name, outarg); - err = fuse_simple_request(fc, &args); - /* Zero nodeid is same as -ENOENT, but with valid timeout */ - if (err || !outarg->nodeid) - goto out_put_forget; + fuse_lookup_init(fm->fc, &args, nodeid, name, outarg, &bpf_arg.out); + err = fuse_simple_request(fm, &args); - err = -EIO; - if (!outarg->nodeid) - goto out_put_forget; - if (fuse_invalid_attr(&outarg->attr)) - goto out_put_forget; +#ifdef CONFIG_FUSE_BPF + if (err == sizeof(bpf_arg.out)) { + /* TODO Make sure this handles invalid handles */ + struct file *backing_file; + struct inode *backing_inode; + + err = -ENOENT; + if (!entry) + goto out_put_forget; + + err = -EINVAL; + backing_file = bpf_arg.backing_file; + if (!backing_file) + goto out_put_forget; + + if (IS_ERR(backing_file)) { + err = PTR_ERR(backing_file); + goto out_put_forget; + } + + backing_inode = backing_file->f_inode; + *inode = fuse_iget_backing(sb, outarg->nodeid, backing_inode); + if (!*inode) + goto out_put_forget; + + err = fuse_handle_backing(&bpf_arg, + &get_fuse_inode(*inode)->backing_inode, + &get_fuse_dentry(entry)->backing_path); + if (!err) + err = fuse_handle_bpf_prog(&bpf_arg, NULL, + &get_fuse_inode(*inode)->bpf); + if (err) { + iput(*inode); + *inode = NULL; + goto out_put_forget; + } + } else +#endif + { + /* Zero nodeid is same as -ENOENT, but with valid timeout */ + if (err || !outarg->nodeid) + goto out_put_forget; + + err = -EIO; + if (!outarg->nodeid) + goto out_put_forget; + if (fuse_invalid_attr(&outarg->attr)) + goto out_put_forget; + if (outarg->nodeid == FUSE_ROOT_ID && outarg->generation != 0) { + pr_warn_once("root generation should be zero\n"); + outarg->generation = 0; + } + + *inode = fuse_iget(sb, outarg->nodeid, outarg->generation, + &outarg->attr, entry_attr_timeout(outarg), + attr_version); + } - *inode = fuse_iget(sb, outarg->nodeid, outarg->generation, - &outarg->attr, entry_attr_timeout(outarg), - attr_version); err = -ENOMEM; - if (!*inode) { - fuse_queue_forget(fc, forget, outarg->nodeid, 1); + if (!*inode && outarg->nodeid) { + fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1); goto out; } err = 0; @@ -410,6 +634,8 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name out_put_forget: kfree(forget); out: + if (bpf_arg.backing_file) + fput(bpf_arg.backing_file); return err; } @@ -423,12 +649,23 @@ static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry, bool outarg_valid = true; bool locked; +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(dir, struct fuse_lookup_io, + fuse_lookup_initialize, fuse_lookup_backing, + fuse_lookup_finalize, + dir, entry, flags); + if (fer.ret) + return fer.result; +#endif + if (fuse_is_bad(dir)) return ERR_PTR(-EIO); locked = fuse_lock_inode(dir); err = fuse_lookup_name(dir->i_sb, get_node_id(dir), &entry->d_name, - &outarg, &inode); + &outarg, entry, &inode); fuse_unlock_inode(dir, locked); if (err == -ENOENT) { outarg_valid = false; @@ -475,6 +712,7 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry, int err; struct inode *inode; struct fuse_conn *fc = get_fuse_conn(dir); + struct fuse_mount *fm = get_fuse_mount(dir); FUSE_ARGS(args); struct fuse_forget_link *forget; struct fuse_create_in inarg; @@ -482,21 +720,36 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry, struct fuse_entry_out outentry; struct fuse_inode *fi; struct fuse_file *ff; + bool trunc = flags & O_TRUNC; /* Userspace expects S_IFREG in create mode */ BUG_ON((mode & S_IFMT) != S_IFREG); +#ifdef CONFIG_FUSE_BPF + { + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(dir, struct fuse_create_open_io, + fuse_create_open_initialize, + fuse_create_open_backing, + fuse_create_open_finalize, + dir, entry, file, flags, mode); + if (fer.ret) + return PTR_ERR(fer.result); + } +#endif + forget = fuse_alloc_forget(); err = -ENOMEM; if (!forget) goto out_err; err = -ENOMEM; - ff = fuse_file_alloc(fc); + ff = fuse_file_alloc(fm); if (!ff) goto out_put_forget_req; - if (!fc->dont_mask) + if (!fm->fc->dont_mask) mode &= ~current_umask(); flags &= ~O_NOCTTY; @@ -517,7 +770,7 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry, args.out_args[0].value = &outentry; args.out_args[1].size = sizeof(outopen); args.out_args[1].value = &outopen; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err) goto out_free_ff; @@ -535,7 +788,7 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry, if (!inode) { flags &= ~(O_CREAT | O_EXCL | O_TRUNC); fuse_sync_release(NULL, ff, flags); - fuse_queue_forget(fc, forget, outentry.nodeid, 1); + fuse_queue_forget(fm->fc, forget, outentry.nodeid, 1); err = -ENOMEM; goto out_err; } @@ -550,6 +803,10 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry, } else { file->private_data = ff; fuse_finish_open(inode, file); + if (fm->fc->atomic_o_trunc && trunc) + truncate_pagecache(inode, 0); + else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + invalidate_inode_pages2(inode->i_mapping); } return err; @@ -611,7 +868,7 @@ no_open: /* * Code shared between mknod, mkdir, symlink and link */ -static int create_new_entry(struct fuse_conn *fc, struct fuse_args *args, +static int create_new_entry(struct fuse_mount *fm, struct fuse_args *args, struct inode *dir, struct dentry *entry, umode_t mode) { @@ -633,7 +890,7 @@ static int create_new_entry(struct fuse_conn *fc, struct fuse_args *args, args->out_numargs = 1; args->out_args[0].size = sizeof(outarg); args->out_args[0].value = &outarg; - err = fuse_simple_request(fc, args); + err = fuse_simple_request(fm, args); if (err) goto out_put_forget_req; @@ -647,7 +904,7 @@ static int create_new_entry(struct fuse_conn *fc, struct fuse_args *args, inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation, &outarg.attr, entry_attr_timeout(&outarg), 0); if (!inode) { - fuse_queue_forget(fc, forget, outarg.nodeid, 1); + fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1); return -ENOMEM; } kfree(forget); @@ -675,10 +932,21 @@ static int fuse_mknod(struct inode *dir, struct dentry *entry, umode_t mode, dev_t rdev) { struct fuse_mknod_in inarg; - struct fuse_conn *fc = get_fuse_conn(dir); + struct fuse_mount *fm = get_fuse_mount(dir); FUSE_ARGS(args); - if (!fc->dont_mask) +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(dir, struct fuse_mknod_in, + fuse_mknod_initialize, fuse_mknod_backing, + fuse_mknod_finalize, + dir, entry, mode, rdev); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + + if (!fm->fc->dont_mask) mode &= ~current_umask(); memset(&inarg, 0, sizeof(inarg)); @@ -691,7 +959,7 @@ static int fuse_mknod(struct inode *dir, struct dentry *entry, umode_t mode, args.in_args[0].value = &inarg; args.in_args[1].size = entry->d_name.len + 1; args.in_args[1].value = entry->d_name.name; - return create_new_entry(fc, &args, dir, entry, mode); + return create_new_entry(fm, &args, dir, entry, mode); } static int fuse_create(struct inode *dir, struct dentry *entry, umode_t mode, @@ -703,10 +971,21 @@ static int fuse_create(struct inode *dir, struct dentry *entry, umode_t mode, static int fuse_mkdir(struct inode *dir, struct dentry *entry, umode_t mode) { struct fuse_mkdir_in inarg; - struct fuse_conn *fc = get_fuse_conn(dir); + struct fuse_mount *fm = get_fuse_mount(dir); FUSE_ARGS(args); - if (!fc->dont_mask) +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(dir, struct fuse_mkdir_in, + fuse_mkdir_initialize, fuse_mkdir_backing, + fuse_mkdir_finalize, + dir, entry, mode); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + + if (!fm->fc->dont_mask) mode &= ~current_umask(); memset(&inarg, 0, sizeof(inarg)); @@ -718,23 +997,41 @@ static int fuse_mkdir(struct inode *dir, struct dentry *entry, umode_t mode) args.in_args[0].value = &inarg; args.in_args[1].size = entry->d_name.len + 1; args.in_args[1].value = entry->d_name.name; - return create_new_entry(fc, &args, dir, entry, S_IFDIR); + return create_new_entry(fm, &args, dir, entry, S_IFDIR); } static int fuse_symlink(struct inode *dir, struct dentry *entry, const char *link) { - struct fuse_conn *fc = get_fuse_conn(dir); + struct fuse_mount *fm = get_fuse_mount(dir); unsigned len = strlen(link) + 1; FUSE_ARGS(args); +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(dir, struct fuse_dummy_io, + fuse_symlink_initialize, fuse_symlink_backing, + fuse_symlink_finalize, + dir, entry, link, len); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + args.opcode = FUSE_SYMLINK; args.in_numargs = 2; args.in_args[0].size = entry->d_name.len + 1; args.in_args[0].value = entry->d_name.name; args.in_args[1].size = len; args.in_args[1].value = link; - return create_new_entry(fc, &args, dir, entry, S_IFLNK); + return create_new_entry(fm, &args, dir, entry, S_IFLNK); +} + +void fuse_flush_time_update(struct inode *inode) +{ + int err = sync_inode_metadata(inode, 1); + + mapping_set_error(inode->i_mapping, err); } void fuse_update_ctime(struct inode *inode) @@ -742,30 +1039,45 @@ void fuse_update_ctime(struct inode *inode) if (!IS_NOCMTIME(inode)) { inode->i_ctime = current_time(inode); mark_inode_dirty_sync(inode); + fuse_flush_time_update(inode); } } static int fuse_unlink(struct inode *dir, struct dentry *entry) { int err; - struct fuse_conn *fc = get_fuse_conn(dir); + struct fuse_mount *fm = get_fuse_mount(dir); FUSE_ARGS(args); if (fuse_is_bad(dir)) return -EIO; +#ifdef CONFIG_FUSE_BPF + { + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(dir, struct fuse_dummy_io, + fuse_unlink_initialize, + fuse_unlink_backing, + fuse_unlink_finalize, + dir, entry); + if (fer.ret) + return PTR_ERR(fer.result); + } +#endif + args.opcode = FUSE_UNLINK; args.nodeid = get_node_id(dir); args.in_numargs = 1; args.in_args[0].size = entry->d_name.len + 1; args.in_args[0].value = entry->d_name.name; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (!err) { struct inode *inode = d_inode(entry); struct fuse_inode *fi = get_fuse_inode(inode); spin_lock(&fi->lock); - fi->attr_version = atomic64_inc_return(&fc->attr_version); + fi->attr_version = atomic64_inc_return(&fm->fc->attr_version); /* * If i_nlink == 0 then unlink doesn't make sense, yet this can * happen if userspace filesystem is careless. It would be @@ -787,18 +1099,32 @@ static int fuse_unlink(struct inode *dir, struct dentry *entry) static int fuse_rmdir(struct inode *dir, struct dentry *entry) { int err; - struct fuse_conn *fc = get_fuse_conn(dir); + struct fuse_mount *fm = get_fuse_mount(dir); FUSE_ARGS(args); if (fuse_is_bad(dir)) return -EIO; +#ifdef CONFIG_FUSE_BPF + { + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(dir, struct fuse_dummy_io, + fuse_rmdir_initialize, + fuse_rmdir_backing, + fuse_rmdir_finalize, + dir, entry); + if (fer.ret) + return PTR_ERR(fer.result); + } +#endif + args.opcode = FUSE_RMDIR; args.nodeid = get_node_id(dir); args.in_numargs = 1; args.in_args[0].size = entry->d_name.len + 1; args.in_args[0].value = entry->d_name.name; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (!err) { clear_nlink(d_inode(entry)); fuse_dir_changed(dir); @@ -814,7 +1140,7 @@ static int fuse_rename_common(struct inode *olddir, struct dentry *oldent, { int err; struct fuse_rename2_in inarg; - struct fuse_conn *fc = get_fuse_conn(olddir); + struct fuse_mount *fm = get_fuse_mount(olddir); FUSE_ARGS(args); memset(&inarg, 0, argsize); @@ -829,7 +1155,7 @@ static int fuse_rename_common(struct inode *olddir, struct dentry *oldent, args.in_args[1].value = oldent->d_name.name; args.in_args[2].size = newent->d_name.len + 1; args.in_args[2].value = newent->d_name.name; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (!err) { /* ctime changes */ fuse_invalidate_attr(d_inode(oldent)); @@ -874,10 +1200,22 @@ static int fuse_rename2(struct inode *olddir, struct dentry *oldent, if (fuse_is_bad(olddir)) return -EIO; - if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE)) + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) return -EINVAL; if (flags) { +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(olddir, struct fuse_rename2_in, + fuse_rename2_initialize, fuse_rename2_backing, + fuse_rename2_finalize, + olddir, oldent, newdir, newent, flags); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + + /* TODO: how should this go with bpfs involved? */ if (fc->no_rename2 || fc->minor < 23) return -EINVAL; @@ -889,6 +1227,17 @@ static int fuse_rename2(struct inode *olddir, struct dentry *oldent, err = -EINVAL; } } else { +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(olddir, struct fuse_rename_in, + fuse_rename_initialize, fuse_rename_backing, + fuse_rename_finalize, + olddir, oldent, newdir, newent); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + err = fuse_rename_common(olddir, oldent, newdir, newent, 0, FUSE_RENAME, sizeof(struct fuse_rename_in)); @@ -903,9 +1252,19 @@ static int fuse_link(struct dentry *entry, struct inode *newdir, int err; struct fuse_link_in inarg; struct inode *inode = d_inode(entry); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_link_in, fuse_link_initialize, + fuse_link_backing, fuse_link_finalize, entry, + newdir, newent); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + memset(&inarg, 0, sizeof(inarg)); inarg.oldnodeid = get_node_id(inode); args.opcode = FUSE_LINK; @@ -914,7 +1273,7 @@ static int fuse_link(struct dentry *entry, struct inode *newdir, args.in_args[0].value = &inarg; args.in_args[1].size = newent->d_name.len + 1; args.in_args[1].value = newent->d_name.name; - err = create_new_entry(fc, &args, newdir, newent, inode->i_mode); + err = create_new_entry(fm, &args, newdir, newent, inode->i_mode); /* Contrary to "normal" filesystems it can happen that link makes two "logical" inodes point to the same "physical" inode. We invalidate the attributes of the old one, so it @@ -925,7 +1284,7 @@ static int fuse_link(struct dentry *entry, struct inode *newdir, struct fuse_inode *fi = get_fuse_inode(inode); spin_lock(&fi->lock); - fi->attr_version = atomic64_inc_return(&fc->attr_version); + fi->attr_version = atomic64_inc_return(&fm->fc->attr_version); if (likely(inode->i_nlink < UINT_MAX)) inc_nlink(inode); spin_unlock(&fi->lock); @@ -937,7 +1296,7 @@ static int fuse_link(struct dentry *entry, struct inode *newdir, return err; } -static void fuse_fillattr(struct inode *inode, struct fuse_attr *attr, +void fuse_fillattr(struct inode *inode, struct fuse_attr *attr, struct kstat *stat) { unsigned int blkbits; @@ -982,11 +1341,11 @@ static int fuse_do_getattr(struct inode *inode, struct kstat *stat, int err; struct fuse_getattr_in inarg; struct fuse_attr_out outarg; - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); u64 attr_version; - attr_version = fuse_get_attr_version(fc); + attr_version = fuse_get_attr_version(fm->fc); memset(&inarg, 0, sizeof(inarg)); memset(&outarg, 0, sizeof(outarg)); @@ -1005,24 +1364,14 @@ static int fuse_do_getattr(struct inode *inode, struct kstat *stat, args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; - err = fuse_simple_request(fc, &args); - if (!err) { - if (fuse_invalid_attr(&outarg.attr) || - (inode->i_mode ^ outarg.attr.mode) & S_IFMT) { - fuse_make_bad(inode); - err = -EIO; - } else { - fuse_change_attributes(inode, &outarg.attr, - attr_timeout(&outarg), - attr_version); - if (stat) - fuse_fillattr(inode, &outarg.attr, stat); - } - } + err = fuse_simple_request(fm, &args); + if (!err) + err = finalize_attr(inode, &outarg, attr_version, stat); return err; } static int fuse_update_get_attr(struct inode *inode, struct file *file, + const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { @@ -1030,6 +1379,17 @@ static int fuse_update_get_attr(struct inode *inode, struct file *file, int err = 0; bool sync; +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_getattr_io, + fuse_getattr_initialize, fuse_getattr_backing, + fuse_getattr_finalize, + path->dentry, stat, request_mask, flags); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + if (flags & AT_STATX_FORCE_SYNC) sync = true; else if (flags & AT_STATX_DONT_SYNC) @@ -1054,11 +1414,11 @@ static int fuse_update_get_attr(struct inode *inode, struct file *file, int fuse_update_attributes(struct inode *inode, struct file *file) { /* Do *not* need to get atime for internal purposes */ - return fuse_update_get_attr(inode, file, NULL, + return fuse_update_get_attr(inode, file, &file->f_path, NULL, STATX_BASIC_STATS & ~STATX_ATIME, 0); } -int fuse_reverse_inval_entry(struct super_block *sb, u64 parent_nodeid, +int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid, u64 child_nodeid, struct qstr *name) { int err = -ENOTDIR; @@ -1066,7 +1426,7 @@ int fuse_reverse_inval_entry(struct super_block *sb, u64 parent_nodeid, struct dentry *dir; struct dentry *entry; - parent = ilookup5(sb, parent_nodeid, fuse_inode_eq, &parent_nodeid); + parent = fuse_ilookup(fc, parent_nodeid, NULL); if (!parent) return -ENOENT; @@ -1158,14 +1518,24 @@ int fuse_allow_current_process(struct fuse_conn *fc) static int fuse_access(struct inode *inode, int mask) { - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); struct fuse_access_in inarg; int err; +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_access_in, + fuse_access_initialize, fuse_access_backing, + fuse_access_finalize, inode, mask); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + BUG_ON(mask & MAY_NOT_BLOCK); - if (fc->no_access) + if (fm->fc->no_access) return 0; memset(&inarg, 0, sizeof(inarg)); @@ -1175,9 +1545,9 @@ static int fuse_access(struct inode *inode, int mask) args.in_numargs = 1; args.in_args[0].size = sizeof(inarg); args.in_args[0].value = &inarg; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err == -ENOSYS) { - fc->no_access = 1; + fm->fc->no_access = 1; err = 0; } return err; @@ -1210,6 +1580,10 @@ static int fuse_permission(struct inode *inode, int mask) struct fuse_conn *fc = get_fuse_conn(inode); bool refreshed = false; int err = 0; + struct fuse_inode *fi = get_fuse_inode(inode); +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; +#endif if (fuse_is_bad(inode)) return -EIO; @@ -1217,12 +1591,19 @@ static int fuse_permission(struct inode *inode, int mask) if (!fuse_allow_current_process(fc)) return -EACCES; +#ifdef CONFIG_FUSE_BPF + fer = fuse_bpf_backing(inode, struct fuse_access_in, + fuse_access_initialize, fuse_access_backing, + fuse_access_finalize, inode, mask); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + /* * If attributes are needed, refresh them before proceeding */ if (fc->default_permissions || ((mask & MAY_EXEC) && S_ISREG(inode->i_mode))) { - struct fuse_inode *fi = get_fuse_inode(inode); u32 perm_mask = STATX_MODE | STATX_UID | STATX_GID; if (perm_mask & READ_ONCE(fi->inval_mask) || @@ -1268,7 +1649,7 @@ static int fuse_permission(struct inode *inode, int mask) static int fuse_readlink_page(struct inode *inode, struct page *page) { - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_page_desc desc = { .length = PAGE_SIZE - 1 }; struct fuse_args_pages ap = { .num_pages = 1, @@ -1285,7 +1666,7 @@ static int fuse_readlink_page(struct inode *inode, struct page *page) ap.args.page_zeroing = true; ap.args.out_numargs = 1; ap.args.out_args[0].size = desc.length; - res = fuse_simple_request(fc, &ap.args); + res = fuse_simple_request(fm, &ap.args); fuse_invalidate_atime(inode); @@ -1312,6 +1693,21 @@ static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, if (fuse_is_bad(inode)) goto out_err; +#ifdef CONFIG_FUSE_BPF + { + struct fuse_err_ret fer; + const char *out = NULL; + + fer = fuse_bpf_backing(inode, struct fuse_dummy_io, + fuse_get_link_initialize, + fuse_get_link_backing, + fuse_get_link_finalize, + inode, dentry, callback, &out); + if (fer.ret) + return fer.result ?: out; + } +#endif + if (fc->cache_symlinks) return page_get_link_raw(dentry, inode, callback); @@ -1346,7 +1742,6 @@ static int fuse_dir_open(struct inode *inode, struct file *file) static int fuse_dir_release(struct inode *inode, struct file *file) { fuse_release_common(file, true); - return 0; } @@ -1360,6 +1755,19 @@ static int fuse_dir_fsync(struct file *file, loff_t start, loff_t end, if (fuse_is_bad(inode)) return -EIO; +#ifdef CONFIG_FUSE_BPF + { + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_fsync_in, + fuse_dir_fsync_initialize, fuse_fsync_backing, + fuse_fsync_finalize, + file, start, end, datasync); + if (fer.ret) + return PTR_ERR(fer.result); + } +#endif + if (fc->no_fsyncdir) return 0; @@ -1398,58 +1806,6 @@ static long fuse_dir_compat_ioctl(struct file *file, unsigned int cmd, FUSE_IOCTL_COMPAT | FUSE_IOCTL_DIR); } -static bool update_mtime(unsigned ivalid, bool trust_local_mtime) -{ - /* Always update if mtime is explicitly set */ - if (ivalid & ATTR_MTIME_SET) - return true; - - /* Or if kernel i_mtime is the official one */ - if (trust_local_mtime) - return true; - - /* If it's an open(O_TRUNC) or an ftruncate(), don't update */ - if ((ivalid & ATTR_SIZE) && (ivalid & (ATTR_OPEN | ATTR_FILE))) - return false; - - /* In all other cases update */ - return true; -} - -static void iattr_to_fattr(struct fuse_conn *fc, struct iattr *iattr, - struct fuse_setattr_in *arg, bool trust_local_cmtime) -{ - unsigned ivalid = iattr->ia_valid; - - if (ivalid & ATTR_MODE) - arg->valid |= FATTR_MODE, arg->mode = iattr->ia_mode; - if (ivalid & ATTR_UID) - arg->valid |= FATTR_UID, arg->uid = from_kuid(fc->user_ns, iattr->ia_uid); - if (ivalid & ATTR_GID) - arg->valid |= FATTR_GID, arg->gid = from_kgid(fc->user_ns, iattr->ia_gid); - if (ivalid & ATTR_SIZE) - arg->valid |= FATTR_SIZE, arg->size = iattr->ia_size; - if (ivalid & ATTR_ATIME) { - arg->valid |= FATTR_ATIME; - arg->atime = iattr->ia_atime.tv_sec; - arg->atimensec = iattr->ia_atime.tv_nsec; - if (!(ivalid & ATTR_ATIME_SET)) - arg->valid |= FATTR_ATIME_NOW; - } - if ((ivalid & ATTR_MTIME) && update_mtime(ivalid, trust_local_cmtime)) { - arg->valid |= FATTR_MTIME; - arg->mtime = iattr->ia_mtime.tv_sec; - arg->mtimensec = iattr->ia_mtime.tv_nsec; - if (!(ivalid & ATTR_MTIME_SET) && !trust_local_cmtime) - arg->valid |= FATTR_MTIME_NOW; - } - if ((ivalid & ATTR_CTIME) && trust_local_cmtime) { - arg->valid |= FATTR_CTIME; - arg->ctime = iattr->ia_ctime.tv_sec; - arg->ctimensec = iattr->ia_ctime.tv_nsec; - } -} - /* * Prevent concurrent writepages on inode * @@ -1513,7 +1869,7 @@ static void fuse_setattr_fill(struct fuse_conn *fc, struct fuse_args *args, */ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) { - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); struct fuse_setattr_in inarg; struct fuse_attr_out outarg; @@ -1524,7 +1880,7 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) inarg.valid = FATTR_MTIME; inarg.mtime = inode->i_mtime.tv_sec; inarg.mtimensec = inode->i_mtime.tv_nsec; - if (fc->minor >= 23) { + if (fm->fc->minor >= 23) { inarg.valid |= FATTR_CTIME; inarg.ctime = inode->i_ctime.tv_sec; inarg.ctimensec = inode->i_ctime.tv_nsec; @@ -1533,9 +1889,9 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) inarg.valid |= FATTR_FH; inarg.fh = ff->fh; } - fuse_setattr_fill(fc, &args, inode, &inarg, &outarg); + fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); - return fuse_simple_request(fc, &args); + return fuse_simple_request(fm, &args); } /* @@ -1550,7 +1906,8 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, struct file *file) { struct inode *inode = d_inode(dentry); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_conn *fc = fm->fc; struct fuse_inode *fi = get_fuse_inode(inode); FUSE_ARGS(args); struct fuse_setattr_in inarg; @@ -1560,6 +1917,17 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, loff_t oldsize; int err; bool trust_local_cmtime = is_wb && S_ISREG(inode->i_mode); + bool fault_blocked = false; + +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_setattr_io, + fuse_setattr_initialize, fuse_setattr_backing, + fuse_setattr_finalize, dentry, attr, file); + if (fer.ret) + return PTR_ERR(fer.result); +#endif if (!fc->default_permissions) attr->ia_valid |= ATTR_FORCE; @@ -1568,6 +1936,22 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, if (err) return err; + if (attr->ia_valid & ATTR_SIZE) { + if (WARN_ON(!S_ISREG(inode->i_mode))) + return -EIO; + is_truncate = true; + } + + if (FUSE_IS_DAX(inode) && is_truncate) { + down_write(&fi->i_mmap_sem); + fault_blocked = true; + err = fuse_dax_break_layouts(inode, 0, 0); + if (err) { + up_write(&fi->i_mmap_sem); + return err; + } + } + if (attr->ia_valid & ATTR_OPEN) { /* This is coming from open(..., ... | O_TRUNC); */ WARN_ON(!(attr->ia_valid & ATTR_SIZE)); @@ -1580,17 +1964,11 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, */ i_size_write(inode, 0); truncate_pagecache(inode, 0); - return 0; + goto out; } file = NULL; } - if (attr->ia_valid & ATTR_SIZE) { - if (WARN_ON(!S_ISREG(inode->i_mode))) - return -EIO; - is_truncate = true; - } - /* Flush dirty data/metadata before non-truncate SETATTR */ if (is_wb && S_ISREG(inode->i_mode) && attr->ia_valid & @@ -1625,7 +2003,7 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, inarg.lock_owner = fuse_lock_owner_id(fc, current->files); } fuse_setattr_fill(fc, &args, inode, &inarg, &outarg); - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err) { if (err == -EINTR) fuse_invalidate_attr(inode); @@ -1633,7 +2011,7 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, } if (fuse_invalid_attr(&outarg.attr) || - (inode->i_mode ^ outarg.attr.mode) & S_IFMT) { + inode_wrong_type(inode, outarg.attr.mode)) { fuse_make_bad(inode); err = -EIO; goto error; @@ -1673,6 +2051,10 @@ int fuse_do_setattr(struct dentry *dentry, struct iattr *attr, } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); +out: + if (fault_blocked) + up_write(&fi->i_mmap_sem); + return 0; error: @@ -1680,6 +2062,9 @@ error: fuse_release_nowrite(inode); clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + + if (fault_blocked) + up_write(&fi->i_mmap_sem); return err; } @@ -1707,11 +2092,22 @@ static int fuse_setattr(struct dentry *entry, struct iattr *attr) * This should be done on write(), truncate() and chown(). */ if (!fc->handle_killpriv) { +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + /* * ia_mode calculation may have used stale i_mode. * Refresh and recalculate. */ - ret = fuse_do_getattr(inode, NULL, file); + fer = fuse_bpf_backing(inode, struct fuse_getattr_io, + fuse_getattr_initialize, fuse_getattr_backing, + fuse_getattr_finalize, + entry, NULL, 0, 0); + if (fer.ret) + ret = PTR_ERR(fer.result); + else +#endif + ret = fuse_do_getattr(inode, NULL, file); if (ret) return ret; @@ -1754,10 +2150,21 @@ static int fuse_getattr(const struct path *path, struct kstat *stat, if (fuse_is_bad(inode)) return -EIO; - if (!fuse_allow_current_process(fc)) + if (!fuse_allow_current_process(fc)) { + if (!request_mask) { + /* + * If user explicitly requested *nothing* then don't + * error out, but return st_dev only. + */ + stat->result_mask = 0; + stat->dev = inode->i_sb->s_dev; + return 0; + } return -EACCES; + } - return fuse_update_get_attr(inode, NULL, stat, request_mask, flags); + return fuse_update_get_attr(inode, NULL, path, stat, request_mask, + flags); } static const struct inode_operations fuse_dir_inode_operations = { diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 8a715e80c486..cb34aa80c4f8 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -8,6 +8,7 @@ #include "fuse_i.h" +#include #include #include #include @@ -32,7 +33,7 @@ static struct page **fuse_pages_alloc(unsigned int npages, gfp_t flags, return pages; } -static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file, +static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, struct file *file, int opcode, struct fuse_open_out *outargp) { struct fuse_open_in inarg; @@ -40,7 +41,7 @@ static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file, memset(&inarg, 0, sizeof(inarg)); inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY); - if (!fc->atomic_o_trunc) + if (!fm->fc->atomic_o_trunc) inarg.flags &= ~O_TRUNC; args.opcode = opcode; args.nodeid = nodeid; @@ -51,7 +52,7 @@ static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file, args.out_args[0].size = sizeof(*outargp); args.out_args[0].value = outargp; - return fuse_simple_request(fc, &args); + return fuse_simple_request(fm, &args); } struct fuse_release_args { @@ -60,7 +61,7 @@ struct fuse_release_args { struct inode *inode; }; -struct fuse_file *fuse_file_alloc(struct fuse_conn *fc) +struct fuse_file *fuse_file_alloc(struct fuse_mount *fm) { struct fuse_file *ff; @@ -68,7 +69,7 @@ struct fuse_file *fuse_file_alloc(struct fuse_conn *fc) if (unlikely(!ff)) return NULL; - ff->fc = fc; + ff->fm = fm; ff->release_args = kzalloc(sizeof(*ff->release_args), GFP_KERNEL_ACCOUNT); if (!ff->release_args) { @@ -82,7 +83,7 @@ struct fuse_file *fuse_file_alloc(struct fuse_conn *fc) RB_CLEAR_NODE(&ff->polled_node); init_waitqueue_head(&ff->poll_wait); - ff->kh = atomic64_inc_return(&fc->khctr); + ff->kh = atomic64_inc_return(&fm->fc->khctr); return ff; } @@ -100,7 +101,7 @@ static struct fuse_file *fuse_file_get(struct fuse_file *ff) return ff; } -static void fuse_release_end(struct fuse_conn *fc, struct fuse_args *args, +static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args, int error) { struct fuse_release_args *ra = container_of(args, typeof(*ra), args); @@ -109,34 +110,53 @@ static void fuse_release_end(struct fuse_conn *fc, struct fuse_args *args, kfree(ra); } -static void fuse_file_put(struct fuse_file *ff, bool sync, bool isdir) +static void fuse_file_put(struct inode *inode, struct fuse_file *ff, + bool sync, bool isdir) { - if (refcount_dec_and_test(&ff->count)) { - struct fuse_args *args = &ff->release_args->args; + struct fuse_args *args = &ff->release_args->args; +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; +#endif - if (isdir ? ff->fc->no_opendir : ff->fc->no_open) { - /* Do nothing when client does not implement 'open' */ - fuse_release_end(ff->fc, args, 0); - } else if (sync) { - fuse_simple_request(ff->fc, args); - fuse_release_end(ff->fc, args, 0); - } else { - args->end = fuse_release_end; - if (fuse_simple_background(ff->fc, args, - GFP_KERNEL | __GFP_NOFAIL)) - fuse_release_end(ff->fc, args, -ENOTCONN); - } - kfree(ff); + if (!refcount_dec_and_test(&ff->count)) + return; + +#ifdef CONFIG_FUSE_BPF + fer = fuse_bpf_backing(inode, struct fuse_release_in, + fuse_release_initialize, fuse_release_backing, + fuse_release_finalize, + inode, ff); + if (fer.ret) { + fuse_release_end(ff->fm, args, 0); + } else +#endif + if (isdir ? ff->fm->fc->no_opendir : ff->fm->fc->no_open) { + /* Do nothing when client does not implement 'open' */ + fuse_release_end(ff->fm, args, 0); + } else if (sync) { + fuse_simple_request(ff->fm, args); + fuse_release_end(ff->fm, args, 0); + } else { + args->end = fuse_release_end; + if (fuse_simple_background(ff->fm, args, + GFP_KERNEL | __GFP_NOFAIL)) + fuse_release_end(ff->fm, args, -ENOTCONN); } + kfree(ff); } -int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file, +int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, bool isdir) { + struct fuse_conn *fc = fm->fc; struct fuse_file *ff; int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN; - ff = fuse_file_alloc(fc); + if (file->private_data) { + ff = file->private_data; + file->private_data = NULL; + } else + ff = fuse_file_alloc(fm); if (!ff) return -ENOMEM; @@ -147,7 +167,7 @@ int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file, struct fuse_open_out outarg; int err; - err = fuse_send_open(fc, nodeid, file, opcode, &outarg); + err = fuse_send_open(fm, nodeid, file, opcode, &outarg); if (!err) { ff->fh = outarg.fh; ff->open_flags = outarg.open_flags; @@ -205,25 +225,24 @@ void fuse_finish_open(struct inode *inode, struct file *file) fi->attr_version = atomic64_inc_return(&fc->attr_version); i_size_write(inode, 0); spin_unlock(&fi->lock); - truncate_pagecache(inode, 0); fuse_invalidate_attr(inode); if (fc->writeback_cache) file_update_time(file); - } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { - invalidate_inode_pages2(inode->i_mapping); } - if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache) fuse_link_write_file(file); } int fuse_open_common(struct inode *inode, struct file *file, bool isdir) { - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_conn *fc = fm->fc; int err; bool is_wb_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc && fc->writeback_cache; + bool dax_truncate = (file->f_flags & O_TRUNC) && + fc->atomic_o_trunc && FUSE_IS_DAX(inode); if (fuse_is_bad(inode)) return -EIO; @@ -232,20 +251,53 @@ int fuse_open_common(struct inode *inode, struct file *file, bool isdir) if (err) return err; - if (is_wb_truncate) { +#ifdef CONFIG_FUSE_BPF + { + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_open_io, + fuse_open_initialize, + fuse_open_backing, + fuse_open_finalize, + inode, file, isdir); + if (fer.ret) + return PTR_ERR(fer.result); + } +#endif + + if (is_wb_truncate || dax_truncate) inode_lock(inode); - fuse_set_nowrite(inode); + + if (dax_truncate) { + down_write(&get_fuse_inode(inode)->i_mmap_sem); + err = fuse_dax_break_layouts(inode, 0, 0); + if (err) + goto out_inode_unlock; } - err = fuse_do_open(fc, get_node_id(inode), file, isdir); + if (is_wb_truncate || dax_truncate) + fuse_set_nowrite(inode); + err = fuse_do_open(fm, get_node_id(inode), file, isdir); if (!err) fuse_finish_open(inode, file); - if (is_wb_truncate) { + if (is_wb_truncate || dax_truncate) fuse_release_nowrite(inode); - inode_unlock(inode); + if (!err) { + struct fuse_file *ff = file->private_data; + + if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) + truncate_pagecache(inode, 0); + else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + invalidate_inode_pages2(inode->i_mapping); } + if (dax_truncate) + up_write(&get_fuse_inode(inode)->i_mmap_sem); + +out_inode_unlock: + if (is_wb_truncate || dax_truncate) + inode_unlock(inode); return err; } @@ -253,7 +305,7 @@ int fuse_open_common(struct inode *inode, struct file *file, bool isdir) static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, int flags, int opcode) { - struct fuse_conn *fc = ff->fc; + struct fuse_conn *fc = ff->fm->fc; struct fuse_release_args *ra = ff->release_args; /* Inode is NULL on error path of fuse_create_open() */ @@ -293,7 +345,7 @@ void fuse_release_common(struct file *file, bool isdir) if (ff->flock) { ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; - ra->inarg.lock_owner = fuse_lock_owner_id(ff->fc, + ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, (fl_owner_t) file); } /* Hold inode until release is finished */ @@ -307,8 +359,14 @@ void fuse_release_common(struct file *file, bool isdir) * Make the release synchronous if this is a fuseblk mount, * synchronous RELEASE is allowed (and desirable) in this case * because the server can be trusted not to screw up. + * + * Always use the asynchronous file put because the current thread + * might be the fuse server. This can happen if a process starts some + * aio and closes the fd before the aio completes. Since aio takes its + * own ref to the file, the IO completion has to drop the ref, which is + * how the fuse server can end up closing its clients' files. */ - fuse_file_put(ff, ff->fc->destroy, isdir); + fuse_file_put(ra->inode, ff, false, isdir); } static int fuse_open(struct inode *inode, struct file *file) @@ -338,7 +396,7 @@ void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, int flags) * iput(NULL) is a no-op and since the refcount is 1 and everything's * synchronous, we are fine with not doing igrab() here" */ - fuse_file_put(ff, true, false); + fuse_file_put(&fi->inode, ff, true, false); } EXPORT_SYMBOL_GPL(fuse_sync_release); @@ -366,7 +424,7 @@ u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id) struct fuse_writepage_args { struct fuse_io_args ia; - struct list_head writepages_entry; + struct rb_node writepages_entry; struct list_head queue_entry; struct fuse_writepage_args *next; struct inode *inode; @@ -375,17 +433,23 @@ struct fuse_writepage_args { static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi, pgoff_t idx_from, pgoff_t idx_to) { - struct fuse_writepage_args *wpa; + struct rb_node *n; - list_for_each_entry(wpa, &fi->writepages, writepages_entry) { + n = fi->writepages.rb_node; + + while (n) { + struct fuse_writepage_args *wpa; pgoff_t curr_index; + wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry); WARN_ON(get_fuse_inode(wpa->inode) != fi); curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT; - if (idx_from < curr_index + wpa->ia.ap.num_pages && - curr_index <= idx_to) { + if (idx_from >= curr_index + wpa->ia.ap.num_pages) + n = n->rb_right; + else if (idx_to < curr_index) + n = n->rb_left; + else return wpa; - } } return NULL; } @@ -445,18 +509,26 @@ static void fuse_sync_writes(struct inode *inode) static int fuse_flush(struct file *file, fl_owner_t id) { struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_file *ff = file->private_data; struct fuse_flush_in inarg; FUSE_ARGS(args); int err; +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(file->f_inode, struct fuse_flush_in, + fuse_flush_initialize, fuse_flush_backing, + fuse_flush_finalize, + file, id); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + if (fuse_is_bad(inode)) return -EIO; - if (fc->no_flush) - return 0; - err = write_inode_now(inode, 1); if (err) return err; @@ -469,9 +541,13 @@ static int fuse_flush(struct file *file, fl_owner_t id) if (err) return err; + err = 0; + if (fm->fc->no_flush) + goto inval_attr_out; + memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - inarg.lock_owner = fuse_lock_owner_id(fc, id); + inarg.lock_owner = fuse_lock_owner_id(fm->fc, id); args.opcode = FUSE_FLUSH; args.nodeid = get_node_id(inode); args.in_numargs = 1; @@ -479,11 +555,19 @@ static int fuse_flush(struct file *file, fl_owner_t id) args.in_args[0].value = &inarg; args.force = true; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err == -ENOSYS) { - fc->no_flush = 1; + fm->fc->no_flush = 1; err = 0; } + +inval_attr_out: + /* + * In memory i_blocks is not maintained by fuse, if writeback cache is + * enabled, i_blocks from cached attr may not be accurate. + */ + if (!err && fm->fc->writeback_cache) + fuse_invalidate_attr(inode); return err; } @@ -491,7 +575,7 @@ int fuse_fsync_common(struct file *file, loff_t start, loff_t end, int datasync, int opcode) { struct inode *inode = file->f_mapping->host; - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_file *ff = file->private_data; FUSE_ARGS(args); struct fuse_fsync_in inarg; @@ -504,7 +588,7 @@ int fuse_fsync_common(struct file *file, loff_t start, loff_t end, args.in_numargs = 1; args.in_args[0].size = sizeof(inarg); args.in_args[0].value = &inarg; - return fuse_simple_request(fc, &args); + return fuse_simple_request(fm, &args); } static int fuse_fsync(struct file *file, loff_t start, loff_t end, @@ -514,6 +598,17 @@ static int fuse_fsync(struct file *file, loff_t start, loff_t end, struct fuse_conn *fc = get_fuse_conn(inode); int err; +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_fsync_in, + fuse_fsync_initialize, fuse_fsync_backing, + fuse_fsync_finalize, + file, start, end, datasync); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + if (fuse_is_bad(inode)) return -EIO; @@ -679,7 +774,7 @@ static void fuse_io_free(struct fuse_io_args *ia) kfree(ia); } -static void fuse_aio_complete_req(struct fuse_conn *fc, struct fuse_args *args, +static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args, int err) { struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); @@ -708,7 +803,7 @@ static void fuse_aio_complete_req(struct fuse_conn *fc, struct fuse_args *args, fuse_io_free(ia); } -static ssize_t fuse_async_req_send(struct fuse_conn *fc, +static ssize_t fuse_async_req_send(struct fuse_mount *fm, struct fuse_io_args *ia, size_t num_bytes) { ssize_t err; @@ -722,9 +817,9 @@ static ssize_t fuse_async_req_send(struct fuse_conn *fc, ia->ap.args.end = fuse_aio_complete_req; ia->ap.args.may_block = io->should_dirty; - err = fuse_simple_background(fc, &ia->ap.args, GFP_KERNEL); + err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL); if (err) - fuse_aio_complete_req(fc, &ia->ap.args, err); + fuse_aio_complete_req(fm, &ia->ap.args, err); return num_bytes; } @@ -734,18 +829,18 @@ static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count, { struct file *file = ia->io->iocb->ki_filp; struct fuse_file *ff = file->private_data; - struct fuse_conn *fc = ff->fc; + struct fuse_mount *fm = ff->fm; fuse_read_args_fill(ia, file, pos, count, FUSE_READ); if (owner != NULL) { ia->read.in.read_flags |= FUSE_READ_LOCKOWNER; - ia->read.in.lock_owner = fuse_lock_owner_id(fc, owner); + ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner); } if (ia->io->async) - return fuse_async_req_send(fc, ia, count); + return fuse_async_req_send(fm, ia, count); - return fuse_simple_request(fc, &ia->ap.args); + return fuse_simple_request(fm, &ia->ap.args); } static void fuse_read_update_size(struct inode *inode, loff_t size, @@ -755,7 +850,7 @@ static void fuse_read_update_size(struct inode *inode, loff_t size, struct fuse_inode *fi = get_fuse_inode(inode); spin_lock(&fi->lock); - if (attr_ver == fi->attr_version && size < inode->i_size && + if (attr_ver >= fi->attr_version && size < inode->i_size && !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) { fi->attr_version = atomic64_inc_return(&fc->attr_version); i_size_write(inode, size); @@ -791,7 +886,7 @@ static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, static int fuse_do_readpage(struct file *file, struct page *page) { struct inode *inode = page->mapping->host; - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); loff_t pos = page_offset(page); struct fuse_page_desc desc = { .length = PAGE_SIZE }; struct fuse_io_args ia = { @@ -811,14 +906,14 @@ static int fuse_do_readpage(struct file *file, struct page *page) */ fuse_wait_on_page_writeback(inode, page->index); - attr_ver = fuse_get_attr_version(fc); + attr_ver = fuse_get_attr_version(fm->fc); /* Don't overflow end offset */ if (pos + (desc.length - 1) == LLONG_MAX) desc.length--; fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); - res = fuse_simple_request(fc, &ia.ap.args); + res = fuse_simple_request(fm, &ia.ap.args); if (res < 0) return res; /* @@ -848,7 +943,7 @@ static int fuse_readpage(struct file *file, struct page *page) return err; } -static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_args *args, +static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, int err) { int i; @@ -883,8 +978,11 @@ static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_args *args, unlock_page(page); put_page(page); } - if (ia->ff) - fuse_file_put(ia->ff, false, false); + if (ia->ff) { + WARN_ON(!mapping); + fuse_file_put(mapping ? mapping->host : NULL, ia->ff, + false, false); + } fuse_io_free(ia); } @@ -892,7 +990,7 @@ static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_args *args, static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) { struct fuse_file *ff = file->private_data; - struct fuse_conn *fc = ff->fc; + struct fuse_mount *fm = ff->fm; struct fuse_args_pages *ap = &ia->ap; loff_t pos = page_offset(ap->pages[0]); size_t count = ap->num_pages << PAGE_SHIFT; @@ -911,18 +1009,18 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) WARN_ON((loff_t) (pos + count) < 0); fuse_read_args_fill(ia, file, pos, count, FUSE_READ); - ia->read.attr_ver = fuse_get_attr_version(fc); - if (fc->async_read) { + ia->read.attr_ver = fuse_get_attr_version(fm->fc); + if (fm->fc->async_read) { ia->ff = fuse_file_get(ff); ap->args.end = fuse_readpages_end; - err = fuse_simple_background(fc, &ap->args, GFP_KERNEL); + err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); if (!err) return; } else { - res = fuse_simple_request(fc, &ap->args); + res = fuse_simple_request(fm, &ap->args); err = res < 0 ? res : 0; } - fuse_readpages_end(fc, &ap->args, err); + fuse_readpages_end(fm, &ap->args, err); } struct fuse_fill_data { @@ -941,6 +1039,19 @@ static int fuse_readpages_fill(void *_data, struct page *page) struct inode *inode = data->inode; struct fuse_conn *fc = get_fuse_conn(inode); +#ifdef CONFIG_FUSE_BPF + /* + * Currently no meaningful readahead is possible with fuse-bpf within + * the kernel, so unless the daemon is aware of this file, ignore this + * call. + */ + if (!get_fuse_inode(inode)->nodeid) + return -EIO; +#endif + + if (fuse_is_bad(inode)) + return -EIO; + fuse_wait_on_page_writeback(inode, page->index); if (ap->num_pages && @@ -1037,7 +1148,7 @@ static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, args->opcode = FUSE_WRITE; args->nodeid = ff->nodeid; args->in_numargs = 2; - if (ff->fc->minor < 9) + if (ff->fm->fc->minor < 9) args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE; else args->in_args[0].size = sizeof(ia->write.in); @@ -1066,7 +1177,7 @@ static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos, struct kiocb *iocb = ia->io->iocb; struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; - struct fuse_conn *fc = ff->fc; + struct fuse_mount *fm = ff->fm; struct fuse_write_in *inarg = &ia->write.in; ssize_t err; @@ -1074,13 +1185,13 @@ static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos, inarg->flags = fuse_write_flags(iocb); if (owner != NULL) { inarg->write_flags |= FUSE_WRITE_LOCKOWNER; - inarg->lock_owner = fuse_lock_owner_id(fc, owner); + inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner); } if (ia->io->async) - return fuse_async_req_send(fc, ia, count); + return fuse_async_req_send(fm, ia, count); - err = fuse_simple_request(fc, &ia->ap.args); + err = fuse_simple_request(fm, &ia->ap.args); if (!err && ia->write.out.size > count) err = -EIO; @@ -1111,7 +1222,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, struct fuse_args_pages *ap = &ia->ap; struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; - struct fuse_conn *fc = ff->fc; + struct fuse_mount *fm = ff->fm; unsigned int offset, i; bool short_write; int err; @@ -1122,7 +1233,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, fuse_write_args_fill(ia, ff, pos, count); ia->write.in.flags = fuse_write_flags(iocb); - err = fuse_simple_request(fc, &ap->args); + err = fuse_simple_request(fm, &ap->args); if (!err && ia->write.out.size > count) err = -EIO; @@ -1437,9 +1548,9 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, ap->args.user_pages = true; if (write) - ap->args.in_pages = 1; + ap->args.in_pages = true; else - ap->args.out_pages = 1; + ap->args.out_pages = true; *nbytesp = nbytes; @@ -1454,7 +1565,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, struct file *file = io->iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct fuse_file *ff = file->private_data; - struct fuse_conn *fc = ff->fc; + struct fuse_conn *fc = ff->fm->fc; size_t nmax = write ? fc->max_write : fc->max_read; loff_t pos = *ppos; size_t count = iov_iter_count(iter); @@ -1594,10 +1705,28 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); - if (fuse_is_bad(file_inode(file))) + if (fuse_is_bad(inode)) return -EIO; + if (FUSE_IS_DAX(inode)) + return fuse_dax_read_iter(iocb, to); + +#ifdef CONFIG_FUSE_BPF + { + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_file_read_iter_io, + fuse_file_read_iter_initialize, + fuse_file_read_iter_backing, + fuse_file_read_iter_finalize, + iocb, to); + if (fer.ret) + return PTR_ERR(fer.result); + } +#endif + if (ff->passthrough.filp) return fuse_passthrough_read_iter(iocb, to); else if (!(ff->open_flags & FOPEN_DIRECT_IO)) @@ -1610,10 +1739,28 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); - if (fuse_is_bad(file_inode(file))) + if (fuse_is_bad(inode)) return -EIO; + if (FUSE_IS_DAX(inode)) + return fuse_dax_write_iter(iocb, from); + +#ifdef CONFIG_FUSE_BPF + { + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_file_write_iter_io, + fuse_file_write_iter_initialize, + fuse_file_write_iter_backing, + fuse_file_write_iter_finalize, + iocb, from); + if (fer.ret) + return PTR_ERR(fer.result); + } +#endif + if (ff->passthrough.filp) return fuse_passthrough_write_iter(iocb, from); else if (!(ff->open_flags & FOPEN_DIRECT_IO)) @@ -1631,13 +1778,13 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) __free_page(ap->pages[i]); if (wpa->ia.ff) - fuse_file_put(wpa->ia.ff, false, false); + fuse_file_put(wpa->inode, wpa->ia.ff, false, false); kfree(ap->pages); kfree(wpa); } -static void fuse_writepage_finish(struct fuse_conn *fc, +static void fuse_writepage_finish(struct fuse_mount *fm, struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -1646,7 +1793,6 @@ static void fuse_writepage_finish(struct fuse_conn *fc, struct backing_dev_info *bdi = inode_to_bdi(inode); int i; - list_del(&wpa->writepages_entry); for (i = 0; i < ap->num_pages; i++) { dec_wb_stat(&bdi->wb, WB_WRITEBACK); dec_node_page_state(ap->pages[i], NR_WRITEBACK_TEMP); @@ -1656,7 +1802,7 @@ static void fuse_writepage_finish(struct fuse_conn *fc, } /* Called under fi->lock, may release and reacquire it */ -static void fuse_send_writepage(struct fuse_conn *fc, +static void fuse_send_writepage(struct fuse_mount *fm, struct fuse_writepage_args *wpa, loff_t size) __releases(fi->lock) __acquires(fi->lock) @@ -1682,10 +1828,10 @@ __acquires(fi->lock) args->force = true; args->nocreds = true; - err = fuse_simple_background(fc, args, GFP_ATOMIC); + err = fuse_simple_background(fm, args, GFP_ATOMIC); if (err == -ENOMEM) { spin_unlock(&fi->lock); - err = fuse_simple_background(fc, args, GFP_NOFS | __GFP_NOFAIL); + err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL); spin_lock(&fi->lock); } @@ -1697,7 +1843,8 @@ __acquires(fi->lock) out_free: fi->writectr--; - fuse_writepage_finish(fc, wpa); + rb_erase(&wpa->writepages_entry, &fi->writepages); + fuse_writepage_finish(fm, wpa); spin_unlock(&fi->lock); /* After rb_erase() aux request list is private */ @@ -1727,7 +1874,7 @@ void fuse_flush_writepages(struct inode *inode) __releases(fi->lock) __acquires(fi->lock) { - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_inode *fi = get_fuse_inode(inode); loff_t crop = i_size_read(inode); struct fuse_writepage_args *wpa; @@ -1736,29 +1883,76 @@ __acquires(fi->lock) wpa = list_entry(fi->queued_writes.next, struct fuse_writepage_args, queue_entry); list_del_init(&wpa->queue_entry); - fuse_send_writepage(fc, wpa, crop); + fuse_send_writepage(fm, wpa, crop); } } -static void fuse_writepage_end(struct fuse_conn *fc, struct fuse_args *args, +static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, + struct fuse_writepage_args *wpa) +{ + pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT; + pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + + WARN_ON(!wpa->ia.ap.num_pages); + while (*p) { + struct fuse_writepage_args *curr; + pgoff_t curr_index; + + parent = *p; + curr = rb_entry(parent, struct fuse_writepage_args, + writepages_entry); + WARN_ON(curr->inode != wpa->inode); + curr_index = curr->ia.write.in.offset >> PAGE_SHIFT; + + if (idx_from >= curr_index + curr->ia.ap.num_pages) + p = &(*p)->rb_right; + else if (idx_to < curr_index) + p = &(*p)->rb_left; + else + return curr; + } + + rb_link_node(&wpa->writepages_entry, parent, p); + rb_insert_color(&wpa->writepages_entry, root); + return NULL; +} + +static void tree_insert(struct rb_root *root, struct fuse_writepage_args *wpa) +{ + WARN_ON(fuse_insert_writeback(root, wpa)); +} + +static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, int error) { struct fuse_writepage_args *wpa = container_of(args, typeof(*wpa), ia.ap.args); struct inode *inode = wpa->inode; struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); mapping_set_error(inode->i_mapping, error); + /* + * A writeback finished and this might have updated mtime/ctime on + * server making local mtime/ctime stale. Hence invalidate attrs. + * Do this only if writeback_cache is not enabled. If writeback_cache + * is enabled, we trust local ctime/mtime. + */ + if (!fc->writeback_cache) + fuse_invalidate_attr(inode); spin_lock(&fi->lock); + rb_erase(&wpa->writepages_entry, &fi->writepages); while (wpa->next) { - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_write_in *inarg = &wpa->ia.write.in; struct fuse_writepage_args *next = wpa->next; wpa->next = next->next; next->next = NULL; next->ia.ff = fuse_file_get(wpa->ia.ff); - list_add(&next->writepages_entry, &fi->writepages); + tree_insert(&fi->writepages, next); /* * Skip fuse_flush_writepages() to make it easy to crop requests @@ -1783,10 +1977,10 @@ static void fuse_writepage_end(struct fuse_conn *fc, struct fuse_args *args, * no invocations of fuse_writepage_end() while we're in * fuse_set_nowrite..fuse_release_nowrite section. */ - fuse_send_writepage(fc, next, inarg->offset + inarg->size); + fuse_send_writepage(fm, next, inarg->offset + inarg->size); } fi->writectr--; - fuse_writepage_finish(fc, wpa); + fuse_writepage_finish(fm, wpa); spin_unlock(&fi->lock); fuse_writepage_free(wpa); } @@ -1822,10 +2016,33 @@ int fuse_write_inode(struct inode *inode, struct writeback_control *wbc) struct fuse_file *ff; int err; + /** + * TODO - fully understand why this is necessary + * + * With fuse-bpf, fsstress fails if rename is enabled without this + * + * We are getting writes here on directory inodes, which do not have an + * initialized file list so crash. + * + * The question is why we are getting those writes + */ + if (!S_ISREG(inode->i_mode)) + return 0; + /* + * Inode is always written before the last reference is dropped and + * hence this should not be reached from reclaim. + * + * Writing back the inode from reclaim can deadlock if the request + * processing itself needs an allocation. Allocations triggering + * reclaim while serving a request can't be prevented, because it can + * involve any number of unrelated userspace processes. + */ + WARN_ON(wbc->for_reclaim); + ff = __fuse_write_file_get(fc, fi); err = fuse_flush_times(inode, ff); if (ff) - fuse_file_put(ff, false, false); + fuse_file_put(inode, ff, false, false); return err; } @@ -1893,7 +2110,7 @@ static int fuse_writepage_locked(struct page *page) inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP); spin_lock(&fi->lock); - list_add(&wpa->writepages_entry, &fi->writepages); + tree_insert(&fi->writepages, wpa); list_add_tail(&wpa->queue_entry, &fi->queued_writes); fuse_flush_writepages(inode); spin_unlock(&fi->lock); @@ -1988,14 +2205,14 @@ static void fuse_writepages_send(struct fuse_fill_wb_data *data) } /* - * First recheck under fi->lock if the offending offset is still under - * writeback. If yes, then iterate auxiliary write requests, to see if there's + * Check under fi->lock if the page is under writeback, and insert it onto the + * rb_tree if not. Otherwise iterate auxiliary write requests, to see if there's * one already added for a page at this offset. If there's none, then insert * this new request onto the auxiliary list, otherwise reuse the existing one by - * copying the new page contents over to the old temporary page. + * swapping the new temp page with the old one. */ -static bool fuse_writepage_in_flight(struct fuse_writepage_args *new_wpa, - struct page *page) +static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, + struct page *page) { struct fuse_inode *fi = get_fuse_inode(new_wpa->inode); struct fuse_writepage_args *tmp; @@ -2003,17 +2220,15 @@ static bool fuse_writepage_in_flight(struct fuse_writepage_args *new_wpa, struct fuse_args_pages *new_ap = &new_wpa->ia.ap; WARN_ON(new_ap->num_pages != 0); + new_ap->num_pages = 1; spin_lock(&fi->lock); - list_del(&new_wpa->writepages_entry); - old_wpa = fuse_find_writeback(fi, page->index, page->index); + old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa); if (!old_wpa) { - list_add(&new_wpa->writepages_entry, &fi->writepages); spin_unlock(&fi->lock); - return false; + return true; } - new_ap->num_pages = 1; for (tmp = old_wpa->next; tmp; tmp = tmp->next) { pgoff_t curr_index; @@ -2042,7 +2257,41 @@ static bool fuse_writepage_in_flight(struct fuse_writepage_args *new_wpa, fuse_writepage_free(new_wpa); } - return true; + return false; +} + +static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, + struct fuse_args_pages *ap, + struct fuse_fill_wb_data *data) +{ + WARN_ON(!ap->num_pages); + + /* + * Being under writeback is unlikely but possible. For example direct + * read to an mmaped fuse file will set the page dirty twice; once when + * the pages are faulted with get_user_pages(), and then after the read + * completed. + */ + if (fuse_page_is_writeback(data->inode, page->index)) + return true; + + /* Reached max pages */ + if (ap->num_pages == fc->max_pages) + return true; + + /* Reached max write bytes */ + if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write) + return true; + + /* Discontinuity */ + if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index) + return true; + + /* Need to grow the pages array? If so, did the expansion fail? */ + if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data)) + return true; + + return false; } static int fuse_writepages_fill(struct page *page, @@ -2055,7 +2304,6 @@ static int fuse_writepages_fill(struct page *page, struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); struct page *tmp_page; - bool is_writeback; int err; if (!data->ff) { @@ -2065,25 +2313,9 @@ static int fuse_writepages_fill(struct page *page, goto out_unlock; } - /* - * Being under writeback is unlikely but possible. For example direct - * read to an mmaped fuse file will set the page dirty twice; once when - * the pages are faulted with get_user_pages(), and then after the read - * completed. - */ - is_writeback = fuse_page_is_writeback(inode, page->index); - - if (wpa && ap->num_pages && - (is_writeback || ap->num_pages == fc->max_pages || - (ap->num_pages + 1) * PAGE_SIZE > fc->max_write || - data->orig_pages[ap->num_pages - 1]->index + 1 != page->index)) { + if (wpa && fuse_writepage_need_send(fc, page, ap, data)) { fuse_writepages_send(data); data->wpa = NULL; - } else if (wpa && ap->num_pages == data->max_pages) { - if (!fuse_pages_realloc(data)) { - fuse_writepages_send(data); - data->wpa = NULL; - } } err = -ENOMEM; @@ -2121,12 +2353,6 @@ static int fuse_writepages_fill(struct page *page, ap->args.end = fuse_writepage_end; ap->num_pages = 0; wpa->inode = inode; - - spin_lock(&fi->lock); - list_add(&wpa->writepages_entry, &fi->writepages); - spin_unlock(&fi->lock); - - data->wpa = wpa; } set_page_writeback(page); @@ -2134,26 +2360,25 @@ static int fuse_writepages_fill(struct page *page, ap->pages[ap->num_pages] = tmp_page; ap->descs[ap->num_pages].offset = 0; ap->descs[ap->num_pages].length = PAGE_SIZE; + data->orig_pages[ap->num_pages] = page; inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP); err = 0; - if (is_writeback && fuse_writepage_in_flight(wpa, page)) { + if (data->wpa) { + /* + * Protected by fi->lock against concurrent access by + * fuse_page_is_writeback(). + */ + spin_lock(&fi->lock); + ap->num_pages++; + spin_unlock(&fi->lock); + } else if (fuse_writepage_add(wpa, page)) { + data->wpa = wpa; + } else { end_page_writeback(page); - data->wpa = NULL; - goto out_unlock; } - data->orig_pages[ap->num_pages] = page; - - /* - * Protected by fi->lock against concurrent access by - * fuse_page_is_writeback(). - */ - spin_lock(&fi->lock); - ap->num_pages++; - spin_unlock(&fi->lock); - out_unlock: unlock_page(page); @@ -2189,7 +2414,7 @@ static int fuse_writepages(struct address_space *mapping, fuse_writepages_send(&data); } if (data.ff) - fuse_file_put(data.ff, false, false); + fuse_file_put(inode, data.ff, false, false); kfree(data.orig_pages); out: @@ -2336,6 +2561,16 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) { struct fuse_file *ff = file->private_data; + /* DAX mmap is superior to direct_io mmap */ + if (FUSE_IS_DAX(file_inode(file))) + return fuse_dax_mmap(file, vma); + +#ifdef CONFIG_FUSE_BPF + /* TODO - this is simply passthrough, not a proper BPF filter */ + if (ff->backing_file) + return fuse_backing_mmap(file, vma); +#endif + if (ff->passthrough.filp) return fuse_passthrough_mmap(file, vma); @@ -2417,7 +2652,7 @@ static void fuse_lk_fill(struct fuse_args *args, struct file *file, static int fuse_getlk(struct file *file, struct file_lock *fl) { struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); struct fuse_lk_in inarg; struct fuse_lk_out outarg; @@ -2427,9 +2662,9 @@ static int fuse_getlk(struct file *file, struct file_lock *fl) args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (!err) - err = convert_fuse_file_lock(fc, &outarg.lk, fl); + err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl); return err; } @@ -2437,12 +2672,12 @@ static int fuse_getlk(struct file *file, struct file_lock *fl) static int fuse_setlk(struct file *file, struct file_lock *fl, int flock) { struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); struct fuse_lk_in inarg; int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK; struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL; - pid_t pid_nr = pid_nr_ns(pid, fc->pid_ns); + pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns); int err; if (fl->fl_lmops && fl->fl_lmops->lm_grant) { @@ -2455,7 +2690,7 @@ static int fuse_setlk(struct file *file, struct file_lock *fl, int flock) return 0; fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg); - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); /* locking is restartable */ if (err == -EINTR) @@ -2491,12 +2726,18 @@ static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl) { struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_file *ff = file->private_data; int err; +#ifdef CONFIG_FUSE_BPF + /* TODO - this is simply passthrough, not a proper BPF filter */ + if (ff->backing_file) + return fuse_file_flock_backing(file, cmd, fl); +#endif + if (fc->no_flock) { err = locks_lock_file_wait(file, fl); } else { - struct fuse_file *ff = file->private_data; /* emulate flock with POSIX locks */ ff->flock = true; @@ -2509,13 +2750,13 @@ static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl) static sector_t fuse_bmap(struct address_space *mapping, sector_t block) { struct inode *inode = mapping->host; - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); struct fuse_bmap_in inarg; struct fuse_bmap_out outarg; int err; - if (!inode->i_sb->s_bdev || fc->no_bmap) + if (!inode->i_sb->s_bdev || fm->fc->no_bmap) return 0; memset(&inarg, 0, sizeof(inarg)); @@ -2529,9 +2770,9 @@ static sector_t fuse_bmap(struct address_space *mapping, sector_t block) args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err == -ENOSYS) - fc->no_bmap = 1; + fm->fc->no_bmap = 1; return err ? 0 : outarg.block; } @@ -2539,7 +2780,7 @@ static sector_t fuse_bmap(struct address_space *mapping, sector_t block) static loff_t fuse_lseek(struct file *file, loff_t offset, int whence) { struct inode *inode = file->f_mapping->host; - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_file *ff = file->private_data; FUSE_ARGS(args); struct fuse_lseek_in inarg = { @@ -2550,7 +2791,7 @@ static loff_t fuse_lseek(struct file *file, loff_t offset, int whence) struct fuse_lseek_out outarg; int err; - if (fc->no_lseek) + if (fm->fc->no_lseek) goto fallback; args.opcode = FUSE_LSEEK; @@ -2561,10 +2802,10 @@ static loff_t fuse_lseek(struct file *file, loff_t offset, int whence) args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err) { if (err == -ENOSYS) { - fc->no_lseek = 1; + fm->fc->no_lseek = 1; goto fallback; } return err; @@ -2584,6 +2825,17 @@ static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence) { loff_t retval; struct inode *inode = file_inode(file); +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_lseek_io, + fuse_lseek_initialize, + fuse_lseek_backing, + fuse_lseek_finalize, + file, offset, whence); + if (fer.ret) + return PTR_ERR(fer.result); +#endif switch (whence) { case SEEK_SET: @@ -2750,7 +3002,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, unsigned int flags) { struct fuse_file *ff = file->private_data; - struct fuse_conn *fc = ff->fc; + struct fuse_mount *fm = ff->fm; struct fuse_ioctl_in inarg = { .fh = ff->fh, .cmd = cmd, @@ -2783,12 +3035,12 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE); err = -ENOMEM; - ap.pages = fuse_pages_alloc(fc->max_pages, GFP_KERNEL, &ap.descs); + ap.pages = fuse_pages_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs); iov_page = (struct iovec *) __get_free_page(GFP_KERNEL); if (!ap.pages || !iov_page) goto out; - fuse_page_descs_length_init(ap.descs, 0, fc->max_pages); + fuse_page_descs_length_init(ap.descs, 0, fm->fc->max_pages); /* * If restricted, initialize IO parameters as encoded in @cmd. @@ -2833,7 +3085,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, /* make sure there are enough buffer pages and init request with them */ err = -ENOMEM; - if (max_pages > fc->max_pages) + if (max_pages > fm->fc->max_pages) goto out; while (ap.num_pages < max_pages) { ap.pages[ap.num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); @@ -2870,7 +3122,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, ap.args.out_pages = true; ap.args.out_argvar = true; - transferred = fuse_simple_request(fc, &ap.args); + transferred = fuse_simple_request(fm, &ap.args); err = transferred; if (transferred < 0) goto out; @@ -2898,7 +3150,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, goto out; vaddr = kmap_atomic(ap.pages[0]); - err = fuse_copy_ioctl_iovec(fc, iov_page, vaddr, + err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr, transferred, in_iovs + out_iovs, (flags & FUSE_IOCTL_COMPAT) != 0); kunmap_atomic(vaddr); @@ -2908,11 +3160,11 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, in_iov = iov_page; out_iov = in_iov + in_iovs; - err = fuse_verify_ioctl_iov(fc, in_iov, in_iovs); + err = fuse_verify_ioctl_iov(fm->fc, in_iov, in_iovs); if (err) goto out; - err = fuse_verify_ioctl_iov(fc, out_iov, out_iovs); + err = fuse_verify_ioctl_iov(fm->fc, out_iov, out_iovs); if (err) goto out; @@ -2953,6 +3205,15 @@ long fuse_ioctl_common(struct file *file, unsigned int cmd, if (fuse_is_bad(inode)) return -EIO; +#ifdef CONFIG_FUSE_BPF + { + struct fuse_file *ff = file->private_data; + + /* TODO - this is simply passthrough, not a proper BPF filter */ + if (ff->backing_file) + return fuse_backing_ioctl(file, cmd, arg, flags); + } +#endif return fuse_do_ioctl(file, cmd, arg, flags); } @@ -3022,13 +3283,13 @@ static void fuse_register_polled_file(struct fuse_conn *fc, __poll_t fuse_file_poll(struct file *file, poll_table *wait) { struct fuse_file *ff = file->private_data; - struct fuse_conn *fc = ff->fc; + struct fuse_mount *fm = ff->fm; struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh }; struct fuse_poll_out outarg; FUSE_ARGS(args); int err; - if (fc->no_poll) + if (fm->fc->no_poll) return DEFAULT_POLLMASK; poll_wait(file, &ff->poll_wait, wait); @@ -3040,7 +3301,7 @@ __poll_t fuse_file_poll(struct file *file, poll_table *wait) */ if (waitqueue_active(&ff->poll_wait)) { inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY; - fuse_register_polled_file(fc, ff); + fuse_register_polled_file(fm->fc, ff); } args.opcode = FUSE_POLL; @@ -3051,12 +3312,12 @@ __poll_t fuse_file_poll(struct file *file, poll_table *wait) args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (!err) return demangle_poll(outarg.revents); if (err == -ENOSYS) { - fc->no_poll = 1; + fm->fc->no_poll = 1; return DEFAULT_POLLMASK; } return EPOLLERR; @@ -3142,13 +3403,13 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) * By default, we want to optimize all I/Os with async request * submission to the client filesystem if supported. */ - io->async = ff->fc->async_dio; + io->async = ff->fm->fc->async_dio; io->iocb = iocb; io->blocking = is_sync_kiocb(iocb); /* optimization for short read */ if (io->async && !io->write && offset + count > i_size) { - iov_iter_truncate(iter, fuse_round_up(ff->fc, i_size - offset)); + iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset)); shortened = count - iov_iter_count(iter); count -= shortened; } @@ -3218,7 +3479,7 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); struct fuse_inode *fi = get_fuse_inode(inode); - struct fuse_conn *fc = ff->fc; + struct fuse_mount *fm = ff->fm; FUSE_ARGS(args); struct fuse_fallocate_in inarg = { .fh = ff->fh, @@ -3227,13 +3488,36 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, .mode = mode }; int err; + bool block_faults = FUSE_IS_DAX(inode) && + (!(mode & FALLOC_FL_KEEP_SIZE) || + (mode & FALLOC_FL_PUNCH_HOLE)); + +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_fallocate_in, + fuse_file_fallocate_initialize, + fuse_file_fallocate_backing, + fuse_file_fallocate_finalize, + file, mode, offset, length); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) return -EOPNOTSUPP; - if (fc->no_fallocate) + if (fm->fc->no_fallocate) return -EOPNOTSUPP; inode_lock(inode); + if (block_faults) { + down_write(&fi->i_mmap_sem); + err = fuse_dax_break_layouts(inode, 0, 0); + if (err) + goto out; + } + if (mode & FALLOC_FL_PUNCH_HOLE) { loff_t endbyte = offset + length - 1; @@ -3261,9 +3545,9 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, args.in_numargs = 1; args.in_args[0].size = sizeof(inarg); args.in_args[0].value = &inarg; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err == -ENOSYS) { - fc->no_fallocate = 1; + fm->fc->no_fallocate = 1; err = -EOPNOTSUPP; } if (err) @@ -3273,7 +3557,7 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, if (!(mode & FALLOC_FL_KEEP_SIZE)) { bool changed = fuse_write_update_size(inode, offset + length); - if (changed && fc->writeback_cache) + if (changed && fm->fc->writeback_cache) file_update_time(file); } @@ -3286,8 +3570,13 @@ out: if (!(mode & FALLOC_FL_KEEP_SIZE)) clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + if (block_faults) + up_write(&fi->i_mmap_sem); + inode_unlock(inode); + fuse_flush_time_update(inode); + return err; } @@ -3300,7 +3589,8 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, struct inode *inode_in = file_inode(file_in); struct inode *inode_out = file_inode(file_out); struct fuse_inode *fi_out = get_fuse_inode(inode_out); - struct fuse_conn *fc = ff_in->fc; + struct fuse_mount *fm = ff_in->fm; + struct fuse_conn *fc = fm->fc; FUSE_ARGS(args); struct fuse_copy_file_range_in inarg = { .fh_in = ff_in->fh, @@ -3318,6 +3608,18 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, bool is_unstable = (!fc->writeback_cache) && ((pos_out + len) > inode_out->i_size); +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(file_in->f_inode, struct fuse_copy_file_range_io, + fuse_copy_file_range_initialize, + fuse_copy_file_range_backing, + fuse_copy_file_range_finalize, + file_in, pos_in, file_out, pos_out, len, flags); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + if (fc->no_copy_file_range) return -EOPNOTSUPP; @@ -3369,7 +3671,7 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err == -ENOSYS) { fc->no_copy_file_range = 1; err = -EOPNOTSUPP; @@ -3427,6 +3729,7 @@ static const struct file_operations fuse_file_operations = { .release = fuse_release, .fsync = fuse_fsync, .lock = fuse_file_lock, + .get_unmapped_area = thp_get_unmapped_area, .flock = fuse_file_flock, .splice_read = generic_file_splice_read, .splice_write = iter_file_splice_write, @@ -3461,5 +3764,8 @@ void fuse_init_file_inode(struct inode *inode) INIT_LIST_HEAD(&fi->queued_writes); fi->writectr = 0; init_waitqueue_head(&fi->page_waitq); - INIT_LIST_HEAD(&fi->writepages); + fi->writepages = RB_ROOT; + + if (IS_ENABLED(CONFIG_FUSE_DAX)) + fuse_dax_inode_init(inode); } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 2eec57bb3707..97b44bbf5274 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -13,6 +13,8 @@ # define pr_fmt(fmt) "fuse: " fmt #endif +#include +#include #include #include #include @@ -31,6 +33,9 @@ #include #include #include +#include + +#define FUSE_SUPER_MAGIC 0x65735546 /** Default max number of pages that can be used in a single read request */ #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 @@ -63,11 +68,63 @@ struct fuse_forget_link { struct fuse_forget_link *next; }; +/** FUSE specific dentry data */ +#if BITS_PER_LONG < 64 || defined(CONFIG_FUSE_BPF) +struct fuse_dentry { + union { + u64 time; + struct rcu_head rcu; + }; + +#ifdef CONFIG_FUSE_BPF + struct path backing_path; + + /* bpf program *only* set for negative dentries */ + struct bpf_prog *bpf; +#endif +}; + +static inline struct fuse_dentry *get_fuse_dentry(const struct dentry *entry) +{ + return entry->d_fsdata; +} +#endif + +#ifdef CONFIG_FUSE_BPF +static inline void get_fuse_backing_path(const struct dentry *d, + struct path *path) +{ + struct fuse_dentry *di = get_fuse_dentry(d); + + if (!di) { + *path = (struct path) {}; + return; + } + + *path = di->backing_path; + path_get(path); +} +#endif + /** FUSE inode */ struct fuse_inode { /** Inode data */ struct inode inode; +#ifdef CONFIG_FUSE_BPF + /** + * Backing inode, if this inode is from a backing file system. + * If this is set, nodeid is 0. + */ + struct inode *backing_inode; + + /** + * bpf_prog, run on all operations to determine whether to pass through + * or handle in place + */ + struct bpf_prog *bpf; +#endif + /** Unique ID, which identifies the inode between userspace * and kernel */ u64 nodeid; @@ -111,7 +168,7 @@ struct fuse_inode { wait_queue_head_t page_waitq; /* List of writepage requestst (pending or sent) */ - struct list_head writepages; + struct rb_root writepages; }; /* readdir cache (directory only) */ @@ -148,6 +205,20 @@ struct fuse_inode { /** Lock to protect write related fields */ spinlock_t lock; + + /** + * Can't take inode lock in fault path (leads to circular dependency). + * Introduce another semaphore which can be taken in fault path and + * then other filesystem paths can take this to block faults. + */ + struct rw_semaphore i_mmap_sem; + +#ifdef CONFIG_FUSE_DAX + /* + * Dax specific inode data + */ + struct fuse_inode_dax *dax; +#endif }; /** FUSE inode state bits */ @@ -163,6 +234,7 @@ enum { }; struct fuse_conn; +struct fuse_mount; struct fuse_release_args; /** @@ -179,7 +251,7 @@ struct fuse_passthrough { /** FUSE specific file data */ struct fuse_file { /** Fuse connection for this file */ - struct fuse_conn *fc; + struct fuse_mount *fm; /* Argument space reserved for release */ struct fuse_release_args *release_args; @@ -224,6 +296,14 @@ struct fuse_file { /** Container for data related to the passthrough functionality */ struct fuse_passthrough passthrough; +#ifdef CONFIG_FUSE_BPF + /** + * TODO: Reconcile with passthrough file + * backing file when in bpf mode + */ + struct file *backing_file; +#endif + /** RB node to be linked on fuse_conn->polled_files */ struct rb_node polled_node; @@ -255,6 +335,7 @@ struct fuse_page_desc { struct fuse_args { uint64_t nodeid; uint32_t opcode; + uint32_t error_in; unsigned short in_numargs; unsigned short out_numargs; bool force:1; @@ -267,9 +348,9 @@ struct fuse_args { bool page_zeroing:1; bool page_replace:1; bool may_block:1; - struct fuse_in_arg in_args[3]; - struct fuse_arg out_args[2]; - void (*end)(struct fuse_conn *fc, struct fuse_args *args, int error); + struct fuse_in_arg in_args[FUSE_MAX_IN_ARGS]; + struct fuse_arg out_args[FUSE_MAX_OUT_ARGS]; + void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); /* Path used for completing d_canonical_path */ struct path *canonical_path; @@ -376,10 +457,11 @@ struct fuse_req { /** Used to wake up the task waiting for completion of request*/ wait_queue_head_t waitq; -#if IS_ENABLED(CONFIG_VIRTIO_FS) /** virtio-fs's physically contiguous buffer for in and out args */ void *argbuf; -#endif + + /** fuse_mount this request belongs to */ + struct fuse_mount *fm; }; struct fuse_iqueue; @@ -502,10 +584,17 @@ struct fuse_fs_context { bool destroy:1; bool no_control:1; bool no_force_umount:1; - bool no_mount_options:1; + bool legacy_opts_show:1; + bool dax:1; + bool no_daemon:1; unsigned int max_read; unsigned int blksize; const char *subtype; + struct bpf_prog *root_bpf; + struct file *root_dir; + + /* DAX device, may be NULL */ + struct dax_device *dax_dev; /* fuse_dev pointer to fill in, should contain NULL on entry */ void **fudptr; @@ -514,9 +603,9 @@ struct fuse_fs_context { /** * A Fuse connection. * - * This structure is created, when the filesystem is mounted, and is - * destroyed, when the client device is closed and the filesystem is - * unmounted. + * This structure is created, when the root filesystem is mounted, and + * is destroyed, when the client device is closed and the last + * fuse_mount is destroyed. */ struct fuse_conn { /** Lock protecting accessess to members of this structure */ @@ -551,6 +640,9 @@ struct fuse_conn { /** Maxmum number of pages that can be used in a single request */ unsigned int max_pages; + /** Constrain ->max_pages to this value during feature negotiation */ + unsigned int max_pages_limit; + /** Input queue */ struct fuse_iqueue iq; @@ -630,6 +722,9 @@ struct fuse_conn { /** cache READLINK responses in page cache */ unsigned cache_symlinks:1; + /* show legacy mount options */ + unsigned int legacy_opts_show:1; + /* * The following bitfields are only for optimization purposes * and hence races in setting them will not cause malfunction @@ -737,22 +832,25 @@ struct fuse_conn { /** Do not allow MNT_FORCE umount */ unsigned int no_force_umount:1; - /* Do not show mount options */ - unsigned int no_mount_options:1; + /* Auto-mount submounts announced by the server */ + unsigned int auto_submounts:1; /** Passthrough mode for read/write IO */ unsigned int passthrough:1; + /** BPF Only, no Daemon running */ + unsigned int no_daemon:1; + /** The number of requests waiting for completion */ atomic_t num_waiting; /** Negotiated minor version */ unsigned minor; - /** Entry on the fuse_conn_list */ + /** Entry on the fuse_mount_list */ struct list_head entry; - /** Device ID from super block */ + /** Device ID from the root super block */ dev_t dev; /** Dentries in the control filesystem */ @@ -770,15 +868,23 @@ struct fuse_conn { /** Called on final put */ void (*release)(struct fuse_conn *); - /** Super block for this connection. */ - struct super_block *sb; - - /** Read/write semaphore to hold when accessing sb. */ + /** + * Read/write semaphore to hold when accessing the sb of any + * fuse_mount belonging to this connection + */ struct rw_semaphore killsb; /** List of device instances belonging to this connection */ struct list_head devices; +#ifdef CONFIG_FUSE_DAX + /* Dax specific conn data, non-NULL if DAX is enabled */ + struct fuse_conn_dax *dax; +#endif + + /** List of filesystems using this connection */ + struct list_head mounts; + /** IDR for passthrough requests */ struct idr passthrough_req; @@ -786,14 +892,52 @@ struct fuse_conn { spinlock_t passthrough_req_lock; }; -static inline struct fuse_conn *get_fuse_conn_super(struct super_block *sb) +/* + * Represents a mounted filesystem, potentially a submount. + * + * This object allows sharing a fuse_conn between separate mounts to + * allow submounts with dedicated superblocks and thus separate device + * IDs. + */ +struct fuse_mount { + /* Underlying (potentially shared) connection to the FUSE server */ + struct fuse_conn *fc; + + /* Refcount */ + refcount_t count; + + /* + * Super block for this connection (fc->killsb must be held when + * accessing this). + */ + struct super_block *sb; + + /* Entry on fc->mounts */ + struct list_head fc_entry; +}; + +static inline struct fuse_mount *get_fuse_mount_super(struct super_block *sb) { return sb->s_fs_info; } +static inline struct fuse_conn *get_fuse_conn_super(struct super_block *sb) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + + return fm ? fm->fc : NULL; +} + +static inline struct fuse_mount *get_fuse_mount(struct inode *inode) +{ + return get_fuse_mount_super(inode->i_sb); +} + static inline struct fuse_conn *get_fuse_conn(struct inode *inode) { - return get_fuse_conn_super(inode->i_sb); + struct fuse_mount *fm = get_fuse_mount(inode); + + return fm ? fm->fc : NULL; } static inline struct fuse_inode *get_fuse_inode(struct inode *inode) @@ -816,6 +960,13 @@ static inline u64 fuse_get_attr_version(struct fuse_conn *fc) return atomic64_read(&fc->attr_version); } +static inline bool fuse_stale_inode(const struct inode *inode, int generation, + struct fuse_attr *attr) +{ + return inode->i_generation != generation || + inode_wrong_type(inode, attr->mode); +} + static inline void fuse_make_bad(struct inode *inode) { set_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state); @@ -833,19 +984,18 @@ extern const struct dentry_operations fuse_dentry_operations; extern const struct dentry_operations fuse_root_dentry_operations; /** - * Inode to nodeid comparison. - */ -int fuse_inode_eq(struct inode *inode, void *_nodeidp); - -/** - * Get a filled in inode + * Get a filled-in inode */ +struct inode *fuse_iget_backing(struct super_block *sb, + u64 nodeid, + struct inode *backing_inode); struct inode *fuse_iget(struct super_block *sb, u64 nodeid, int generation, struct fuse_attr *attr, u64 attr_valid, u64 attr_version); int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name, - struct fuse_entry_out *outarg, struct inode **inode); + struct fuse_entry_out *outarg, + struct dentry *entry, struct inode **inode); /** * Send FORGET command @@ -882,13 +1032,12 @@ struct fuse_io_args { void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, size_t count, int opcode); - /** * Send OPEN or OPENDIR request */ int fuse_open_common(struct inode *inode, struct file *file, bool isdir); -struct fuse_file *fuse_file_alloc(struct fuse_conn *fc); +struct fuse_file *fuse_file_alloc(struct fuse_mount *fm); void fuse_file_free(struct fuse_file *ff); void fuse_finish_open(struct inode *inode, struct file *file); @@ -951,19 +1100,19 @@ int fuse_dev_init(void); void fuse_dev_cleanup(void); int fuse_ctl_init(void); -void __exit fuse_ctl_cleanup(void); +void fuse_ctl_cleanup(void); /** * Simple request sending that does request allocation and freeing */ -ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args); -int fuse_simple_background(struct fuse_conn *fc, struct fuse_args *args, +ssize_t fuse_simple_request(struct fuse_mount *fm, struct fuse_args *args); +int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, gfp_t gfp_flags); /** * End a finished request */ -void fuse_request_end(struct fuse_conn *fc, struct fuse_req *req); +void fuse_request_end(struct fuse_req *req); /* Abort all requests */ void fuse_abort_conn(struct fuse_conn *fc); @@ -979,6 +1128,7 @@ void fuse_invalidate_entry_cache(struct dentry *entry); void fuse_invalidate_atime(struct inode *inode); u64 entry_attr_timeout(struct fuse_entry_out *o); +void fuse_init_dentry_root(struct dentry *root, struct file *backing_dir); void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o); /** @@ -989,7 +1139,8 @@ struct fuse_conn *fuse_conn_get(struct fuse_conn *fc); /** * Initialize fuse_conn */ -void fuse_conn_init(struct fuse_conn *fc, struct user_namespace *user_ns, +void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, + struct user_namespace *user_ns, const struct fuse_iqueue_ops *fiq_ops, void *fiq_priv); /** @@ -997,11 +1148,21 @@ void fuse_conn_init(struct fuse_conn *fc, struct user_namespace *user_ns, */ void fuse_conn_put(struct fuse_conn *fc); +/** + * Acquire reference to fuse_mount + */ +struct fuse_mount *fuse_mount_get(struct fuse_mount *fm); + +/** + * Release reference to fuse_mount + */ +void fuse_mount_put(struct fuse_mount *fm); + struct fuse_dev *fuse_dev_alloc_install(struct fuse_conn *fc); struct fuse_dev *fuse_dev_alloc(void); void fuse_dev_install(struct fuse_dev *fud, struct fuse_conn *fc); void fuse_dev_free(struct fuse_dev *fud); -void fuse_send_init(struct fuse_conn *fc); +void fuse_send_init(struct fuse_mount *fm); /** * Fill in superblock and initialize fuse connection @@ -1010,12 +1171,26 @@ void fuse_send_init(struct fuse_conn *fc); */ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx); -/** - * Disassociate fuse connection from superblock and kill the superblock - * - * Calls kill_anon_super(), do not use with bdev mounts. +/* + * Fill in superblock for submounts + * @sb: partially-initialized superblock to fill in + * @parent_fi: The fuse_inode of the parent filesystem where this submount is + * mounted */ -void fuse_kill_sb_anon(struct super_block *sb); +int fuse_fill_super_submount(struct super_block *sb, + struct fuse_inode *parent_fi); + +/* + * Remove the mount from the connection + * + * Returns whether this was the last mount + */ +bool fuse_mount_remove(struct fuse_mount *fm); + +/* + * Shut down the connection (possibly sending DESTROY request). + */ +void fuse_conn_destroy(struct fuse_mount *fm); /** * Add connection to control filesystem @@ -1041,6 +1216,7 @@ int fuse_allow_current_process(struct fuse_conn *fc); u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id); +void fuse_flush_time_update(struct inode *inode); void fuse_update_ctime(struct inode *inode); int fuse_update_attributes(struct inode *inode, struct file *file); @@ -1050,10 +1226,20 @@ void fuse_flush_writepages(struct inode *inode); void fuse_set_nowrite(struct inode *inode); void fuse_release_nowrite(struct inode *inode); +/** + * Scan all fuse_mounts belonging to fc to find the first where + * ilookup5() returns a result. Return that result and the + * respective fuse_mount in *fm (unless fm is NULL). + * + * The caller must hold fc->killsb. + */ +struct inode *fuse_ilookup(struct fuse_conn *fc, u64 nodeid, + struct fuse_mount **fm); + /** * File-system tells the kernel to invalidate cache for the given node id. */ -int fuse_reverse_inval_inode(struct super_block *sb, u64 nodeid, +int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len); /** @@ -1066,10 +1252,10 @@ int fuse_reverse_inval_inode(struct super_block *sb, u64 nodeid, * - is a file or oan empty directory * then the dentry is unhashed (d_delete()). */ -int fuse_reverse_inval_entry(struct super_block *sb, u64 parent_nodeid, +int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid, u64 child_nodeid, struct qstr *name); -int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file, +int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, bool isdir); /** @@ -1133,7 +1319,24 @@ unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args); u64 fuse_get_unique(struct fuse_iqueue *fiq); void fuse_free_conn(struct fuse_conn *fc); +/* dax.c */ + +#define FUSE_IS_DAX(inode) (IS_ENABLED(CONFIG_FUSE_DAX) && IS_DAX(inode)) + +ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to); +ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from); +int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma); +int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start, u64 dmap_end); +int fuse_dax_conn_alloc(struct fuse_conn *fc, struct dax_device *dax_dev); +void fuse_dax_conn_free(struct fuse_conn *fc); +bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi); +void fuse_dax_inode_init(struct inode *inode); +void fuse_dax_inode_cleanup(struct inode *inode); +bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment); +void fuse_dax_cancel_work(struct fuse_conn *fc); + /* passthrough.c */ +void fuse_copyattr(struct file *dst_file, struct file *src_file); int fuse_passthrough_open(struct fuse_dev *fud, u32 lower_fd); int fuse_passthrough_setup(struct fuse_conn *fc, struct fuse_file *ff, struct fuse_open_out *openarg); @@ -1142,4 +1345,648 @@ ssize_t fuse_passthrough_read_iter(struct kiocb *iocb, struct iov_iter *to); ssize_t fuse_passthrough_write_iter(struct kiocb *iocb, struct iov_iter *from); ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma); +/* backing.c */ + +struct bpf_prog *fuse_get_bpf_prog(struct file *file); + +/* + * Dummy io passed to fuse_bpf_backing when io operation needs no scratch space + */ +struct fuse_dummy_io { + int unused; +}; + +struct fuse_open_io { + struct fuse_open_in foi; + struct fuse_open_out foo; +}; + +int fuse_open_initialize(struct fuse_bpf_args *fa, struct fuse_open_io *foi, + struct inode *inode, struct file *file, bool isdir); +int fuse_open_backing(struct fuse_bpf_args *fa, + struct inode *inode, struct file *file, bool isdir); +void *fuse_open_finalize(struct fuse_bpf_args *fa, + struct inode *inode, struct file *file, bool isdir); + +struct fuse_create_open_io { + struct fuse_create_in fci; + struct fuse_entry_out feo; + struct fuse_open_out foo; +}; + +int fuse_create_open_initialize( + struct fuse_bpf_args *fa, struct fuse_create_open_io *fcoi, + struct inode *dir, struct dentry *entry, + struct file *file, unsigned int flags, umode_t mode); +int fuse_create_open_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, + struct file *file, unsigned int flags, umode_t mode); +void *fuse_create_open_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, + struct file *file, unsigned int flags, umode_t mode); + +int fuse_mknod_initialize( + struct fuse_bpf_args *fa, struct fuse_mknod_in *fmi, + struct inode *dir, struct dentry *entry, umode_t mode, dev_t rdev); +int fuse_mknod_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, umode_t mode, dev_t rdev); +void *fuse_mknod_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, umode_t mode, dev_t rdev); + +int fuse_mkdir_initialize( + struct fuse_bpf_args *fa, struct fuse_mkdir_in *fmi, + struct inode *dir, struct dentry *entry, umode_t mode); +int fuse_mkdir_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, umode_t mode); +void *fuse_mkdir_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, umode_t mode); + +int fuse_rmdir_initialize( + struct fuse_bpf_args *fa, struct fuse_dummy_io *fmi, + struct inode *dir, struct dentry *entry); +int fuse_rmdir_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry); +void *fuse_rmdir_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry); + +int fuse_rename2_initialize(struct fuse_bpf_args *fa, struct fuse_rename2_in *fri, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent, + unsigned int flags); +int fuse_rename2_backing(struct fuse_bpf_args *fa, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent, + unsigned int flags); +void *fuse_rename2_finalize(struct fuse_bpf_args *fa, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent, + unsigned int flags); + +int fuse_rename_initialize(struct fuse_bpf_args *fa, struct fuse_rename_in *fri, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent); +int fuse_rename_backing(struct fuse_bpf_args *fa, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent); +void *fuse_rename_finalize(struct fuse_bpf_args *fa, + struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent); + +int fuse_unlink_initialize( + struct fuse_bpf_args *fa, struct fuse_dummy_io *fmi, + struct inode *dir, struct dentry *entry); +int fuse_unlink_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry); +void *fuse_unlink_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry); + +int fuse_link_initialize(struct fuse_bpf_args *fa, struct fuse_link_in *fli, + struct dentry *entry, struct inode *dir, + struct dentry *newent); +int fuse_link_backing(struct fuse_bpf_args *fa, struct dentry *entry, + struct inode *dir, struct dentry *newent); +void *fuse_link_finalize(struct fuse_bpf_args *fa, struct dentry *entry, + struct inode *dir, struct dentry *newent); + +int fuse_release_initialize(struct fuse_bpf_args *fa, struct fuse_release_in *fri, + struct inode *inode, struct fuse_file *ff); +int fuse_release_backing(struct fuse_bpf_args *fa, + struct inode *inode, struct fuse_file *ff); +void *fuse_release_finalize(struct fuse_bpf_args *fa, + struct inode *inode, struct fuse_file *ff); + +int fuse_flush_initialize(struct fuse_bpf_args *fa, struct fuse_flush_in *ffi, + struct file *file, fl_owner_t id); +int fuse_flush_backing(struct fuse_bpf_args *fa, struct file *file, fl_owner_t id); +void *fuse_flush_finalize(struct fuse_bpf_args *fa, + struct file *file, fl_owner_t id); + +struct fuse_lseek_io { + struct fuse_lseek_in fli; + struct fuse_lseek_out flo; +}; + +int fuse_lseek_initialize(struct fuse_bpf_args *fa, struct fuse_lseek_io *fli, + struct file *file, loff_t offset, int whence); +int fuse_lseek_backing(struct fuse_bpf_args *fa, struct file *file, loff_t offset, int whence); +void *fuse_lseek_finalize(struct fuse_bpf_args *fa, struct file *file, loff_t offset, int whence); + +struct fuse_copy_file_range_io { + struct fuse_copy_file_range_in fci; + struct fuse_write_out fwo; +}; + +int fuse_copy_file_range_initialize(struct fuse_bpf_args *fa, + struct fuse_copy_file_range_io *fcf, + struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t len, unsigned int flags); +int fuse_copy_file_range_backing(struct fuse_bpf_args *fa, + struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t len, unsigned int flags); +void *fuse_copy_file_range_finalize(struct fuse_bpf_args *fa, + struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t len, unsigned int flags); + +int fuse_fsync_initialize(struct fuse_bpf_args *fa, struct fuse_fsync_in *ffi, + struct file *file, loff_t start, loff_t end, int datasync); +int fuse_fsync_backing(struct fuse_bpf_args *fa, + struct file *file, loff_t start, loff_t end, int datasync); +void *fuse_fsync_finalize(struct fuse_bpf_args *fa, + struct file *file, loff_t start, loff_t end, int datasync); +int fuse_dir_fsync_initialize(struct fuse_bpf_args *fa, struct fuse_fsync_in *ffi, + struct file *file, loff_t start, loff_t end, int datasync); + +struct fuse_getxattr_io { + struct fuse_getxattr_in fgi; + struct fuse_getxattr_out fgo; +}; + +int fuse_getxattr_initialize( + struct fuse_bpf_args *fa, struct fuse_getxattr_io *fgio, + struct dentry *dentry, const char *name, void *value, + size_t size); +int fuse_getxattr_backing( + struct fuse_bpf_args *fa, + struct dentry *dentry, const char *name, void *value, + size_t size); +void *fuse_getxattr_finalize( + struct fuse_bpf_args *fa, + struct dentry *dentry, const char *name, void *value, + size_t size); + +int fuse_listxattr_initialize(struct fuse_bpf_args *fa, + struct fuse_getxattr_io *fgio, + struct dentry *dentry, char *list, size_t size); +int fuse_listxattr_backing(struct fuse_bpf_args *fa, struct dentry *dentry, + char *list, size_t size); +void *fuse_listxattr_finalize(struct fuse_bpf_args *fa, struct dentry *dentry, + char *list, size_t size); + +int fuse_setxattr_initialize(struct fuse_bpf_args *fa, + struct fuse_setxattr_in *fsxi, + struct dentry *dentry, const char *name, + const void *value, size_t size, int flags); +int fuse_setxattr_backing(struct fuse_bpf_args *fa, struct dentry *dentry, + const char *name, const void *value, size_t size, + int flags); +void *fuse_setxattr_finalize(struct fuse_bpf_args *fa, struct dentry *dentry, + const char *name, const void *value, size_t size, + int flags); + +int fuse_removexattr_initialize(struct fuse_bpf_args *fa, + struct fuse_dummy_io *unused, + struct dentry *dentry, const char *name); +int fuse_removexattr_backing(struct fuse_bpf_args *fa, + struct dentry *dentry, const char *name); +void *fuse_removexattr_finalize(struct fuse_bpf_args *fa, + struct dentry *dentry, const char *name); + +struct fuse_read_iter_out { + uint64_t ret; +}; +struct fuse_file_read_iter_io { + struct fuse_read_in fri; + struct fuse_read_iter_out frio; +}; + +int fuse_file_read_iter_initialize( + struct fuse_bpf_args *fa, struct fuse_file_read_iter_io *fri, + struct kiocb *iocb, struct iov_iter *to); +int fuse_file_read_iter_backing(struct fuse_bpf_args *fa, + struct kiocb *iocb, struct iov_iter *to); +void *fuse_file_read_iter_finalize(struct fuse_bpf_args *fa, + struct kiocb *iocb, struct iov_iter *to); + +struct fuse_write_iter_out { + uint64_t ret; +}; +struct fuse_file_write_iter_io { + struct fuse_write_in fwi; + struct fuse_write_out fwo; + struct fuse_write_iter_out fwio; +}; + +int fuse_file_write_iter_initialize( + struct fuse_bpf_args *fa, struct fuse_file_write_iter_io *fwio, + struct kiocb *iocb, struct iov_iter *from); +int fuse_file_write_iter_backing(struct fuse_bpf_args *fa, + struct kiocb *iocb, struct iov_iter *from); +void *fuse_file_write_iter_finalize(struct fuse_bpf_args *fa, + struct kiocb *iocb, struct iov_iter *from); + +long fuse_backing_ioctl(struct file *file, unsigned int command, unsigned long arg, int flags); + +int fuse_file_flock_backing(struct file *file, int cmd, struct file_lock *fl); +ssize_t fuse_backing_mmap(struct file *file, struct vm_area_struct *vma); + +int fuse_file_fallocate_initialize(struct fuse_bpf_args *fa, + struct fuse_fallocate_in *ffi, + struct file *file, int mode, loff_t offset, loff_t length); +int fuse_file_fallocate_backing(struct fuse_bpf_args *fa, + struct file *file, int mode, loff_t offset, loff_t length); +void *fuse_file_fallocate_finalize(struct fuse_bpf_args *fa, + struct file *file, int mode, loff_t offset, loff_t length); + +struct fuse_lookup_io { + struct fuse_entry_out feo; + struct fuse_entry_bpf feb; +}; + +int fuse_handle_backing(struct fuse_entry_bpf *feb, struct inode **backing_inode, + struct path *backing_path); +int fuse_handle_bpf_prog(struct fuse_entry_bpf *feb, struct inode *parent, + struct bpf_prog **bpf); + +int fuse_lookup_initialize(struct fuse_bpf_args *fa, struct fuse_lookup_io *feo, + struct inode *dir, struct dentry *entry, unsigned int flags); +int fuse_lookup_backing(struct fuse_bpf_args *fa, struct inode *dir, + struct dentry *entry, unsigned int flags); +struct dentry *fuse_lookup_finalize(struct fuse_bpf_args *fa, struct inode *dir, + struct dentry *entry, unsigned int flags); +int fuse_revalidate_backing(struct dentry *entry, unsigned int flags); + +int fuse_canonical_path_initialize(struct fuse_bpf_args *fa, + struct fuse_dummy_io *fdi, + const struct path *path, + struct path *canonical_path); +int fuse_canonical_path_backing(struct fuse_bpf_args *fa, const struct path *path, + struct path *canonical_path); +void *fuse_canonical_path_finalize(struct fuse_bpf_args *fa, + const struct path *path, + struct path *canonical_path); + +struct fuse_getattr_io { + struct fuse_getattr_in fgi; + struct fuse_attr_out fao; +}; +int fuse_getattr_initialize(struct fuse_bpf_args *fa, struct fuse_getattr_io *fgio, + const struct dentry *entry, struct kstat *stat, + u32 request_mask, unsigned int flags); +int fuse_getattr_backing(struct fuse_bpf_args *fa, + const struct dentry *entry, struct kstat *stat, + u32 request_mask, unsigned int flags); +void *fuse_getattr_finalize(struct fuse_bpf_args *fa, + const struct dentry *entry, struct kstat *stat, + u32 request_mask, unsigned int flags); + +struct fuse_setattr_io { + struct fuse_setattr_in fsi; + struct fuse_attr_out fao; +}; + +int fuse_setattr_initialize(struct fuse_bpf_args *fa, struct fuse_setattr_io *fsi, + struct dentry *dentry, struct iattr *attr, struct file *file); +int fuse_setattr_backing(struct fuse_bpf_args *fa, + struct dentry *dentry, struct iattr *attr, struct file *file); +void *fuse_setattr_finalize(struct fuse_bpf_args *fa, + struct dentry *dentry, struct iattr *attr, struct file *file); + +int fuse_statfs_initialize(struct fuse_bpf_args *fa, struct fuse_statfs_out *fso, + struct dentry *dentry, struct kstatfs *buf); +int fuse_statfs_backing(struct fuse_bpf_args *fa, + struct dentry *dentry, struct kstatfs *buf); +void *fuse_statfs_finalize(struct fuse_bpf_args *fa, + struct dentry *dentry, struct kstatfs *buf); + +int fuse_get_link_initialize(struct fuse_bpf_args *fa, struct fuse_dummy_io *dummy, + struct inode *inode, struct dentry *dentry, + struct delayed_call *callback, const char **out); +int fuse_get_link_backing(struct fuse_bpf_args *fa, + struct inode *inode, struct dentry *dentry, + struct delayed_call *callback, const char **out); +void *fuse_get_link_finalize(struct fuse_bpf_args *fa, + struct inode *inode, struct dentry *dentry, + struct delayed_call *callback, const char **out); + +int fuse_symlink_initialize( + struct fuse_bpf_args *fa, struct fuse_dummy_io *unused, + struct inode *dir, struct dentry *entry, const char *link, int len); +int fuse_symlink_backing( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, const char *link, int len); +void *fuse_symlink_finalize( + struct fuse_bpf_args *fa, + struct inode *dir, struct dentry *entry, const char *link, int len); + +struct fuse_read_io { + struct fuse_read_in fri; + struct fuse_read_out fro; +}; + +int fuse_readdir_initialize(struct fuse_bpf_args *fa, struct fuse_read_io *frio, + struct file *file, struct dir_context *ctx, + bool *force_again, bool *allow_force, bool is_continued); +int fuse_readdir_backing(struct fuse_bpf_args *fa, + struct file *file, struct dir_context *ctx, + bool *force_again, bool *allow_force, bool is_continued); +void *fuse_readdir_finalize(struct fuse_bpf_args *fa, + struct file *file, struct dir_context *ctx, + bool *force_again, bool *allow_force, bool is_continued); + +int fuse_access_initialize(struct fuse_bpf_args *fa, struct fuse_access_in *fai, + struct inode *inode, int mask); +int fuse_access_backing(struct fuse_bpf_args *fa, struct inode *inode, int mask); +void *fuse_access_finalize(struct fuse_bpf_args *fa, struct inode *inode, int mask); + +/* + * FUSE caches dentries and attributes with separate timeout. The + * time in jiffies until the dentry/attributes are valid is stored in + * dentry->d_fsdata and fuse_inode->i_time respectively. + */ + +/* + * Calculate the time in jiffies until a dentry/attributes are valid + */ +static inline u64 time_to_jiffies(u64 sec, u32 nsec) +{ + if (sec || nsec) { + struct timespec64 ts = { + sec, + min_t(u32, nsec, NSEC_PER_SEC - 1) + }; + + return get_jiffies_64() + timespec64_to_jiffies(&ts); + } else + return 0; +} + +static inline u64 attr_timeout(struct fuse_attr_out *o) +{ + return time_to_jiffies(o->attr_valid, o->attr_valid_nsec); +} + +static inline bool update_mtime(unsigned ivalid, bool trust_local_mtime) +{ + /* Always update if mtime is explicitly set */ + if (ivalid & ATTR_MTIME_SET) + return true; + + /* Or if kernel i_mtime is the official one */ + if (trust_local_mtime) + return true; + + /* If it's an open(O_TRUNC) or an ftruncate(), don't update */ + if ((ivalid & ATTR_SIZE) && (ivalid & (ATTR_OPEN | ATTR_FILE))) + return false; + + /* In all other cases update */ + return true; +} + +void fuse_fillattr(struct inode *inode, struct fuse_attr *attr, + struct kstat *stat); + +static inline void iattr_to_fattr(struct fuse_conn *fc, struct iattr *iattr, + struct fuse_setattr_in *arg, bool trust_local_cmtime) +{ + unsigned ivalid = iattr->ia_valid; + + if (ivalid & ATTR_MODE) + arg->valid |= FATTR_MODE, arg->mode = iattr->ia_mode; + if (ivalid & ATTR_UID) + arg->valid |= FATTR_UID, arg->uid = from_kuid(fc->user_ns, iattr->ia_uid); + if (ivalid & ATTR_GID) + arg->valid |= FATTR_GID, arg->gid = from_kgid(fc->user_ns, iattr->ia_gid); + if (ivalid & ATTR_SIZE) + arg->valid |= FATTR_SIZE, arg->size = iattr->ia_size; + if (ivalid & ATTR_ATIME) { + arg->valid |= FATTR_ATIME; + arg->atime = iattr->ia_atime.tv_sec; + arg->atimensec = iattr->ia_atime.tv_nsec; + if (!(ivalid & ATTR_ATIME_SET)) + arg->valid |= FATTR_ATIME_NOW; + } + if ((ivalid & ATTR_MTIME) && update_mtime(ivalid, trust_local_cmtime)) { + arg->valid |= FATTR_MTIME; + arg->mtime = iattr->ia_mtime.tv_sec; + arg->mtimensec = iattr->ia_mtime.tv_nsec; + if (!(ivalid & ATTR_MTIME_SET) && !trust_local_cmtime) + arg->valid |= FATTR_MTIME_NOW; + } + if ((ivalid & ATTR_CTIME) && trust_local_cmtime) { + arg->valid |= FATTR_CTIME; + arg->ctime = iattr->ia_ctime.tv_sec; + arg->ctimensec = iattr->ia_ctime.tv_nsec; + } +} + +static inline int finalize_attr(struct inode *inode, struct fuse_attr_out *outarg, + u64 attr_version, struct kstat *stat) +{ + int err = 0; + + if (fuse_invalid_attr(&outarg->attr) || + ((inode->i_mode ^ outarg->attr.mode) & S_IFMT)) { + fuse_make_bad(inode); + err = -EIO; + } else { + fuse_change_attributes(inode, &outarg->attr, + attr_timeout(outarg), + attr_version); + if (stat) + fuse_fillattr(inode, &outarg->attr, stat); + } + return err; +} + +static inline void convert_statfs_to_fuse(struct fuse_kstatfs *attr, struct kstatfs *stbuf) +{ + attr->bsize = stbuf->f_bsize; + attr->frsize = stbuf->f_frsize; + attr->blocks = stbuf->f_blocks; + attr->bfree = stbuf->f_bfree; + attr->bavail = stbuf->f_bavail; + attr->files = stbuf->f_files; + attr->ffree = stbuf->f_ffree; + attr->namelen = stbuf->f_namelen; + /* fsid is left zero */ +} + +static inline void convert_fuse_statfs(struct kstatfs *stbuf, struct fuse_kstatfs *attr) +{ + stbuf->f_type = FUSE_SUPER_MAGIC; + stbuf->f_bsize = attr->bsize; + stbuf->f_frsize = attr->frsize; + stbuf->f_blocks = attr->blocks; + stbuf->f_bfree = attr->bfree; + stbuf->f_bavail = attr->bavail; + stbuf->f_files = attr->files; + stbuf->f_ffree = attr->ffree; + stbuf->f_namelen = attr->namelen; + /* fsid is left zero */ +} + +#ifdef CONFIG_FUSE_BPF +struct fuse_err_ret { + void *result; + bool ret; +}; + +int __init fuse_bpf_init(void); +void __exit fuse_bpf_cleanup(void); + +ssize_t fuse_bpf_simple_request(struct fuse_mount *fm, struct fuse_bpf_args *args); + +static inline int fuse_bpf_run(struct bpf_prog *prog, struct fuse_bpf_args *fba) +{ + int ret; + + migrate_disable(); + ret = BPF_PROG_RUN(prog, fba); + migrate_enable(); + return ret; +} + +/* + * expression statement to wrap the backing filter logic + * struct inode *inode: inode with bpf and backing inode + * typedef io: (typically complex) type whose components fuse_args can point to. + * An instance of this type is created locally and passed to initialize + * void initialize(struct fuse_bpf_args *fa, io *in_out, args...): function that sets + * up fa and io based on args + * int backing(struct fuse_bpf_args *fa, args...): function that actually performs + * the backing io operation + * void *finalize(struct fuse_bpf_args *, args...): function that performs any final + * work needed to commit the backing io + */ +#define fuse_bpf_backing(inode, io, initialize, backing, finalize, \ + args...) \ +({ \ + struct fuse_err_ret fer = {0}; \ + int ext_flags; \ + struct fuse_inode *fuse_inode = get_fuse_inode(inode); \ + struct fuse_mount *fm = get_fuse_mount(inode); \ + io feo = {0}; \ + struct fuse_bpf_args fa = {0}, fa_backup = {0}; \ + bool locked; \ + ssize_t res; \ + void *err; \ + int i; \ + bool initialized = false; \ + \ + do { \ + if (!fuse_inode || !fuse_inode->backing_inode) \ + break; \ + \ + err = ERR_PTR(initialize(&fa, &feo, args)); \ + if (err) { \ + fer = (struct fuse_err_ret) { \ + err, \ + true, \ + }; \ + break; \ + } \ + initialized = true; \ + \ + fa_backup = fa; \ + fa.opcode |= FUSE_PREFILTER; \ + for (i = 0; i < fa.in_numargs; ++i) \ + fa.out_args[i] = (struct fuse_bpf_arg) { \ + .size = fa.in_args[i].size, \ + .value = (void *)fa.in_args[i].value, \ + }; \ + fa.out_numargs = fa.in_numargs; \ + \ + ext_flags = fuse_inode->bpf ? \ + fuse_bpf_run(fuse_inode->bpf, &fa) : \ + FUSE_BPF_BACKING; \ + if (ext_flags < 0) { \ + fer = (struct fuse_err_ret) { \ + ERR_PTR(ext_flags), \ + true, \ + }; \ + break; \ + } \ + \ + if (ext_flags & FUSE_BPF_USER_FILTER) { \ + locked = fuse_lock_inode(inode); \ + res = fuse_bpf_simple_request(fm, &fa); \ + fuse_unlock_inode(inode, locked); \ + if (res < 0) { \ + fer = (struct fuse_err_ret) { \ + ERR_PTR(res), \ + true, \ + }; \ + break; \ + } \ + } \ + \ + if (!(ext_flags & FUSE_BPF_BACKING)) \ + break; \ + \ + fa.opcode &= ~FUSE_PREFILTER; \ + for (i = 0; i < fa.in_numargs; ++i) \ + fa.in_args[i] = (struct fuse_bpf_in_arg) { \ + .size = fa.out_args[i].size, \ + .value = fa.out_args[i].value, \ + }; \ + for (i = 0; i < fa_backup.out_numargs; ++i) \ + fa.out_args[i] = (struct fuse_bpf_arg) { \ + .size = fa_backup.out_args[i].size, \ + .value = fa_backup.out_args[i].value, \ + }; \ + fa.out_numargs = fa_backup.out_numargs; \ + \ + fer = (struct fuse_err_ret) { \ + ERR_PTR(backing(&fa, args)), \ + true, \ + }; \ + if (IS_ERR(fer.result)) \ + fa.error_in = PTR_ERR(fer.result); \ + if (!(ext_flags & FUSE_BPF_POST_FILTER)) \ + break; \ + \ + fa.opcode |= FUSE_POSTFILTER; \ + for (i = 0; i < fa.out_numargs; ++i) \ + fa.in_args[fa.in_numargs++] = \ + (struct fuse_bpf_in_arg) { \ + .size = fa.out_args[i].size, \ + .value = fa.out_args[i].value, \ + }; \ + ext_flags = fuse_bpf_run(fuse_inode->bpf, &fa); \ + if (ext_flags < 0) { \ + fer = (struct fuse_err_ret) { \ + ERR_PTR(ext_flags), \ + true, \ + }; \ + break; \ + } \ + if (!(ext_flags & FUSE_BPF_USER_FILTER)) \ + break; \ + \ + fa.out_args[0].size = fa_backup.out_args[0].size; \ + fa.out_args[1].size = fa_backup.out_args[1].size; \ + fa.out_numargs = fa_backup.out_numargs; \ + locked = fuse_lock_inode(inode); \ + res = fuse_bpf_simple_request(fm, &fa); \ + fuse_unlock_inode(inode, locked); \ + if (res < 0) { \ + fer.result = ERR_PTR(res); \ + break; \ + } \ + } while (false); \ + \ + if (initialized && fer.ret) { \ + err = finalize(&fa, args); \ + if (err) \ + fer.result = err; \ + } \ + \ + fer; \ +}) +#endif /* CONFIG_FUSE_BPF */ + #endif /* _FS_FUSE_I_H */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index b32c5c84004c..e37fba257c8b 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -50,8 +50,6 @@ MODULE_PARM_DESC(max_user_congthresh, "Global limit for the maximum congestion threshold an " "unprivileged user can set"); -#define FUSE_SUPER_MAGIC 0x65735546 - #define FUSE_DEFAULT_BLKSIZE 512 /** Maximum number of outstanding background requests */ @@ -79,20 +77,32 @@ static struct inode *fuse_alloc_inode(struct super_block *sb) fi->i_time = 0; fi->inval_mask = 0; +#ifdef CONFIG_FUSE_BPF + fi->backing_inode = NULL; + fi->bpf = NULL; +#endif fi->nodeid = 0; fi->nlookup = 0; fi->attr_version = 0; fi->orig_ino = 0; fi->state = 0; mutex_init(&fi->mutex); + init_rwsem(&fi->i_mmap_sem); spin_lock_init(&fi->lock); fi->forget = fuse_alloc_forget(); - if (!fi->forget) { - kmem_cache_free(fuse_inode_cachep, fi); - return NULL; - } + if (!fi->forget) + goto out_free; + + if (IS_ENABLED(CONFIG_FUSE_DAX) && !fuse_dax_inode_alloc(sb, fi)) + goto out_free_forget; return &fi->inode; + +out_free_forget: + kfree(fi->forget); +out_free: + kmem_cache_free(fuse_inode_cachep, fi); + return NULL; } static void fuse_free_inode(struct inode *inode) @@ -101,6 +111,13 @@ static void fuse_free_inode(struct inode *inode) mutex_destroy(&fi->mutex); kfree(fi->forget); +#ifdef CONFIG_FUSE_DAX + kfree(fi->dax); +#endif +#ifdef CONFIG_FUSE_BPF + if (fi->bpf) + bpf_prog_put(fi->bpf); +#endif kmem_cache_free(fuse_inode_cachep, fi); } @@ -108,12 +125,21 @@ static void fuse_evict_inode(struct inode *inode) { struct fuse_inode *fi = get_fuse_inode(inode); + /* Will write inode on close/munmap and in all other dirtiers */ + WARN_ON(inode->i_state & I_DIRTY_INODE); + truncate_inode_pages_final(&inode->i_data); clear_inode(inode); if (inode->i_sb->s_flags & SB_ACTIVE) { struct fuse_conn *fc = get_fuse_conn(inode); - fuse_queue_forget(fc, fi->forget, fi->nodeid, fi->nlookup); - fi->forget = NULL; + + if (FUSE_IS_DAX(inode)) + fuse_dax_inode_cleanup(inode); + if (fi->nlookup) { + fuse_queue_forget(fc, fi->forget, fi->nodeid, + fi->nlookup); + fi->forget = NULL; + } } if (S_ISREG(inode->i_mode) && !fuse_is_bad(inode)) { WARN_ON(!list_empty(&fi->write_files)); @@ -121,12 +147,21 @@ static void fuse_evict_inode(struct inode *inode) } } -static int fuse_reconfigure(struct fs_context *fc) +#ifdef CONFIG_FUSE_BPF +static void fuse_destroy_inode(struct inode *inode) { - struct super_block *sb = fc->root->d_sb; + struct fuse_inode *fi = get_fuse_inode(inode); + + iput(fi->backing_inode); +} +#endif + +static int fuse_reconfigure(struct fs_context *fsc) +{ + struct super_block *sb = fsc->root->d_sb; sync_filesystem(sb); - if (fc->sb_flags & SB_MANDLOCK) + if (fsc->sb_flags & SB_MANDLOCK) return -EINVAL; return 0; @@ -144,6 +179,28 @@ static ino_t fuse_squash_ino(u64 ino64) return ino; } +static void fuse_fill_attr_from_inode(struct fuse_attr *attr, + const struct inode *inode) +{ + *attr = (struct fuse_attr){ + .ino = inode->i_ino, + .size = inode->i_size, + .blocks = inode->i_blocks, + .atime = inode->i_atime.tv_sec, + .mtime = inode->i_mtime.tv_sec, + .ctime = inode->i_ctime.tv_sec, + .atimensec = inode->i_atime.tv_nsec, + .mtimensec = inode->i_mtime.tv_nsec, + .ctimensec = inode->i_ctime.tv_nsec, + .mode = inode->i_mode, + .nlink = inode->i_nlink, + .uid = inode->i_uid.val, + .gid = inode->i_gid.val, + .rdev = inode->i_rdev, + .blksize = 1u << inode->i_blkbits, + }; +} + void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, u64 attr_valid) { @@ -268,28 +325,104 @@ static void fuse_init_inode(struct inode *inode, struct fuse_attr *attr) else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { fuse_init_common(inode); - init_special_inode(inode, inode->i_mode, - new_decode_dev(attr->rdev)); + init_special_inode(inode, inode->i_mode, attr->rdev); } else BUG(); } -int fuse_inode_eq(struct inode *inode, void *_nodeidp) +struct fuse_inode_identifier { + u64 nodeid; + struct inode *backing_inode; +}; + +static int fuse_inode_eq(struct inode *inode, void *_nodeidp) { - u64 nodeid = *(u64 *) _nodeidp; - if (get_node_id(inode) == nodeid) - return 1; - else - return 0; + struct fuse_inode_identifier *fii = + (struct fuse_inode_identifier *) _nodeidp; + struct fuse_inode *fi = get_fuse_inode(inode); + + return fii->nodeid == fi->nodeid; +} + +static int fuse_inode_backing_eq(struct inode *inode, void *_nodeidp) +{ + struct fuse_inode_identifier *fii = + (struct fuse_inode_identifier *) _nodeidp; + struct fuse_inode *fi = get_fuse_inode(inode); + + return fii->nodeid == fi->nodeid +#ifdef CONFIG_FUSE_BPF + && fii->backing_inode == fi->backing_inode +#endif + ; } static int fuse_inode_set(struct inode *inode, void *_nodeidp) { - u64 nodeid = *(u64 *) _nodeidp; - get_fuse_inode(inode)->nodeid = nodeid; + struct fuse_inode_identifier *fii = + (struct fuse_inode_identifier *) _nodeidp; + struct fuse_inode *fi = get_fuse_inode(inode); + + fi->nodeid = fii->nodeid; + return 0; } +static int fuse_inode_backing_set(struct inode *inode, void *_nodeidp) +{ + struct fuse_inode_identifier *fii = + (struct fuse_inode_identifier *) _nodeidp; + struct fuse_inode *fi = get_fuse_inode(inode); + + fi->nodeid = fii->nodeid; +#ifdef CONFIG_FUSE_BPF + fi->backing_inode = fii->backing_inode; + if (fi->backing_inode) + ihold(fi->backing_inode); +#endif + + return 0; +} + +struct inode *fuse_iget_backing(struct super_block *sb, u64 nodeid, + struct inode *backing_inode) +{ + struct inode *inode; + struct fuse_inode *fi; + struct fuse_conn *fc = get_fuse_conn_super(sb); + struct fuse_inode_identifier fii = { + .nodeid = nodeid, + .backing_inode = backing_inode, + }; + struct fuse_attr attr; + unsigned long hash = (unsigned long) backing_inode; + + if (nodeid) + hash = nodeid; + + fuse_fill_attr_from_inode(&attr, backing_inode); + inode = iget5_locked(sb, hash, fuse_inode_backing_eq, + fuse_inode_backing_set, &fii); + if (!inode) + return NULL; + + if ((inode->i_state & I_NEW)) { + inode->i_flags |= S_NOATIME; + if (!fc->writeback_cache) + inode->i_flags |= S_NOCMTIME; + fuse_init_common(inode); + unlock_new_inode(inode); + } + + fi = get_fuse_inode(inode); + fuse_init_inode(inode, &attr); + spin_lock(&fi->lock); + fi->nlookup++; + spin_unlock(&fi->lock); + + return inode; +} + struct inode *fuse_iget(struct super_block *sb, u64 nodeid, int generation, struct fuse_attr *attr, u64 attr_valid, u64 attr_version) @@ -297,9 +430,31 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid, struct inode *inode; struct fuse_inode *fi; struct fuse_conn *fc = get_fuse_conn_super(sb); + struct fuse_inode_identifier fii = { + .nodeid = nodeid, + }; - retry: - inode = iget5_locked(sb, nodeid, fuse_inode_eq, fuse_inode_set, &nodeid); + /* + * Auto mount points get their node id from the submount root, which is + * not a unique identifier within this filesystem. + * + * To avoid conflicts, do not place submount points into the inode hash + * table. + */ + if (fc->auto_submounts && (attr->flags & FUSE_ATTR_SUBMOUNT) && + S_ISDIR(attr->mode)) { + inode = new_inode(sb); + if (!inode) + return NULL; + + fuse_init_inode(inode, attr); + get_fuse_inode(inode)->nodeid = nodeid; + inode->i_flags |= S_AUTOMOUNT; + goto done; + } + +retry: + inode = iget5_locked(sb, nodeid, fuse_inode_eq, fuse_inode_set, &fii); if (!inode) return NULL; @@ -310,8 +465,8 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid, inode->i_generation = generation; fuse_init_inode(inode, attr); unlock_new_inode(inode); - } else if ((inode->i_mode ^ attr->mode) & S_IFMT) { - /* Inode has changed type, any I/O on the old should fail */ + } else if (fuse_stale_inode(inode, generation, attr)) { + /* nodeid was reused, any I/O on the old inode should fail */ fuse_make_bad(inode); if (inode != d_inode(sb->s_root)) { remove_inode_hash(inode); @@ -319,7 +474,7 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid, goto retry; } } - +done: fi = get_fuse_inode(inode); spin_lock(&fi->lock); fi->nlookup++; @@ -329,16 +484,40 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid, return inode; } -int fuse_reverse_inval_inode(struct super_block *sb, u64 nodeid, +struct inode *fuse_ilookup(struct fuse_conn *fc, u64 nodeid, + struct fuse_mount **fm) +{ + struct fuse_mount *fm_iter; + struct inode *inode; + struct fuse_inode_identifier fii = { + .nodeid = nodeid, + }; + + WARN_ON(!rwsem_is_locked(&fc->killsb)); + list_for_each_entry(fm_iter, &fc->mounts, fc_entry) { + if (!fm_iter->sb) + continue; + + inode = ilookup5(fm_iter->sb, nodeid, fuse_inode_eq, &fii); + if (inode) { + if (fm) + *fm = fm_iter; + return inode; + } + } + + return NULL; +} + +int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { - struct fuse_conn *fc = get_fuse_conn_super(sb); struct fuse_inode *fi; struct inode *inode; pgoff_t pg_start; pgoff_t pg_end; - inode = ilookup5(sb, nodeid, fuse_inode_eq, &nodeid); + inode = fuse_ilookup(fc, nodeid, NULL); if (!inode) return -ENOENT; @@ -388,57 +567,50 @@ static void fuse_umount_begin(struct super_block *sb) fuse_abort_conn(fc); } -static void fuse_send_destroy(struct fuse_conn *fc) +static void fuse_send_destroy(struct fuse_mount *fm) { - if (fc->conn_init) { + if (fm->fc->conn_init) { FUSE_ARGS(args); args.opcode = FUSE_DESTROY; args.force = true; args.nocreds = true; - fuse_simple_request(fc, &args); + fuse_simple_request(fm, &args); } } static void fuse_put_super(struct super_block *sb) { - struct fuse_conn *fc = get_fuse_conn_super(sb); + struct fuse_mount *fm = get_fuse_mount_super(sb); - mutex_lock(&fuse_mutex); - list_del(&fc->entry); - fuse_ctl_remove_conn(fc); - mutex_unlock(&fuse_mutex); - - fuse_conn_put(fc); -} - -static void convert_fuse_statfs(struct kstatfs *stbuf, struct fuse_kstatfs *attr) -{ - stbuf->f_type = FUSE_SUPER_MAGIC; - stbuf->f_bsize = attr->bsize; - stbuf->f_frsize = attr->frsize; - stbuf->f_blocks = attr->blocks; - stbuf->f_bfree = attr->bfree; - stbuf->f_bavail = attr->bavail; - stbuf->f_files = attr->files; - stbuf->f_ffree = attr->ffree; - stbuf->f_namelen = attr->namelen; - /* fsid is left zero */ + fuse_mount_put(fm); } static int fuse_statfs(struct dentry *dentry, struct kstatfs *buf) { struct super_block *sb = dentry->d_sb; - struct fuse_conn *fc = get_fuse_conn_super(sb); + struct fuse_mount *fm = get_fuse_mount_super(sb); FUSE_ARGS(args); struct fuse_statfs_out outarg; int err; +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; +#endif - if (!fuse_allow_current_process(fc)) { + if (!fuse_allow_current_process(fm->fc)) { buf->f_type = FUSE_SUPER_MAGIC; return 0; } +#ifdef CONFIG_FUSE_BPF + fer = fuse_bpf_backing(dentry->d_inode, struct fuse_statfs_out, + fuse_statfs_initialize, fuse_statfs_backing, + fuse_statfs_finalize, + dentry, buf); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + memset(&outarg, 0, sizeof(outarg)); args.in_numargs = 0; args.opcode = FUSE_STATFS; @@ -446,7 +618,7 @@ static int fuse_statfs(struct dentry *dentry, struct kstatfs *buf) args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (!err) convert_fuse_statfs(buf, &outarg.st); return err; @@ -463,6 +635,9 @@ enum { OPT_ALLOW_OTHER, OPT_MAX_READ, OPT_BLKSIZE, + OPT_ROOT_BPF, + OPT_ROOT_DIR, + OPT_NO_DAEMON, OPT_ERR }; @@ -477,6 +652,9 @@ static const struct fs_parameter_spec fuse_param_specs[] = { fsparam_u32 ("max_read", OPT_MAX_READ), fsparam_u32 ("blksize", OPT_BLKSIZE), fsparam_string ("subtype", OPT_SUBTYPE), + fsparam_u32 ("root_bpf", OPT_ROOT_BPF), + fsparam_u32 ("root_dir", OPT_ROOT_DIR), + fsparam_flag ("no_daemon", OPT_NO_DAEMON), {} }; @@ -485,70 +663,90 @@ static const struct fs_parameter_description fuse_fs_parameters = { .specs = fuse_param_specs, }; -static int fuse_parse_param(struct fs_context *fc, struct fs_parameter *param) +static int fuse_parse_param(struct fs_context *fsc, struct fs_parameter *param) { struct fs_parse_result result; - struct fuse_fs_context *ctx = fc->fs_private; + struct fuse_fs_context *ctx = fsc->fs_private; int opt; + kuid_t kuid; + kgid_t kgid; - /* - * Ignore options coming from mount(MS_REMOUNT) for backward - * compatibility. - */ - if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) - return 0; + if (fsc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + /* + * Ignore options coming from mount(MS_REMOUNT) for backward + * compatibility. + */ + if (fsc->oldapi) + return 0; - opt = fs_parse(fc, &fuse_fs_parameters, param, &result); + return invalf(fsc, "No changes allowed in reconfigure"); + } + + opt = fs_parse(fsc, &fuse_fs_parameters, param, &result); if (opt < 0) return opt; switch (opt) { case OPT_SOURCE: - if (fc->source) - return invalf(fc, "fuse: Multiple sources specified"); - fc->source = param->string; + if (fsc->source) + return invalf(fsc, "Multiple sources specified"); + fsc->source = param->string; param->string = NULL; break; case OPT_SUBTYPE: if (ctx->subtype) - return invalf(fc, "fuse: Multiple subtypes specified"); + return invalf(fsc, "Multiple subtypes specified"); ctx->subtype = param->string; param->string = NULL; return 0; case OPT_FD: ctx->fd = result.uint_32; - ctx->fd_present = 1; + ctx->fd_present = true; break; case OPT_ROOTMODE: if (!fuse_valid_type(result.uint_32)) - return invalf(fc, "fuse: Invalid rootmode"); + return invalf(fsc, "Invalid rootmode"); ctx->rootmode = result.uint_32; - ctx->rootmode_present = 1; + ctx->rootmode_present = true; break; case OPT_USER_ID: - ctx->user_id = make_kuid(fc->user_ns, result.uint_32); - if (!uid_valid(ctx->user_id)) - return invalf(fc, "fuse: Invalid user_id"); - ctx->user_id_present = 1; + kuid = make_kuid(fsc->user_ns, result.uint_32); + if (!uid_valid(kuid)) + return invalf(fsc, "Invalid user_id"); + /* + * The requested uid must be representable in the + * filesystem's idmapping. + */ + if (!kuid_has_mapping(fsc->user_ns, kuid)) + return invalf(fsc, "Invalid user_id"); + ctx->user_id = kuid; + ctx->user_id_present = true; break; case OPT_GROUP_ID: - ctx->group_id = make_kgid(fc->user_ns, result.uint_32); - if (!gid_valid(ctx->group_id)) - return invalf(fc, "fuse: Invalid group_id"); - ctx->group_id_present = 1; + kgid = make_kgid(fsc->user_ns, result.uint_32);; + if (!gid_valid(kgid)) + return invalf(fsc, "Invalid group_id"); + /* + * The requested gid must be representable in the + * filesystem's idmapping. + */ + if (!kgid_has_mapping(fsc->user_ns, kgid)) + return invalf(fsc, "Invalid group_id"); + ctx->group_id = kgid; + ctx->group_id_present = true; break; case OPT_DEFAULT_PERMISSIONS: - ctx->default_permissions = 1; + ctx->default_permissions = true; break; case OPT_ALLOW_OTHER: - ctx->allow_other = 1; + ctx->allow_other = true; break; case OPT_MAX_READ: @@ -557,10 +755,30 @@ static int fuse_parse_param(struct fs_context *fc, struct fs_parameter *param) case OPT_BLKSIZE: if (!ctx->is_bdev) - return invalf(fc, "fuse: blksize only supported for fuseblk"); + return invalf(fsc, "blksize only supported for fuseblk"); ctx->blksize = result.uint_32; break; + case OPT_ROOT_BPF: + ctx->root_bpf = bpf_prog_get_type_dev(result.uint_32, + BPF_PROG_TYPE_FUSE, false); + if (IS_ERR(ctx->root_bpf)) { + ctx->root_bpf = NULL; + return invalf(fsc, "Unable to open bpf program"); + } + break; + + case OPT_ROOT_DIR: + ctx->root_dir = fget(result.uint_32); + if (!ctx->root_dir) + return invalf(fsc, "Unable to open root directory"); + break; + + case OPT_NO_DAEMON: + ctx->no_daemon = true; + ctx->fd_present = true; + break; + default: return -EINVAL; } @@ -568,11 +786,15 @@ static int fuse_parse_param(struct fs_context *fc, struct fs_parameter *param) return 0; } -static void fuse_free_fc(struct fs_context *fc) +static void fuse_free_fsc(struct fs_context *fsc) { - struct fuse_fs_context *ctx = fc->fs_private; + struct fuse_fs_context *ctx = fsc->fs_private; if (ctx) { + if (ctx->root_dir) + fput(ctx->root_dir); + if (ctx->root_bpf) + bpf_prog_put(ctx->root_bpf); kfree(ctx->subtype); kfree(ctx); } @@ -583,19 +805,25 @@ static int fuse_show_options(struct seq_file *m, struct dentry *root) struct super_block *sb = root->d_sb; struct fuse_conn *fc = get_fuse_conn_super(sb); - if (fc->no_mount_options) - return 0; + if (fc->legacy_opts_show) { + seq_printf(m, ",user_id=%u", + from_kuid_munged(fc->user_ns, fc->user_id)); + seq_printf(m, ",group_id=%u", + from_kgid_munged(fc->user_ns, fc->group_id)); + if (fc->default_permissions) + seq_puts(m, ",default_permissions"); + if (fc->allow_other) + seq_puts(m, ",allow_other"); + if (fc->max_read != ~0) + seq_printf(m, ",max_read=%u", fc->max_read); + if (sb->s_bdev && sb->s_blocksize != FUSE_DEFAULT_BLKSIZE) + seq_printf(m, ",blksize=%lu", sb->s_blocksize); + } +#ifdef CONFIG_FUSE_DAX + if (fc->dax) + seq_puts(m, ",dax"); +#endif - seq_printf(m, ",user_id=%u", from_kuid_munged(fc->user_ns, fc->user_id)); - seq_printf(m, ",group_id=%u", from_kgid_munged(fc->user_ns, fc->group_id)); - if (fc->default_permissions) - seq_puts(m, ",default_permissions"); - if (fc->allow_other) - seq_puts(m, ",allow_other"); - if (fc->max_read != ~0) - seq_printf(m, ",max_read=%u", fc->max_read); - if (sb->s_bdev && sb->s_blocksize != FUSE_DEFAULT_BLKSIZE) - seq_printf(m, ",blksize=%lu", sb->s_blocksize); return 0; } @@ -625,7 +853,8 @@ static void fuse_pqueue_init(struct fuse_pqueue *fpq) fpq->connected = 1; } -void fuse_conn_init(struct fuse_conn *fc, struct user_namespace *user_ns, +void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, + struct user_namespace *user_ns, const struct fuse_iqueue_ops *fiq_ops, void *fiq_priv) { memset(fc, 0, sizeof(*fc)); @@ -654,6 +883,12 @@ void fuse_conn_init(struct fuse_conn *fc, struct user_namespace *user_ns, fc->pid_ns = get_pid_ns(task_active_pid_ns(current)); fc->user_ns = get_user_ns(user_ns); fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ; + fc->max_pages_limit = FUSE_MAX_MAX_PAGES; + + INIT_LIST_HEAD(&fc->mounts); + list_add(&fm->fc_entry, &fc->mounts); + fm->fc = fc; + refcount_set(&fm->count, 1); } EXPORT_SYMBOL_GPL(fuse_conn_init); @@ -662,6 +897,8 @@ void fuse_conn_put(struct fuse_conn *fc) if (refcount_dec_and_test(&fc->count)) { struct fuse_iqueue *fiq = &fc->iq; + if (IS_ENABLED(CONFIG_FUSE_DAX)) + fuse_dax_conn_free(fc); if (fiq->ops->release) fiq->ops->release(fiq); put_pid_ns(fc->pid_ns); @@ -678,15 +915,51 @@ struct fuse_conn *fuse_conn_get(struct fuse_conn *fc) } EXPORT_SYMBOL_GPL(fuse_conn_get); -static struct inode *fuse_get_root_inode(struct super_block *sb, unsigned mode) +void fuse_mount_put(struct fuse_mount *fm) +{ + if (refcount_dec_and_test(&fm->count)) { + if (fm->fc) + fuse_conn_put(fm->fc); + kfree(fm); + } +} +EXPORT_SYMBOL_GPL(fuse_mount_put); + +struct fuse_mount *fuse_mount_get(struct fuse_mount *fm) +{ + refcount_inc(&fm->count); + return fm; +} +EXPORT_SYMBOL_GPL(fuse_mount_get); + +static struct inode *fuse_get_root_inode(struct super_block *sb, + unsigned int mode, + struct bpf_prog *root_bpf, + struct file *backing_fd) { struct fuse_attr attr; - memset(&attr, 0, sizeof(attr)); + struct inode *inode; + memset(&attr, 0, sizeof(attr)); attr.mode = mode; attr.ino = FUSE_ROOT_ID; attr.nlink = 1; - return fuse_iget(sb, 1, 0, &attr, 0, 0); + inode = fuse_iget(sb, 1, 0, &attr, 0, 0); + if (!inode) + return NULL; + +#ifdef CONFIG_FUSE_BPF + get_fuse_inode(inode)->bpf = root_bpf; + if (root_bpf) + bpf_prog_inc(root_bpf); + + if (backing_fd) { + get_fuse_inode(inode)->backing_inode = backing_fd->f_inode; + ihold(backing_fd->f_inode); + } +#endif + + return inode; } struct fuse_inode_handle { @@ -701,11 +974,14 @@ static struct dentry *fuse_get_dentry(struct super_block *sb, struct inode *inode; struct dentry *entry; int err = -ESTALE; + struct fuse_inode_identifier fii = { + .nodeid = handle->nodeid, + }; if (handle->nodeid == 0) goto out_err; - inode = ilookup5(sb, handle->nodeid, fuse_inode_eq, &handle->nodeid); + inode = ilookup5(sb, handle->nodeid, fuse_inode_eq, &fii); if (!inode) { struct fuse_entry_out outarg; const struct qstr name = QSTR_INIT(".", 1); @@ -714,7 +990,7 @@ static struct dentry *fuse_get_dentry(struct super_block *sb, goto out_err; err = fuse_lookup_name(sb, handle->nodeid, &name, &outarg, - &inode); + NULL, &inode); if (err && err != -ENOENT) goto out_err; if (err || !inode) { @@ -815,7 +1091,7 @@ static struct dentry *fuse_get_parent(struct dentry *child) return ERR_PTR(-ESTALE); err = fuse_lookup_name(child_inode->i_sb, get_node_id(child_inode), - &name, &outarg, &inode); + &name, &outarg, NULL, &inode); if (err) { if (err == -ENOENT) return ERR_PTR(-ESTALE); @@ -838,6 +1114,9 @@ static const struct export_operations fuse_export_operations = { static const struct super_operations fuse_super_operations = { .alloc_inode = fuse_alloc_inode, +#ifdef CONFIG_FUSE_BPF + .destroy_inode = fuse_destroy_inode, +#endif .free_inode = fuse_free_inode, .evict_inode = fuse_evict_inode, .write_inode = fuse_write_inode, @@ -907,14 +1186,16 @@ struct fuse_init_args { struct fuse_init_out out; }; -static void process_init_reply(struct fuse_conn *fc, struct fuse_args *args, +static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, int error) { + struct fuse_conn *fc = fm->fc; struct fuse_init_args *ia = container_of(args, typeof(*ia), args); struct fuse_init_out *arg = &ia->out; + bool ok = true; if (error || arg->major != FUSE_KERNEL_VERSION) - fc->conn_error = 1; + ok = false; else { unsigned long ra_pages; @@ -962,11 +1243,11 @@ static void process_init_reply(struct fuse_conn *fc, struct fuse_args *args, if (arg->flags & FUSE_HANDLE_KILLPRIV) fc->handle_killpriv = 1; if (arg->time_gran && arg->time_gran <= 1000000000) - fc->sb->s_time_gran = arg->time_gran; + fm->sb->s_time_gran = arg->time_gran; if ((arg->flags & FUSE_POSIX_ACL)) { fc->default_permissions = 1; fc->posix_acl = 1; - fc->sb->s_xattr = fuse_acl_xattr_handlers; + fm->sb->s_xattr = fuse_acl_xattr_handlers; } if (arg->flags & FUSE_CACHE_SYMLINKS) fc->cache_symlinks = 1; @@ -974,13 +1255,18 @@ static void process_init_reply(struct fuse_conn *fc, struct fuse_args *args, fc->abort_err = 1; if (arg->flags & FUSE_MAX_PAGES) { fc->max_pages = - min_t(unsigned int, FUSE_MAX_MAX_PAGES, + min_t(unsigned int, fc->max_pages_limit, max_t(unsigned int, arg->max_pages, 1)); } + if (IS_ENABLED(CONFIG_FUSE_DAX) && + arg->flags & FUSE_MAP_ALIGNMENT && + !fuse_dax_check_alignment(fc, arg->map_alignment)) { + ok = false; + } if (arg->flags & FUSE_PASSTHROUGH) { fc->passthrough = 1; /* Prevent further stacking */ - fc->sb->s_stack_depth = + fm->sb->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH; } } else { @@ -989,8 +1275,8 @@ static void process_init_reply(struct fuse_conn *fc, struct fuse_args *args, fc->no_flock = 1; } - fc->sb->s_bdi->ra_pages = - min(fc->sb->s_bdi->ra_pages, ra_pages); + fm->sb->s_bdi->ra_pages = + min(fm->sb->s_bdi->ra_pages, ra_pages); fc->minor = arg->minor; fc->max_write = arg->minor < 5 ? 4096 : arg->max_write; fc->max_write = max_t(unsigned, 4096, fc->max_write); @@ -998,11 +1284,16 @@ static void process_init_reply(struct fuse_conn *fc, struct fuse_args *args, } kfree(ia); + if (!ok) { + fc->conn_init = 0; + fc->conn_error = 1; + } + fuse_set_initialized(fc); wake_up_all(&fc->blocked_waitq); } -void fuse_send_init(struct fuse_conn *fc) +void fuse_send_init(struct fuse_mount *fm) { struct fuse_init_args *ia; @@ -1010,7 +1301,7 @@ void fuse_send_init(struct fuse_conn *fc) ia->in.major = FUSE_KERNEL_VERSION; ia->in.minor = FUSE_KERNEL_MINOR_VERSION; - ia->in.max_readahead = fc->sb->s_bdi->ra_pages * PAGE_SIZE; + ia->in.max_readahead = fm->sb->s_bdi->ra_pages * PAGE_SIZE; ia->in.flags |= FUSE_ASYNC_READ | FUSE_POSIX_LOCKS | FUSE_ATOMIC_O_TRUNC | FUSE_EXPORT_SUPPORT | FUSE_BIG_WRITES | FUSE_DONT_MASK | @@ -1022,6 +1313,13 @@ void fuse_send_init(struct fuse_conn *fc) FUSE_ABORT_ERROR | FUSE_MAX_PAGES | FUSE_CACHE_SYMLINKS | FUSE_NO_OPENDIR_SUPPORT | FUSE_EXPLICIT_INVAL_DATA | FUSE_PASSTHROUGH; +#ifdef CONFIG_FUSE_DAX + if (fm->fc->dax) + ia->in.flags |= FUSE_MAP_ALIGNMENT; +#endif + if (fm->fc->auto_submounts) + ia->in.flags |= FUSE_SUBMOUNTS; + ia->args.opcode = FUSE_INIT; ia->args.in_numargs = 1; ia->args.in_args[0].size = sizeof(ia->in); @@ -1030,15 +1328,15 @@ void fuse_send_init(struct fuse_conn *fc) /* Variable length argument used for backward compatibility with interface version < 7.5. Rest of init_out is zeroed by do_get_request(), so a short reply is not a problem */ - ia->args.out_argvar = 1; + ia->args.out_argvar = true; ia->args.out_args[0].size = sizeof(ia->out); ia->args.out_args[0].value = &ia->out; ia->args.force = true; ia->args.nocreds = true; ia->args.end = process_init_reply; - if (fuse_simple_background(fc, &ia->args, GFP_KERNEL) != 0) - process_init_reply(fc, &ia->args, -ENOTCONN); + if (unlikely(fm->fc->no_daemon) || fuse_simple_background(fm, &ia->args, GFP_KERNEL) != 0) + process_init_reply(fm, &ia->args, -ENOTCONN); } EXPORT_SYMBOL_GPL(fuse_send_init); @@ -1080,7 +1378,6 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb) if (err) return err; - sb->s_bdi->ra_pages = VM_READAHEAD_PAGES; /* fuse does it's own writeback accounting */ sb->s_bdi->capabilities = BDI_CAP_NO_ACCT_WB | BDI_CAP_STRICTLIMIT; @@ -1161,10 +1458,70 @@ void fuse_dev_free(struct fuse_dev *fud) } EXPORT_SYMBOL_GPL(fuse_dev_free); +static void fuse_sb_defaults(struct super_block *sb) +{ + sb->s_magic = FUSE_SUPER_MAGIC; + sb->s_op = &fuse_super_operations; + sb->s_xattr = fuse_xattr_handlers; + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_time_gran = 1; + sb->s_export_op = &fuse_export_operations; + sb->s_iflags |= SB_I_IMA_UNVERIFIABLE_SIGNATURE; + if (sb->s_user_ns != &init_user_ns) + sb->s_iflags |= SB_I_UNTRUSTED_MOUNTER; + sb->s_flags &= ~(SB_NOSEC | SB_I_VERSION); + + /* + * If we are not in the initial user namespace posix + * acls must be translated. + */ + if (sb->s_user_ns != &init_user_ns) + sb->s_xattr = fuse_no_acl_xattr_handlers; +} + +int fuse_fill_super_submount(struct super_block *sb, + struct fuse_inode *parent_fi) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + struct super_block *parent_sb = parent_fi->inode.i_sb; + struct fuse_attr root_attr; + struct inode *root; + + fuse_sb_defaults(sb); + fm->sb = sb; + + WARN_ON(sb->s_bdi != &noop_backing_dev_info); + sb->s_bdi = bdi_get(parent_sb->s_bdi); + + sb->s_xattr = parent_sb->s_xattr; + sb->s_time_gran = parent_sb->s_time_gran; + sb->s_blocksize = parent_sb->s_blocksize; + sb->s_blocksize_bits = parent_sb->s_blocksize_bits; + sb->s_subtype = kstrdup(parent_sb->s_subtype, GFP_KERNEL); + if (parent_sb->s_subtype && !sb->s_subtype) + return -ENOMEM; + + fuse_fill_attr_from_inode(&root_attr, &parent_fi->inode); + root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0); + /* + * This inode is just a duplicate, so it is not looked up and + * its nlookup should not be incremented. fuse_iget() does + * that, though, so undo it here. + */ + get_fuse_inode(root)->nlookup--; + sb->s_d_op = &fuse_dentry_operations; + sb->s_root = d_make_root(root); + if (!sb->s_root) + return -ENOMEM; + + return 0; +} + int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx) { - struct fuse_dev *fud; - struct fuse_conn *fc = get_fuse_conn_super(sb); + struct fuse_dev *fud = NULL; + struct fuse_mount *fm = get_fuse_mount_super(sb); + struct fuse_conn *fc = fm->fc; struct inode *root; struct dentry *root_dentry; int err; @@ -1173,7 +1530,7 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx) if (sb->s_flags & SB_MANDLOCK) goto err; - sb->s_flags &= ~(SB_NOSEC | SB_I_VERSION); + fuse_sb_defaults(sb); if (ctx->is_bdev) { #ifdef CONFIG_BLOCK @@ -1188,29 +1545,21 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx) sb->s_subtype = ctx->subtype; ctx->subtype = NULL; - sb->s_magic = FUSE_SUPER_MAGIC; - sb->s_op = &fuse_super_operations; - sb->s_xattr = fuse_xattr_handlers; - sb->s_maxbytes = MAX_LFS_FILESIZE; - sb->s_time_gran = 1; - sb->s_export_op = &fuse_export_operations; - sb->s_iflags |= SB_I_IMA_UNVERIFIABLE_SIGNATURE; - if (sb->s_user_ns != &init_user_ns) - sb->s_iflags |= SB_I_UNTRUSTED_MOUNTER; + if (IS_ENABLED(CONFIG_FUSE_DAX)) { + err = fuse_dax_conn_alloc(fc, ctx->dax_dev); + if (err) + goto err; + } - /* - * If we are not in the initial user namespace posix - * acls must be translated. - */ - if (sb->s_user_ns != &init_user_ns) - sb->s_xattr = fuse_no_acl_xattr_handlers; - - fud = fuse_dev_alloc_install(fc); - if (!fud) - goto err; + if (ctx->fudptr) { + err = -ENOMEM; + fud = fuse_dev_alloc_install(fc); + if (!fud) + goto err_free_dax; + } fc->dev = sb->s_dev; - fc->sb = sb; + fm->sb = sb; err = fuse_bdi_init(fc, sb); if (err) goto err_dev_free; @@ -1224,24 +1573,27 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx) fc->allow_other = ctx->allow_other; fc->user_id = ctx->user_id; fc->group_id = ctx->group_id; - fc->max_read = max_t(unsigned, 4096, ctx->max_read); + fc->legacy_opts_show = ctx->legacy_opts_show; + fc->max_read = max_t(unsigned int, 4096, ctx->max_read); fc->destroy = ctx->destroy; fc->no_control = ctx->no_control; fc->no_force_umount = ctx->no_force_umount; - fc->no_mount_options = ctx->no_mount_options; + fc->no_daemon = ctx->no_daemon; err = -ENOMEM; - root = fuse_get_root_inode(sb, ctx->rootmode); + root = fuse_get_root_inode(sb, ctx->rootmode, ctx->root_bpf, + ctx->root_dir); sb->s_d_op = &fuse_root_dentry_operations; root_dentry = d_make_root(root); if (!root_dentry) goto err_dev_free; + fuse_init_dentry_root(root_dentry, ctx->root_dir); /* Root dentry doesn't have .d_revalidate */ sb->s_d_op = &fuse_dentry_operations; mutex_lock(&fuse_mutex); err = -EINVAL; - if (*ctx->fudptr) + if (ctx->fudptr && *ctx->fudptr) goto err_unlock; err = fuse_ctl_add_conn(fc); @@ -1250,7 +1602,8 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx) list_add_tail(&fc->entry, &fuse_conn_list); sb->s_root = root_dentry; - *ctx->fudptr = fud; + if (ctx->fudptr) + *ctx->fudptr = fud; mutex_unlock(&fuse_mutex); return 0; @@ -1258,7 +1611,11 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx) mutex_unlock(&fuse_mutex); dput(root_dentry); err_dev_free: - fuse_dev_free(fud); + if (fud) + fuse_dev_free(fud); + err_free_dax: + if (IS_ENABLED(CONFIG_FUSE_DAX)) + fuse_dax_conn_free(fc); err: return err; } @@ -1270,29 +1627,39 @@ static int fuse_fill_super(struct super_block *sb, struct fs_context *fsc) struct file *file; int err; struct fuse_conn *fc; + struct fuse_mount *fm; err = -EINVAL; - file = fget(ctx->fd); - if (!file) - goto err; + if (!ctx->no_daemon) { + file = fget(ctx->fd); + if (!file) + goto err; - /* - * Require mount to happen from the same user namespace which - * opened /dev/fuse to prevent potential attacks. - */ - if ((file->f_op != &fuse_dev_operations) || - (file->f_cred->user_ns != sb->s_user_ns)) - goto err_fput; - ctx->fudptr = &file->private_data; + /* + * Require mount to happen from the same user namespace which + * opened /dev/fuse to prevent potential attacks. + */ + if ((file->f_op != &fuse_dev_operations) || + (file->f_cred->user_ns != sb->s_user_ns)) + goto err_fput; + ctx->fudptr = &file->private_data; + } fc = kmalloc(sizeof(*fc), GFP_KERNEL); err = -ENOMEM; if (!fc) goto err_fput; - fuse_conn_init(fc, sb->s_user_ns, &fuse_dev_fiq_ops, NULL); + fm = kzalloc(sizeof(*fm), GFP_KERNEL); + if (!fm) { + kfree(fc); + goto err_fput; + } + + fuse_conn_init(fc, fm, sb->s_user_ns, &fuse_dev_fiq_ops, NULL); fc->release = fuse_free_conn; - sb->s_fs_info = fc; + + sb->s_fs_info = fm; err = fuse_fill_super_common(sb, ctx); if (err) @@ -1302,22 +1669,24 @@ static int fuse_fill_super(struct super_block *sb, struct fs_context *fsc) * memory barrier for file->private_data to be visible on all * CPUs after this */ - fput(file); - fuse_send_init(get_fuse_conn_super(sb)); + if (!ctx->no_daemon) + fput(file); + fuse_send_init(get_fuse_mount_super(sb)); return 0; err_put_conn: - fuse_conn_put(fc); + fuse_mount_put(fm); sb->s_fs_info = NULL; err_fput: - fput(file); + if (!ctx->no_daemon) + fput(file); err: return err; } -static int fuse_get_tree(struct fs_context *fc) +static int fuse_get_tree(struct fs_context *fsc) { - struct fuse_fs_context *ctx = fc->fs_private; + struct fuse_fs_context *ctx = fsc->fs_private; if (!ctx->fd_present || !ctx->rootmode_present || !ctx->user_id_present || !ctx->group_id_present) @@ -1325,14 +1694,14 @@ static int fuse_get_tree(struct fs_context *fc) #ifdef CONFIG_BLOCK if (ctx->is_bdev) - return get_tree_bdev(fc, fuse_fill_super); + return get_tree_bdev(fsc, fuse_fill_super); #endif - return get_tree_nodev(fc, fuse_fill_super); + return get_tree_nodev(fsc, fuse_fill_super); } static const struct fs_context_operations fuse_context_ops = { - .free = fuse_free_fc, + .free = fuse_free_fsc, .parse_param = fuse_parse_param, .reconfigure = fuse_reconfigure, .get_tree = fuse_get_tree, @@ -1341,7 +1710,7 @@ static const struct fs_context_operations fuse_context_ops = { /* * Set up the filesystem mount context. */ -static int fuse_init_fs_context(struct fs_context *fc) +static int fuse_init_fs_context(struct fs_context *fsc) { struct fuse_fs_context *ctx; @@ -1351,42 +1720,66 @@ static int fuse_init_fs_context(struct fs_context *fc) ctx->max_read = ~0; ctx->blksize = FUSE_DEFAULT_BLKSIZE; + ctx->legacy_opts_show = true; #ifdef CONFIG_BLOCK - if (fc->fs_type == &fuseblk_fs_type) { + if (fsc->fs_type == &fuseblk_fs_type) { ctx->is_bdev = true; ctx->destroy = true; } #endif - fc->fs_private = ctx; - fc->ops = &fuse_context_ops; + fsc->fs_private = ctx; + fsc->ops = &fuse_context_ops; return 0; } -static void fuse_sb_destroy(struct super_block *sb) +bool fuse_mount_remove(struct fuse_mount *fm) { - struct fuse_conn *fc = get_fuse_conn_super(sb); + struct fuse_conn *fc = fm->fc; + bool last = false; - if (fc) { - if (fc->destroy) - fuse_send_destroy(fc); + down_write(&fc->killsb); + list_del_init(&fm->fc_entry); + if (list_empty(&fc->mounts)) + last = true; + up_write(&fc->killsb); - fuse_abort_conn(fc); - fuse_wait_aborted(fc); + return last; +} +EXPORT_SYMBOL_GPL(fuse_mount_remove); - down_write(&fc->killsb); - fc->sb = NULL; - up_write(&fc->killsb); +void fuse_conn_destroy(struct fuse_mount *fm) +{ + struct fuse_conn *fc = fm->fc; + + if (fc->destroy) + fuse_send_destroy(fm); + + fuse_abort_conn(fc); + fuse_wait_aborted(fc); + + if (!list_empty(&fc->entry)) { + mutex_lock(&fuse_mutex); + list_del(&fc->entry); + fuse_ctl_remove_conn(fc); + mutex_unlock(&fuse_mutex); } } +EXPORT_SYMBOL_GPL(fuse_conn_destroy); -void fuse_kill_sb_anon(struct super_block *sb) +static void fuse_kill_sb_anon(struct super_block *sb) { - fuse_sb_destroy(sb); + struct fuse_mount *fm = get_fuse_mount_super(sb); + bool last; + + if (fm) { + last = fuse_mount_remove(fm); + if (last) + fuse_conn_destroy(fm); + } kill_anon_super(sb); } -EXPORT_SYMBOL_GPL(fuse_kill_sb_anon); static struct file_system_type fuse_fs_type = { .owner = THIS_MODULE, @@ -1401,7 +1794,14 @@ MODULE_ALIAS_FS("fuse"); #ifdef CONFIG_BLOCK static void fuse_kill_sb_blk(struct super_block *sb) { - fuse_sb_destroy(sb); + struct fuse_mount *fm = get_fuse_mount_super(sb); + bool last; + + if (sb->s_root) { + last = fuse_mount_remove(fm); + if (last) + fuse_conn_destroy(fm); + } kill_block_super(sb); } @@ -1487,6 +1887,61 @@ static void fuse_fs_cleanup(void) static struct kobject *fuse_kobj; +#ifdef CONFIG_FUSE_BPF +static ssize_t fuse_bpf_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + return sysfs_emit(buff, "supported\n"); +} + +static struct kobj_attribute fuse_bpf_attr = + __ATTR_RO(fuse_bpf); +#endif + +static struct attribute *bpf_features[] = { +#ifdef CONFIG_FUSE_BPF + &fuse_bpf_attr.attr, +#endif + NULL, +}; + +static const struct attribute_group bpf_features_group = { + .name = "features", + .attrs = bpf_features, +}; + +/* + * TODO Remove this once fuse-bpf is upstreamed + * + * bpf_prog_type_fuse exports the bpf_prog_type_fuse 'constant', which cannot be + * constant until the code is upstreamed + */ +static ssize_t bpf_prog_type_fuse_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + return sysfs_emit(buff, "%d\n", BPF_PROG_TYPE_FUSE); +} + +static struct kobj_attribute bpf_prog_type_fuse_attr = + __ATTR_RO(bpf_prog_type_fuse); + +static struct attribute *bpf_attributes[] = { + &bpf_prog_type_fuse_attr.attr, + NULL, +}; + +static const struct attribute_group bpf_attr_group = { + .attrs = bpf_attributes, +}; + +static const struct attribute_group *attribute_groups[] = { + &bpf_features_group, + &bpf_attr_group, + NULL +}; + +/* TODO remove to here */ + static int fuse_sysfs_init(void) { int err; @@ -1501,8 +1956,15 @@ static int fuse_sysfs_init(void) if (err) goto out_fuse_unregister; + /* TODO Remove when BPF_PROG_TYPE_FUSE is upstreamed */ + err = sysfs_create_groups(fuse_kobj, attribute_groups); + if (err) + goto out_fuse_remove_mount_point; + return 0; + out_fuse_remove_mount_point: + sysfs_remove_mount_point(fuse_kobj, "connections"); out_fuse_unregister: kobject_put(fuse_kobj); out_err: @@ -1511,6 +1973,7 @@ static int fuse_sysfs_init(void) static void fuse_sysfs_cleanup(void) { + sysfs_remove_groups(fuse_kobj, attribute_groups); sysfs_remove_mount_point(fuse_kobj, "connections"); kobject_put(fuse_kobj); } @@ -1539,11 +2002,21 @@ static int __init fuse_init(void) if (res) goto err_sysfs_cleanup; +#ifdef CONFIG_FUSE_BPF + res = fuse_bpf_init(); + if (res) + goto err_ctl_cleanup; +#endif + sanitize_global_limit(&max_user_bgreq); sanitize_global_limit(&max_user_congthresh); return 0; +#ifdef CONFIG_FUSE_BPF + err_ctl_cleanup: + fuse_ctl_cleanup(); +#endif err_sysfs_cleanup: fuse_sysfs_cleanup(); err_dev_cleanup: @@ -1561,6 +2034,9 @@ static void __exit fuse_exit(void) fuse_ctl_cleanup(); fuse_sysfs_cleanup(); fuse_fs_cleanup(); +#ifdef CONFIG_FUSE_BPF + fuse_bpf_cleanup(); +#endif fuse_dev_cleanup(); } diff --git a/fs/fuse/passthrough.c b/fs/fuse/passthrough.c index 95368ddfb53a..1845e05db2da 100644 --- a/fs/fuse/passthrough.c +++ b/fs/fuse/passthrough.c @@ -14,18 +14,6 @@ struct fuse_aio_req { struct kiocb *iocb_fuse; }; -static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src, - struct file *filp) -{ - *kiocb = (struct kiocb){ - .ki_filp = filp, - .ki_flags = kiocb_src->ki_flags, - .ki_hint = kiocb_src->ki_hint, - .ki_ioprio = kiocb_src->ki_ioprio, - .ki_pos = kiocb_src->ki_pos, - }; -} - static void fuse_file_accessed(struct file *dst_file, struct file *src_file) { struct inode *dst_inode; @@ -46,7 +34,7 @@ static void fuse_file_accessed(struct file *dst_file, struct file *src_file) touch_atime(&dst_file->f_path); } -static void fuse_copyattr(struct file *dst_file, struct file *src_file) +void fuse_copyattr(struct file *dst_file, struct file *src_file) { struct inode *dst = file_inode(dst_file); struct inode *src = file_inode(src_file); @@ -225,7 +213,8 @@ int fuse_passthrough_open(struct fuse_dev *fud, u32 lower_fd) } if (!passthrough_filp->f_op->read_iter || - !passthrough_filp->f_op->write_iter) { + !((passthrough_filp->f_path.mnt->mnt_flags | MNT_READONLY) || + passthrough_filp->f_op->write_iter)) { pr_err("FUSE: passthrough file misses file operations.\n"); res = -EBADF; goto err_free_file; diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 512609da8590..f9d0be56d34a 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -20,6 +20,8 @@ static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) if (!fc->do_readdirplus) return false; + if (fi->nodeid == 0) + return false; if (!fc->readdirplus_auto) return true; if (test_and_clear_bit(FUSE_I_ADVISE_RDPLUS, &fi->state)) @@ -77,8 +79,10 @@ static void fuse_add_dirent_to_cache(struct file *file, goto unlock; addr = kmap_atomic(page); - if (!offset) + if (!offset) { clear_page(addr); + SetPageUptodate(page); + } memcpy(addr + offset, dirent, reclen); kunmap_atomic(addr); fi->rdc.size = (index << PAGE_SHIFT) + offset + reclen; @@ -200,9 +204,12 @@ retry: if (!d_in_lookup(dentry)) { struct fuse_inode *fi; inode = d_inode(dentry); + if (inode && get_node_id(inode) != o->nodeid) + inode = NULL; if (!inode || - get_node_id(inode) != o->nodeid || - ((o->attr.mode ^ inode->i_mode) & S_IFMT)) { + fuse_stale_inode(inode, o->generation, &o->attr)) { + if (inode) + fuse_make_bad(inode); d_invalidate(dentry); dput(dentry); goto retry; @@ -260,7 +267,7 @@ retry: static void fuse_force_forget(struct file *file, u64 nodeid) { struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_forget_in inarg; FUSE_ARGS(args); @@ -274,7 +281,7 @@ static void fuse_force_forget(struct file *file, u64 nodeid) args.force = true; args.noreply = true; - fuse_simple_request(fc, &args); + fuse_simple_request(fm, &args); /* ignore errors */ } @@ -328,7 +335,7 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) ssize_t res; struct page *page; struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_io_args ia = {}; struct fuse_args_pages *ap = &ia.ap; struct fuse_page_desc desc = { .length = PAGE_SIZE }; @@ -340,12 +347,12 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) return -ENOMEM; plus = fuse_use_readdirplus(inode, ctx); - ap->args.out_pages = 1; + ap->args.out_pages = true; ap->num_pages = 1; ap->pages = &page; ap->descs = &desc; if (plus) { - attr_version = fuse_get_attr_version(fc); + attr_version = fuse_get_attr_version(fm->fc); fuse_read_args_fill(&ia, file, ctx->pos, PAGE_SIZE, FUSE_READDIRPLUS); } else { @@ -353,7 +360,7 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) FUSE_READDIR); } locked = fuse_lock_inode(inode); - res = fuse_simple_request(fc, &ap->args); + res = fuse_simple_request(fm, &ap->args); fuse_unlock_inode(inode, locked); if (res >= 0) { if (!res) { @@ -521,6 +528,12 @@ retry_locked: page = find_get_page_flags(file->f_mapping, index, FGP_ACCESSED | FGP_LOCK); + /* Page gone missing, then re-added to cache, but not initialized? */ + if (page && !PageUptodate(page)) { + unlock_page(page); + put_page(page); + page = NULL; + } spin_lock(&fi->rdc.lock); if (!page) { /* @@ -576,6 +589,26 @@ int fuse_readdir(struct file *file, struct dir_context *ctx) struct inode *inode = file_inode(file); int err; +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + bool allow_force; + bool force_again = false; + bool is_continued = false; + +again: + fer = fuse_bpf_backing(inode, struct fuse_read_io, + fuse_readdir_initialize, fuse_readdir_backing, + fuse_readdir_finalize, + file, ctx, &force_again, &allow_force, is_continued); + if (force_again && !IS_ERR(fer.result)) { + is_continued = true; + goto again; + } + + if (fer.ret) + return PTR_ERR(fer.result); +#endif + if (fuse_is_bad(inode)) return -EIO; diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index 97259c6afac8..c23a94d179fd 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -5,14 +5,25 @@ */ #include +#include +#include +#include #include #include #include #include #include +#include #include +#include #include "fuse_i.h" +/* Used to help calculate the FUSE connection's max_pages limit for a request's + * size. Parts of the struct fuse_req are sliced into scattergather lists in + * addition to the pages used, so this can help account for that overhead. + */ +#define FUSE_HEADER_OVERHEAD 4 + /* List of virtio-fs device instances and a lock for the list. Also provides * mutual exclusion in device removal and mounting path */ @@ -24,6 +35,8 @@ enum { VQ_REQUEST }; +#define VQ_NAME_LEN 24 + /* Per-virtqueue state */ struct virtio_fs_vq { spinlock_t lock; @@ -35,7 +48,8 @@ struct virtio_fs_vq { struct fuse_dev *fud; bool connected; long in_flight; - char name[24]; + struct completion in_flight_zero; /* No inflight requests */ + char name[VQ_NAME_LEN]; } ____cacheline_aligned_in_smp; /* A virtio-fs device instance */ @@ -46,13 +60,23 @@ struct virtio_fs { struct virtio_fs_vq *vqs; unsigned int nvqs; /* number of virtqueues */ unsigned int num_request_queues; /* number of request queues */ + struct dax_device *dax_dev; + + /* DAX memory window where file contents are mapped */ + void *window_kaddr; + phys_addr_t window_phys_addr; + size_t window_len; +}; + +struct virtio_fs_forget_req { + struct fuse_in_header ih; + struct fuse_forget_in arg; }; struct virtio_fs_forget { - struct fuse_in_header ih; - struct fuse_forget_in arg; /* This request can be temporarily queued on virt queue */ struct list_head list; + struct virtio_fs_forget_req req; }; struct virtio_fs_req_work { @@ -64,6 +88,48 @@ struct virtio_fs_req_work { static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, struct fuse_req *req, bool in_flight); +enum { + OPT_DAX, +}; + +static const struct fs_parameter_spec virtio_param_specs[] = { + fsparam_flag("dax", OPT_DAX), + {} +}; + +static const struct fs_parameter_description virtio_fs_parameters = { + .specs = virtio_param_specs, +}; + +static int virtio_fs_parse_param(struct fs_context *fsc, + struct fs_parameter *param) +{ + struct fs_parse_result result; + struct fuse_fs_context *ctx = fsc->fs_private; + int opt; + + opt = fs_parse(fsc, &virtio_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case OPT_DAX: + ctx->dax = 1; + break; + default: + return -EINVAL; + } + + return 0; +} + +static void virtio_fs_free_fsc(struct fs_context *fsc) +{ + struct fuse_fs_context *ctx = fsc->fs_private; + + kfree(ctx); +} + static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq) { struct virtio_fs *fs = vq->vdev->priv; @@ -87,6 +153,8 @@ static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq) { WARN_ON(fsvq->in_flight <= 0); fsvq->in_flight--; + if (!fsvq->in_flight) + complete(&fsvq->in_flight_zero); } static void release_virtio_fs_obj(struct kref *ref) @@ -117,22 +185,23 @@ static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) WARN_ON(fsvq->in_flight < 0); /* Wait for in flight requests to finish.*/ - while (1) { - spin_lock(&fsvq->lock); - if (!fsvq->in_flight) { - spin_unlock(&fsvq->lock); - break; - } + spin_lock(&fsvq->lock); + if (fsvq->in_flight) { + /* We are holding virtio_fs_mutex. There should not be any + * waiters waiting for completion. + */ + reinit_completion(&fsvq->in_flight_zero); + spin_unlock(&fsvq->lock); + wait_for_completion(&fsvq->in_flight_zero); + } else { spin_unlock(&fsvq->lock); - /* TODO use completion instead of timeout */ - usleep_range(1000, 2000); } flush_work(&fsvq->done_work); flush_delayed_work(&fsvq->dispatch_work); } -static void virtio_fs_drain_all_queues(struct virtio_fs *fs) +static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs) { struct virtio_fs_vq *fsvq; int i; @@ -143,6 +212,19 @@ static void virtio_fs_drain_all_queues(struct virtio_fs *fs) } } +static void virtio_fs_drain_all_queues(struct virtio_fs *fs) +{ + /* Provides mutual exclusion between ->remove and ->kill_sb + * paths. We don't want both of these draining queue at the + * same time. Current completion logic reinits completion + * and that means there should not be any other thread + * doing reinit or waiting for completion already. + */ + mutex_lock(&virtio_fs_mutex); + virtio_fs_drain_all_queues_locked(fs); + mutex_unlock(&virtio_fs_mutex); +} + static void virtio_fs_start_all_queues(struct virtio_fs *fs) { struct virtio_fs_vq *fsvq; @@ -278,7 +360,6 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) struct fuse_req *req; struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, dispatch_work.work); - struct fuse_conn *fc = fsvq->fud->fc; int ret; pr_debug("virtio-fs: worker %s called.\n", __func__); @@ -293,7 +374,7 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) list_del_init(&req->list); spin_unlock(&fsvq->lock); - fuse_request_end(fc, req); + fuse_request_end(req); } /* Dispatch pending requests */ @@ -324,22 +405,77 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work) spin_unlock(&fsvq->lock); pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); - fuse_request_end(fc, req); + fuse_request_end(req); } } } +/* + * Returns 1 if queue is full and sender should wait a bit before sending + * next request, 0 otherwise. + */ +static int send_forget_request(struct virtio_fs_vq *fsvq, + struct virtio_fs_forget *forget, + bool in_flight) +{ + struct scatterlist sg; + struct virtqueue *vq; + int ret = 0; + bool notify; + struct virtio_fs_forget_req *req = &forget->req; + + spin_lock(&fsvq->lock); + if (!fsvq->connected) { + if (in_flight) + dec_in_flight_req(fsvq); + kfree(forget); + goto out; + } + + sg_init_one(&sg, req, sizeof(*req)); + vq = fsvq->vq; + dev_dbg(&vq->vdev->dev, "%s\n", __func__); + + ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC); + if (ret < 0) { + if (ret == -ENOMEM || ret == -ENOSPC) { + pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", + ret); + list_add_tail(&forget->list, &fsvq->queued_reqs); + schedule_delayed_work(&fsvq->dispatch_work, + msecs_to_jiffies(1)); + if (!in_flight) + inc_in_flight_req(fsvq); + /* Queue is full */ + ret = 1; + } else { + pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", + ret); + kfree(forget); + if (in_flight) + dec_in_flight_req(fsvq); + } + goto out; + } + + if (!in_flight) + inc_in_flight_req(fsvq); + notify = virtqueue_kick_prepare(vq); + spin_unlock(&fsvq->lock); + + if (notify) + virtqueue_notify(vq); + return ret; +out: + spin_unlock(&fsvq->lock); + return ret; +} + static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) { struct virtio_fs_forget *forget; struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, dispatch_work.work); - struct virtqueue *vq = fsvq->vq; - struct scatterlist sg; - struct scatterlist *sgs[] = {&sg}; - bool notify; - int ret; - pr_debug("virtio-fs: worker %s called.\n", __func__); while (1) { spin_lock(&fsvq->lock); @@ -351,43 +487,9 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) } list_del(&forget->list); - if (!fsvq->connected) { - dec_in_flight_req(fsvq); - spin_unlock(&fsvq->lock); - kfree(forget); - continue; - } - - sg_init_one(&sg, forget, sizeof(*forget)); - - /* Enqueue the request */ - dev_dbg(&vq->vdev->dev, "%s\n", __func__); - ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC); - if (ret < 0) { - if (ret == -ENOMEM || ret == -ENOSPC) { - pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", - ret); - list_add_tail(&forget->list, - &fsvq->queued_reqs); - schedule_delayed_work(&fsvq->dispatch_work, - msecs_to_jiffies(1)); - } else { - pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", - ret); - dec_in_flight_req(fsvq); - kfree(forget); - } - spin_unlock(&fsvq->lock); - return; - } - - notify = virtqueue_kick_prepare(vq); spin_unlock(&fsvq->lock); - - if (notify) - virtqueue_notify(vq); - pr_debug("virtio-fs: worker %s dispatched one forget request.\n", - __func__); + if (send_forget_request(fsvq, forget, true)) + return; } } @@ -463,7 +565,6 @@ static void virtio_fs_request_complete(struct fuse_req *req, struct virtio_fs_vq *fsvq) { struct fuse_pqueue *fpq = &fsvq->fud->pq; - struct fuse_conn *fc = fsvq->fud->fc; struct fuse_args *args; struct fuse_args_pages *ap; unsigned int len, i, thislen; @@ -496,7 +597,7 @@ static void virtio_fs_request_complete(struct fuse_req *req, clear_bit(FR_SENT, &req->flags); spin_unlock(&fpq->lock); - fuse_request_end(fc, req); + fuse_request_end(req); spin_lock(&fsvq->lock); dec_in_flight_req(fsvq); spin_unlock(&fsvq->lock); @@ -564,6 +665,26 @@ static void virtio_fs_vq_done(struct virtqueue *vq) schedule_work(&fsvq->done_work); } +static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name, + int vq_type) +{ + strncpy(fsvq->name, name, VQ_NAME_LEN); + spin_lock_init(&fsvq->lock); + INIT_LIST_HEAD(&fsvq->queued_reqs); + INIT_LIST_HEAD(&fsvq->end_reqs); + init_completion(&fsvq->in_flight_zero); + + if (vq_type == VQ_REQUEST) { + INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work); + INIT_DELAYED_WORK(&fsvq->dispatch_work, + virtio_fs_request_dispatch_work); + } else { + INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work); + INIT_DELAYED_WORK(&fsvq->dispatch_work, + virtio_fs_hiprio_dispatch_work); + } +} + /* Initialize virtqueues */ static int virtio_fs_setup_vqs(struct virtio_device *vdev, struct virtio_fs *fs) @@ -574,12 +695,12 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, unsigned int i; int ret = 0; - virtio_cread(vdev, struct virtio_fs_config, num_request_queues, - &fs->num_request_queues); + virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues, + &fs->num_request_queues); if (fs->num_request_queues == 0) return -EINVAL; - fs->nvqs = 1 + fs->num_request_queues; + fs->nvqs = VQ_REQUEST + fs->num_request_queues; fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL); if (!fs->vqs) return -ENOMEM; @@ -593,27 +714,17 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, goto out; } + /* Initialize the hiprio/forget request virtqueue */ callbacks[VQ_HIPRIO] = virtio_fs_vq_done; - snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name), - "hiprio"); + virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO); names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name; - INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work); - INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs); - INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs); - INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work, - virtio_fs_hiprio_dispatch_work); - spin_lock_init(&fs->vqs[VQ_HIPRIO].lock); /* Initialize the requests virtqueues */ for (i = VQ_REQUEST; i < fs->nvqs; i++) { - spin_lock_init(&fs->vqs[i].lock); - INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work); - INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work, - virtio_fs_request_dispatch_work); - INIT_LIST_HEAD(&fs->vqs[i].queued_reqs); - INIT_LIST_HEAD(&fs->vqs[i].end_reqs); - snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name), - "requests.%u", i - VQ_REQUEST); + char vq_name[VQ_NAME_LEN]; + + snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST); + virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST); callbacks[i] = virtio_fs_vq_done; names[i] = fs->vqs[i].name; } @@ -642,6 +753,130 @@ static void virtio_fs_cleanup_vqs(struct virtio_device *vdev, vdev->config->del_vqs(vdev); } +/* Map a window offset to a page frame number. The window offset will have + * been produced by .iomap_begin(), which maps a file offset to a window + * offset. + */ +static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, + long nr_pages, void **kaddr, pfn_t *pfn) +{ + struct virtio_fs *fs = dax_get_private(dax_dev); + phys_addr_t offset = PFN_PHYS(pgoff); + size_t max_nr_pages = fs->window_len/PAGE_SIZE - pgoff; + + if (kaddr) + *kaddr = fs->window_kaddr + offset; + if (pfn) + *pfn = phys_to_pfn_t(fs->window_phys_addr + offset, + PFN_DEV | PFN_MAP); + return nr_pages > max_nr_pages ? max_nr_pages : nr_pages; +} + +static size_t virtio_fs_copy_from_iter(struct dax_device *dax_dev, + pgoff_t pgoff, void *addr, + size_t bytes, struct iov_iter *i) +{ + return copy_from_iter(addr, bytes, i); +} + +static size_t virtio_fs_copy_to_iter(struct dax_device *dax_dev, + pgoff_t pgoff, void *addr, + size_t bytes, struct iov_iter *i) +{ + return copy_to_iter(addr, bytes, i); +} + +static int virtio_fs_zero_page_range(struct dax_device *dax_dev, + pgoff_t pgoff, size_t nr_pages) +{ + long rc; + void *kaddr; + + rc = dax_direct_access(dax_dev, pgoff, nr_pages, &kaddr, NULL); + if (rc < 0) + return rc; + memset(kaddr, 0, nr_pages << PAGE_SHIFT); + dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT); + return 0; +} + +static const struct dax_operations virtio_fs_dax_ops = { + .direct_access = virtio_fs_direct_access, + .copy_from_iter = virtio_fs_copy_from_iter, + .copy_to_iter = virtio_fs_copy_to_iter, + .zero_page_range = virtio_fs_zero_page_range, +}; + +static void virtio_fs_cleanup_dax(void *data) +{ + struct dax_device *dax_dev = data; + + kill_dax(dax_dev); + put_dax(dax_dev); +} + +static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs) +{ + struct virtio_shm_region cache_reg; + struct dev_pagemap *pgmap; + bool have_cache; + + if (!IS_ENABLED(CONFIG_FUSE_DAX)) + return 0; + + /* Get cache region */ + have_cache = virtio_get_shm_region(vdev, &cache_reg, + (u8)VIRTIO_FS_SHMCAP_ID_CACHE); + if (!have_cache) { + dev_notice(&vdev->dev, "%s: No cache capability\n", __func__); + return 0; + } + + if (!devm_request_mem_region(&vdev->dev, cache_reg.addr, cache_reg.len, + dev_name(&vdev->dev))) { + dev_warn(&vdev->dev, "could not reserve region addr=0x%llx len=0x%llx\n", + cache_reg.addr, cache_reg.len); + return -EBUSY; + } + + dev_notice(&vdev->dev, "Cache len: 0x%llx @ 0x%llx\n", cache_reg.len, + cache_reg.addr); + + pgmap = devm_kzalloc(&vdev->dev, sizeof(*pgmap), GFP_KERNEL); + if (!pgmap) + return -ENOMEM; + + pgmap->type = MEMORY_DEVICE_FS_DAX; + + /* Ideally we would directly use the PCI BAR resource but + * devm_memremap_pages() wants its own copy in pgmap. So + * initialize a struct resource from scratch (only the start + * and end fields will be used). + */ + pgmap->res = (struct resource){ + .name = "virtio-fs dax window", + .start = (phys_addr_t) cache_reg.addr, + .end = (phys_addr_t) cache_reg.addr + cache_reg.len - 1, + }; + + fs->window_kaddr = devm_memremap_pages(&vdev->dev, pgmap); + if (IS_ERR(fs->window_kaddr)) + return PTR_ERR(fs->window_kaddr); + + fs->window_phys_addr = (phys_addr_t) cache_reg.addr; + fs->window_len = (phys_addr_t) cache_reg.len; + + dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n", + __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len); + + fs->dax_dev = alloc_dax(fs, NULL, &virtio_fs_dax_ops, 0); + if (IS_ERR(fs->dax_dev)) + return PTR_ERR(fs->dax_dev); + + return devm_add_action_or_reset(&vdev->dev, virtio_fs_cleanup_dax, + fs->dax_dev); +} + static int virtio_fs_probe(struct virtio_device *vdev) { struct virtio_fs *fs; @@ -663,6 +898,10 @@ static int virtio_fs_probe(struct virtio_device *vdev) /* TODO vq affinity */ + ret = virtio_fs_setup_dax(vdev, fs); + if (ret < 0) + goto out_vqs; + /* Bring the device online in case the filesystem is mounted and * requests need to be sent before we return. */ @@ -706,7 +945,7 @@ static void virtio_fs_remove(struct virtio_device *vdev) /* This device is going away. No one should get new reference */ list_del_init(&fs->list); virtio_fs_stop_all_queues(fs); - virtio_fs_drain_all_queues(fs); + virtio_fs_drain_all_queues_locked(fs); vdev->config->reset(vdev); virtio_fs_cleanup_vqs(vdev, fs); @@ -731,12 +970,12 @@ static int virtio_fs_restore(struct virtio_device *vdev) } #endif /* CONFIG_PM_SLEEP */ -const static struct virtio_device_id id_table[] = { +static const struct virtio_device_id id_table[] = { { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID }, {}, }; -const static unsigned int feature_table[] = {}; +static const unsigned int feature_table[] = {}; static struct virtio_driver virtio_fs_driver = { .driver.name = KBUILD_MODNAME, @@ -757,14 +996,10 @@ __releases(fiq->lock) { struct fuse_forget_link *link; struct virtio_fs_forget *forget; - struct scatterlist sg; - struct scatterlist *sgs[] = {&sg}; + struct virtio_fs_forget_req *req; struct virtio_fs *fs; - struct virtqueue *vq; struct virtio_fs_vq *fsvq; - bool notify; u64 unique; - int ret; link = fuse_dequeue_forget(fiq, 1, NULL); unique = fuse_get_unique(fiq); @@ -775,57 +1010,19 @@ __releases(fiq->lock) /* Allocate a buffer for the request */ forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL); + req = &forget->req; - forget->ih = (struct fuse_in_header){ + req->ih = (struct fuse_in_header){ .opcode = FUSE_FORGET, .nodeid = link->forget_one.nodeid, .unique = unique, - .len = sizeof(*forget), + .len = sizeof(*req), }; - forget->arg = (struct fuse_forget_in){ + req->arg = (struct fuse_forget_in){ .nlookup = link->forget_one.nlookup, }; - sg_init_one(&sg, forget, sizeof(*forget)); - - /* Enqueue the request */ - spin_lock(&fsvq->lock); - - if (!fsvq->connected) { - kfree(forget); - spin_unlock(&fsvq->lock); - goto out; - } - - vq = fsvq->vq; - dev_dbg(&vq->vdev->dev, "%s\n", __func__); - - ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC); - if (ret < 0) { - if (ret == -ENOMEM || ret == -ENOSPC) { - pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later.\n", - ret); - list_add_tail(&forget->list, &fsvq->queued_reqs); - schedule_delayed_work(&fsvq->dispatch_work, - msecs_to_jiffies(1)); - inc_in_flight_req(fsvq); - } else { - pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", - ret); - kfree(forget); - } - spin_unlock(&fsvq->lock); - goto out; - } - - inc_in_flight_req(fsvq); - notify = virtqueue_kick_prepare(vq); - - spin_unlock(&fsvq->lock); - - if (notify) - virtqueue_notify(vq); -out: + send_forget_request(fsvq, forget, false); kfree(link); } @@ -842,18 +1039,37 @@ __releases(fiq->lock) spin_unlock(&fiq->lock); } +/* Count number of scatter-gather elements required */ +static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs, + unsigned int num_pages, + unsigned int total_len) +{ + unsigned int i; + unsigned int this_len; + + for (i = 0; i < num_pages && total_len; i++) { + this_len = min(page_descs[i].length, total_len); + total_len -= this_len; + } + + return i; +} + /* Return the number of scatter-gather list elements required */ static unsigned int sg_count_fuse_req(struct fuse_req *req) { struct fuse_args *args = req->args; struct fuse_args_pages *ap = container_of(args, typeof(*ap), args); - unsigned int total_sgs = 1 /* fuse_in_header */; + unsigned int size, total_sgs = 1 /* fuse_in_header */; if (args->in_numargs - args->in_pages) total_sgs += 1; - if (args->in_pages) - total_sgs += ap->num_pages; + if (args->in_pages) { + size = args->in_args[args->in_numargs - 1].size; + total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, + size); + } if (!test_bit(FR_ISREPLY, &req->flags)) return total_sgs; @@ -863,8 +1079,11 @@ static unsigned int sg_count_fuse_req(struct fuse_req *req) if (args->out_numargs - args->out_pages) total_sgs += 1; - if (args->out_pages) - total_sgs += ap->num_pages; + if (args->out_pages) { + size = args->out_args[args->out_numargs - 1].size; + total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, + size); + } return total_sgs; } @@ -1073,31 +1292,35 @@ __releases(fiq->lock) } } -const static struct fuse_iqueue_ops virtio_fs_fiq_ops = { +static const struct fuse_iqueue_ops virtio_fs_fiq_ops = { .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock, .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock, .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock, .release = virtio_fs_fiq_release, }; -static int virtio_fs_fill_super(struct super_block *sb) +static inline void virtio_fs_ctx_set_defaults(struct fuse_fs_context *ctx) { - struct fuse_conn *fc = get_fuse_conn_super(sb); + ctx->rootmode = S_IFDIR; + ctx->default_permissions = 1; + ctx->allow_other = 1; + ctx->max_read = UINT_MAX; + ctx->blksize = 512; + ctx->destroy = true; + ctx->no_control = true; + ctx->no_force_umount = true; +} + +static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + struct fuse_conn *fc = fm->fc; struct virtio_fs *fs = fc->iq.priv; + struct fuse_fs_context *ctx = fsc->fs_private; unsigned int i; int err; - struct fuse_fs_context ctx = { - .rootmode = S_IFDIR, - .default_permissions = 1, - .allow_other = 1, - .max_read = UINT_MAX, - .blksize = 512, - .destroy = true, - .no_control = true, - .no_force_umount = true, - .no_mount_options = true, - }; + virtio_fs_ctx_set_defaults(ctx); mutex_lock(&virtio_fs_mutex); /* After holding mutex, make sure virtiofs device is still there. @@ -1112,7 +1335,7 @@ static int virtio_fs_fill_super(struct super_block *sb) err = -ENOMEM; /* Allocate fuse_dev for hiprio and notification queues */ - for (i = 0; i < VQ_REQUEST; i++) { + for (i = 0; i < fs->nvqs; i++) { struct virtio_fs_vq *fsvq = &fs->vqs[i]; fsvq->fud = fuse_dev_alloc(); @@ -1120,24 +1343,30 @@ static int virtio_fs_fill_super(struct super_block *sb) goto err_free_fuse_devs; } - ctx.fudptr = (void **)&fs->vqs[VQ_REQUEST].fud; - err = fuse_fill_super_common(sb, &ctx); + /* virtiofs allocates and installs its own fuse devices */ + ctx->fudptr = NULL; + if (ctx->dax) { + if (!fs->dax_dev) { + err = -EINVAL; + pr_err("virtio-fs: dax can't be enabled as filesystem" + " device does not support it.\n"); + goto err_free_fuse_devs; + } + ctx->dax_dev = fs->dax_dev; + } + err = fuse_fill_super_common(sb, ctx); if (err < 0) goto err_free_fuse_devs; - fc = fs->vqs[VQ_REQUEST].fud->fc; - for (i = 0; i < fs->nvqs; i++) { struct virtio_fs_vq *fsvq = &fs->vqs[i]; - if (i == VQ_REQUEST) - continue; /* already initialized */ fuse_dev_install(fsvq->fud, fc); } /* Previous unmount will stop all queues. Start these again */ virtio_fs_start_all_queues(fs); - fuse_send_init(fc); + fuse_send_init(fm); mutex_unlock(&virtio_fs_mutex); return 0; @@ -1148,18 +1377,17 @@ err: return err; } -static void virtio_kill_sb(struct super_block *sb) +static void virtio_fs_conn_destroy(struct fuse_mount *fm) { - struct fuse_conn *fc = get_fuse_conn_super(sb); - struct virtio_fs *vfs; - struct virtio_fs_vq *fsvq; + struct fuse_conn *fc = fm->fc; + struct virtio_fs *vfs = fc->iq.priv; + struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO]; - /* If mount failed, we can still be called without any fc */ - if (!fc) - return fuse_kill_sb_anon(sb); - - vfs = fc->iq.priv; - fsvq = &vfs->vqs[VQ_HIPRIO]; + /* Stop dax worker. Soon evict_inodes() will be called which + * will free all memory ranges belonging to all inodes. + */ + if (IS_ENABLED(CONFIG_FUSE_DAX)) + fuse_dax_cancel_work(fc); /* Stop forget queue. Soon destroy will be sent */ spin_lock(&fsvq->lock); @@ -1167,9 +1395,9 @@ static void virtio_kill_sb(struct super_block *sb) spin_unlock(&fsvq->lock); virtio_fs_drain_all_queues(vfs); - fuse_kill_sb_anon(sb); + fuse_conn_destroy(fm); - /* fuse_kill_sb_anon() must have sent destroy. Stop all queues + /* fuse_conn_destroy() must have sent destroy. Stop all queues * and drain one more time and free fuse devices. Freeing fuse * devices will drop their reference on fuse_conn and that in * turn will drop its reference on virtio_fs object. @@ -1179,12 +1407,27 @@ static void virtio_kill_sb(struct super_block *sb) virtio_fs_free_devs(vfs); } +static void virtio_kill_sb(struct super_block *sb) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + bool last; + + /* If mount failed, we can still be called without any fc */ + if (sb->s_root) { + last = fuse_mount_remove(fm); + if (last) + virtio_fs_conn_destroy(fm); + } + kill_anon_super(sb); +} + static int virtio_fs_test_super(struct super_block *sb, struct fs_context *fsc) { - struct fuse_conn *fc = fsc->s_fs_info; + struct fuse_mount *fsc_fm = fsc->s_fs_info; + struct fuse_mount *sb_fm = get_fuse_mount_super(sb); - return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv; + return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv; } static int virtio_fs_set_super(struct super_block *sb, @@ -1194,7 +1437,7 @@ static int virtio_fs_set_super(struct super_block *sb, err = get_anon_bdev(&sb->s_dev); if (!err) - fuse_conn_get(fsc->s_fs_info); + fuse_mount_get(fsc->s_fs_info); return err; } @@ -1203,8 +1446,10 @@ static int virtio_fs_get_tree(struct fs_context *fsc) { struct virtio_fs *fs; struct super_block *sb; - struct fuse_conn *fc; - int err; + struct fuse_conn *fc = NULL; + struct fuse_mount *fm; + unsigned int virtqueue_size; + int err = -EIO; if (!fsc->source) return invalf(fsc, "No source specified"); @@ -1219,28 +1464,39 @@ static int virtio_fs_get_tree(struct fs_context *fsc) return -EINVAL; } - fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL); - if (!fc) { - mutex_lock(&virtio_fs_mutex); - virtio_fs_put(fs); - mutex_unlock(&virtio_fs_mutex); - return -ENOMEM; - } + virtqueue_size = virtqueue_get_vring_size(fs->vqs[VQ_REQUEST].vq); + if (WARN_ON(virtqueue_size <= FUSE_HEADER_OVERHEAD)) + goto out_err; - fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops, - fs); + err = -ENOMEM; + fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL); + if (!fc) + goto out_err; + + fm = kzalloc(sizeof(struct fuse_mount), GFP_KERNEL); + if (!fm) + goto out_err; + + fuse_conn_init(fc, fm, fsc->user_ns, &virtio_fs_fiq_ops, fs); fc->release = fuse_free_conn; fc->delete_stale = true; + fc->auto_submounts = true; - fsc->s_fs_info = fc; + /* Tell FUSE to split requests that exceed the virtqueue's size */ + fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, + virtqueue_size - FUSE_HEADER_OVERHEAD); + + fsc->s_fs_info = fm; sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super); - fuse_conn_put(fc); + fuse_mount_put(fm); if (IS_ERR(sb)) return PTR_ERR(sb); if (!sb->s_root) { - err = virtio_fs_fill_super(sb); + err = virtio_fs_fill_super(sb, fsc); if (err) { + fuse_mount_put(fm); + sb->s_fs_info = NULL; deactivate_locked_super(sb); return err; } @@ -1251,14 +1507,29 @@ static int virtio_fs_get_tree(struct fs_context *fsc) WARN_ON(fsc->root); fsc->root = dget(sb->s_root); return 0; + +out_err: + kfree(fc); + mutex_lock(&virtio_fs_mutex); + virtio_fs_put(fs); + mutex_unlock(&virtio_fs_mutex); + return err; } static const struct fs_context_operations virtio_fs_context_ops = { + .free = virtio_fs_free_fsc, + .parse_param = virtio_fs_parse_param, .get_tree = virtio_fs_get_tree, }; static int virtio_fs_init_fs_context(struct fs_context *fsc) { + struct fuse_fs_context *ctx; + + ctx = kzalloc(sizeof(struct fuse_fs_context), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + fsc->fs_private = ctx; fsc->ops = &virtio_fs_context_ops; return 0; } diff --git a/fs/fuse/xattr.c b/fs/fuse/xattr.c index 9d3f28b1f6f8..3abdf0e2be89 100644 --- a/fs/fuse/xattr.c +++ b/fs/fuse/xattr.c @@ -14,12 +14,12 @@ int fuse_setxattr(struct inode *inode, const char *name, const void *value, size_t size, int flags) { - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); struct fuse_setxattr_in inarg; int err; - if (fc->no_setxattr) + if (fm->fc->no_setxattr) return -EOPNOTSUPP; memset(&inarg, 0, sizeof(inarg)); @@ -34,9 +34,9 @@ int fuse_setxattr(struct inode *inode, const char *name, const void *value, args.in_args[1].value = name; args.in_args[2].size = size; args.in_args[2].value = value; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err == -ENOSYS) { - fc->no_setxattr = 1; + fm->fc->no_setxattr = 1; err = -EOPNOTSUPP; } if (!err) { @@ -49,13 +49,13 @@ int fuse_setxattr(struct inode *inode, const char *name, const void *value, ssize_t fuse_getxattr(struct inode *inode, const char *name, void *value, size_t size) { - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); struct fuse_getxattr_in inarg; struct fuse_getxattr_out outarg; ssize_t ret; - if (fc->no_getxattr) + if (fm->fc->no_getxattr) return -EOPNOTSUPP; memset(&inarg, 0, sizeof(inarg)); @@ -77,11 +77,11 @@ ssize_t fuse_getxattr(struct inode *inode, const char *name, void *value, args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; } - ret = fuse_simple_request(fc, &args); + ret = fuse_simple_request(fm, &args); if (!ret && !size) ret = min_t(size_t, outarg.size, XATTR_SIZE_MAX); if (ret == -ENOSYS) { - fc->no_getxattr = 1; + fm->fc->no_getxattr = 1; ret = -EOPNOTSUPP; } return ret; @@ -107,19 +107,30 @@ static int fuse_verify_xattr_list(char *list, size_t size) ssize_t fuse_listxattr(struct dentry *entry, char *list, size_t size) { struct inode *inode = d_inode(entry); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); struct fuse_getxattr_in inarg; struct fuse_getxattr_out outarg; ssize_t ret; +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_getxattr_io, + fuse_listxattr_initialize, + fuse_listxattr_backing, fuse_listxattr_finalize, + entry, list, size); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + if (fuse_is_bad(inode)) return -EIO; - if (!fuse_allow_current_process(fc)) + if (!fuse_allow_current_process(fm->fc)) return -EACCES; - if (fc->no_listxattr) + if (fm->fc->no_listxattr) return -EOPNOTSUPP; memset(&inarg, 0, sizeof(inarg)); @@ -139,13 +150,13 @@ ssize_t fuse_listxattr(struct dentry *entry, char *list, size_t size) args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; } - ret = fuse_simple_request(fc, &args); + ret = fuse_simple_request(fm, &args); if (!ret && !size) ret = min_t(size_t, outarg.size, XATTR_LIST_MAX); if (ret > 0 && size) ret = fuse_verify_xattr_list(list, ret); if (ret == -ENOSYS) { - fc->no_listxattr = 1; + fm->fc->no_listxattr = 1; ret = -EOPNOTSUPP; } return ret; @@ -153,11 +164,11 @@ ssize_t fuse_listxattr(struct dentry *entry, char *list, size_t size) int fuse_removexattr(struct inode *inode, const char *name) { - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = get_fuse_mount(inode); FUSE_ARGS(args); int err; - if (fc->no_removexattr) + if (fm->fc->no_removexattr) return -EOPNOTSUPP; args.opcode = FUSE_REMOVEXATTR; @@ -165,9 +176,9 @@ int fuse_removexattr(struct inode *inode, const char *name) args.in_numargs = 1; args.in_args[0].size = strlen(name) + 1; args.in_args[0].value = name; - err = fuse_simple_request(fc, &args); + err = fuse_simple_request(fm, &args); if (err == -ENOSYS) { - fc->no_removexattr = 1; + fm->fc->no_removexattr = 1; err = -EOPNOTSUPP; } if (!err) { @@ -181,6 +192,17 @@ static int fuse_xattr_get(const struct xattr_handler *handler, struct dentry *dentry, struct inode *inode, const char *name, void *value, size_t size, int flags) { +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + fer = fuse_bpf_backing(inode, struct fuse_getxattr_io, + fuse_getxattr_initialize, fuse_getxattr_backing, + fuse_getxattr_finalize, + dentry, name, value, size); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + if (fuse_is_bad(inode)) return -EIO; @@ -192,6 +214,24 @@ static int fuse_xattr_set(const struct xattr_handler *handler, const char *name, const void *value, size_t size, int flags) { +#ifdef CONFIG_FUSE_BPF + struct fuse_err_ret fer; + + if (value) + fer = fuse_bpf_backing(inode, struct fuse_setxattr_in, + fuse_setxattr_initialize, fuse_setxattr_backing, + fuse_setxattr_finalize, dentry, name, value, + size, flags); + else + fer = fuse_bpf_backing(inode, struct fuse_dummy_io, + fuse_removexattr_initialize, + fuse_removexattr_backing, + fuse_removexattr_finalize, + dentry, name); + if (fer.ret) + return PTR_ERR(fer.result); +#endif + if (fuse_is_bad(inode)) return -EIO; diff --git a/fs/namespace.c b/fs/namespace.c index 2fd0bf91aa77..176bccdf6f30 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -2618,6 +2618,7 @@ static int do_remount(struct path *path, int ms_flags, int sb_flags, if (IS_ERR(fc)) return PTR_ERR(fc); + fc->oldapi = true; err = parse_monolithic_mount_data(fc, data); if (!err) { down_write(&sb->s_umount); diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 4f0d2fc810e4..a4605b17f047 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -310,7 +310,7 @@ nfs_find_actor(struct inode *inode, void *opaque) if (NFS_FILEID(inode) != fattr->fileid) return 0; - if ((S_IFMT & inode->i_mode) != (S_IFMT & fattr->mode)) + if (inode_wrong_type(inode, fattr->mode)) return 0; if (nfs_compare_fh(NFS_FH(inode), fh)) return 0; @@ -1418,7 +1418,7 @@ static int nfs_check_inode_attributes(struct inode *inode, struct nfs_fattr *fat return 0; return -ESTALE; } - if ((fattr->valid & NFS_ATTR_FATTR_TYPE) && (inode->i_mode & S_IFMT) != (fattr->mode & S_IFMT)) + if ((fattr->valid & NFS_ATTR_FATTR_TYPE) && inode_wrong_type(inode, fattr->mode)) return -ESTALE; @@ -1834,7 +1834,7 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) /* * Make sure the inode's type hasn't changed. */ - if ((fattr->valid & NFS_ATTR_FATTR_TYPE) && (inode->i_mode & S_IFMT) != (fattr->mode & S_IFMT)) { + if ((fattr->valid & NFS_ATTR_FATTR_TYPE) && inode_wrong_type(inode, fattr->mode)) { /* * Big trouble! The inode has become a different object. */ diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 4aca93e11af7..5f6186b2ca9b 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -376,7 +376,7 @@ nfsd_proc_create(struct svc_rqst *rqstp) /* Make sure the type and device matches */ nfserr = nfserr_exist; - if (inode && type != (inode->i_mode & S_IFMT)) + if (inode && inode_wrong_type(inode, type)) goto out_unlock; } diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index ec4eadf459ae..e774f7851bc0 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -523,7 +523,7 @@ static const struct file_operations fanotify_fops = { .fasync = NULL, .release = fanotify_release, .unlocked_ioctl = fanotify_ioctl, - .compat_ioctl = fanotify_ioctl, + .compat_ioctl = compat_ptr_ioctl, .llseek = noop_llseek, }; diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c index bcfb0b11b819..f7f259614143 100644 --- a/fs/overlayfs/namei.c +++ b/fs/overlayfs/namei.c @@ -345,7 +345,7 @@ int ovl_check_origin_fh(struct ovl_fs *ofs, struct ovl_fh *fh, bool connected, return PTR_ERR(origin); if (upperdentry && !ovl_is_whiteout(upperdentry) && - ((d_inode(origin)->i_mode ^ d_inode(upperdentry)->i_mode) & S_IFMT)) + inode_wrong_type(d_inode(upperdentry), d_inode(origin)->i_mode)) goto invalid; if (!*stackp) @@ -717,7 +717,7 @@ struct dentry *ovl_lookup_index(struct ovl_fs *ofs, struct dentry *upper, index = ERR_PTR(-ESTALE); goto out; } else if (ovl_dentry_weird(index) || ovl_is_whiteout(index) || - ((inode->i_mode ^ d_inode(origin)->i_mode) & S_IFMT)) { + inode_wrong_type(inode, d_inode(origin)->i_mode)) { /* * Index should always be of the same file type as origin * except for the case of a whiteout index. A whiteout diff --git a/fs/read_write.c b/fs/read_write.c index 93d06b4c5f78..301c4a4ba1d0 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -939,6 +939,34 @@ out: return ret; } +ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb, + struct iov_iter *iter) +{ + size_t tot_len; + ssize_t ret = 0; + + if (!file->f_op->read_iter) + return -EINVAL; + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + goto out; + ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len); + if (ret < 0) + return ret; + + ret = call_read_iter(file, iocb, iter); +out: + if (ret >= 0) + fsnotify_access(file); + return ret; +} +EXPORT_SYMBOL(vfs_iocb_iter_read); + ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos, rwf_t flags) { @@ -975,6 +1003,34 @@ static ssize_t do_iter_write(struct file *file, struct iov_iter *iter, return ret; } +ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb, + struct iov_iter *iter) +{ + size_t tot_len; + ssize_t ret = 0; + + if (!file->f_op->write_iter) + return -EINVAL; + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + return 0; + ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len); + if (ret < 0) + return ret; + + ret = call_write_iter(file, iocb, iter); + if (ret > 0) + fsnotify_modify(file); + + return ret; +} +EXPORT_SYMBOL(vfs_iocb_iter_write); + ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos, rwf_t flags) { diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c index 0b114f685be1..d67fd6a875bd 100644 --- a/fs/ubifs/super.c +++ b/fs/ubifs/super.c @@ -2189,6 +2189,8 @@ static int ubifs_fill_super(struct super_block *sb, void *data, int silent) c->vi.vol_id); if (err) goto out_close; + sb->s_bdi->ra_pages = 0; + sb->s_bdi->io_pages = 0; sb->s_fs_info = c; sb->s_magic = UBIFS_SUPER_MAGIC; diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index de1203b48bb9..69321d466379 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -2017,7 +2017,7 @@ static const struct file_operations userfaultfd_fops = { .poll = userfaultfd_poll, .read = userfaultfd_read, .unlocked_ioctl = userfaultfd_ioctl, - .compat_ioctl = userfaultfd_ioctl, + .compat_ioctl = compat_ptr_ioctl, .llseek = noop_llseek, }; diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index 2bb5801b5887..2d8b63574aa8 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -77,6 +77,9 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LSM, lsm, void *, void *) #endif /* CONFIG_BPF_LSM */ #endif +#ifdef CONFIG_FUSE_BPF +BPF_PROG_TYPE(BPF_PROG_TYPE_FUSE, fuse, struct fuse_bpf_args, struct fuse_bpf_args) +#endif BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) diff --git a/include/linux/dax.h b/include/linux/dax.h index 72a7f03a59f4..d307f6cc7e0b 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -34,6 +34,8 @@ struct dax_operations { /* copy_to_iter: required operation for fs-dax direct-i/o */ size_t (*copy_to_iter)(struct dax_device *, pgoff_t, void *, size_t, struct iov_iter *); + /* zero_page_range: required operation. Zero page range */ + int (*zero_page_range)(struct dax_device *, pgoff_t, size_t); }; extern struct attribute_group dax_attribute_group; @@ -226,6 +228,8 @@ size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i); size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i); +int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, + size_t nr_pages); void dax_flush(struct dax_device *dax_dev, void *addr, size_t size); ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, diff --git a/include/linux/fs.h b/include/linux/fs.h index 773492ba2e6e..e79b51ff5807 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2167,6 +2167,18 @@ static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp) }; } +static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src, + struct file *filp) +{ + *kiocb = (struct kiocb) { + .ki_filp = filp, + .ki_flags = kiocb_src->ki_flags, + .ki_hint = kiocb_src->ki_hint, + .ki_ioprio = kiocb_src->ki_ioprio, + .ki_pos = kiocb_src->ki_pos, + }; +} + /* * Inode state bits. Protected by inode->i_lock * @@ -2985,6 +2997,11 @@ static inline bool execute_ok(struct inode *inode) return (inode->i_mode & S_IXUGO) || S_ISDIR(inode->i_mode); } +static inline bool inode_wrong_type(const struct inode *inode, umode_t mode) +{ + return (inode->i_mode ^ mode) & S_IFMT; +} + static inline void file_start_write(struct file *file) { if (!S_ISREG(file_inode(file)->i_mode)) @@ -3215,6 +3232,10 @@ ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos, rwf_t flags); ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos, rwf_t flags); +ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb, + struct iov_iter *iter); +ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb, + struct iov_iter *iter); /* fs/block_dev.c */ extern ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to); diff --git a/include/linux/fs_context.h b/include/linux/fs_context.h index ebcb91a57e86..d5a73cd318f8 100644 --- a/include/linux/fs_context.h +++ b/include/linux/fs_context.h @@ -105,6 +105,7 @@ struct fs_context { enum fs_context_phase phase:8; /* The phase the context is in */ bool need_free:1; /* Need to call ops->free() */ bool global:1; /* Goes into &init_user_ns */ + bool oldapi:1; /* Coming from mount(2) */ }; struct fs_context_operations { diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index ad4d4697a167..3469d890d9e1 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -10,6 +10,11 @@ struct irq_affinity; +struct virtio_shm_region { + u64 addr; + u64 len; +}; + /** * virtio_config_ops - operations for configuring a virtio device * Note: Do not assume that a transport implements all of the operations @@ -66,6 +71,7 @@ struct irq_affinity; * the caller can then copy. * @set_vq_affinity: set the affinity for a virtqueue (optional). * @get_vq_affinity: get the affinity for a virtqueue (optional). + * @get_shm_region: get a shared memory region based on the index. */ typedef void vq_callback_t(struct virtqueue *); struct virtio_config_ops { @@ -89,6 +95,8 @@ struct virtio_config_ops { const struct cpumask *cpu_mask); const struct cpumask *(*get_vq_affinity)(struct virtio_device *vdev, int index); + bool (*get_shm_region)(struct virtio_device *vdev, + struct virtio_shm_region *region, u8 id); }; /* If driver didn't advertise the feature, it will never appear. */ @@ -251,6 +259,15 @@ int virtqueue_set_affinity(struct virtqueue *vq, const struct cpumask *cpu_mask) return 0; } +static inline +bool virtio_get_shm_region(struct virtio_device *vdev, + struct virtio_shm_region *region, u8 id) +{ + if (!vdev->config->get_shm_region) + return false; + return vdev->config->get_shm_region(vdev, region, id); +} + static inline bool virtio_is_little_endian(struct virtio_device *vdev) { return virtio_has_feature(vdev, VIRTIO_F_VERSION_1) || diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index a33c6997adcd..f1307ab8c21b 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -199,6 +199,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_EXT, BPF_PROG_TYPE_LSM, BPF_PROG_TYPE_SK_LOOKUP, + BPF_PROG_TYPE_FUSE, }; enum bpf_attach_type { diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index a62498fbad3e..f0521a39a586 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -172,6 +172,9 @@ * - add FUSE_WRITE_KILL_PRIV flag * - add FUSE_SETUPMAPPING and FUSE_REMOVEMAPPING * - add map_alignment to fuse_init_out, add FUSE_MAP_ALIGNMENT flag + * + * 7.32 + * - add flags to fuse_attr, add FUSE_ATTR_SUBMOUNT, add FUSE_SUBMOUNTS */ #ifndef _LINUX_FUSE_H @@ -207,7 +210,7 @@ #define FUSE_KERNEL_VERSION 7 /** Minor version number of this interface */ -#define FUSE_KERNEL_MINOR_VERSION 31 +#define FUSE_KERNEL_MINOR_VERSION 32 /** The node ID of the root inode */ #define FUSE_ROOT_ID 1 @@ -231,7 +234,7 @@ struct fuse_attr { uint32_t gid; uint32_t rdev; uint32_t blksize; - uint32_t padding; + uint32_t flags; }; struct fuse_kstatfs { @@ -313,7 +316,10 @@ struct fuse_file_lock { * FUSE_CACHE_SYMLINKS: cache READLINK responses * FUSE_NO_OPENDIR_SUPPORT: kernel supports zero-message opendir * FUSE_EXPLICIT_INVAL_DATA: only invalidate cached pages on explicit request - * FUSE_MAP_ALIGNMENT: map_alignment field is valid + * FUSE_MAP_ALIGNMENT: init_out.map_alignment contains log2(byte alignment) for + * foffset and moffset fields in struct + * fuse_setupmapping_out and fuse_removemapping_one. + * FUSE_SUBMOUNTS: kernel supports auto-mounting directory submounts */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -342,6 +348,7 @@ struct fuse_file_lock { #define FUSE_NO_OPENDIR_SUPPORT (1 << 24) #define FUSE_EXPLICIT_INVAL_DATA (1 << 25) #define FUSE_MAP_ALIGNMENT (1 << 26) +#define FUSE_SUBMOUNTS (1 << 27) #define FUSE_PASSTHROUGH (1 << 31) /** @@ -418,6 +425,13 @@ struct fuse_file_lock { */ #define FUSE_FSYNC_FDATASYNC (1 << 0) +/** + * fuse_attr flags + * + * FUSE_ATTR_SUBMOUNT: Object is a submount root + */ +#define FUSE_ATTR_SUBMOUNT (1 << 0) + enum fuse_opcode { FUSE_LOOKUP = 1, FUSE_FORGET = 2, /* no reply */ @@ -502,6 +516,23 @@ struct fuse_entry_out { struct fuse_attr attr; }; +#define FUSE_ACTION_KEEP 0 +#define FUSE_ACTION_REMOVE 1 +#define FUSE_ACTION_REPLACE 2 + +struct fuse_entry_bpf_out { + uint64_t backing_action; + uint64_t backing_fd; + uint64_t bpf_action; + uint64_t bpf_fd; +}; + +struct fuse_entry_bpf { + struct fuse_entry_bpf_out out; + struct file *backing_file; + struct file *bpf_file; +}; + struct fuse_forget_in { uint64_t nlookup; }; @@ -620,6 +651,12 @@ struct fuse_read_in { uint32_t padding; }; +struct fuse_read_out { + uint64_t offset; + uint32_t again; + uint32_t padding; +}; + #define FUSE_COMPAT_WRITE_IN_SIZE 24 struct fuse_write_in { @@ -793,7 +830,7 @@ struct fuse_in_header { uint32_t uid; uint32_t gid; uint32_t pid; - uint32_t padding; + uint32_t error_in; }; struct fuse_out_header { @@ -898,4 +935,82 @@ struct fuse_copy_file_range_in { uint64_t flags; }; +#define FUSE_SETUPMAPPING_FLAG_WRITE (1ull << 0) +#define FUSE_SETUPMAPPING_FLAG_READ (1ull << 1) +struct fuse_setupmapping_in { + /* An already open handle */ + uint64_t fh; + /* Offset into the file to start the mapping */ + uint64_t foffset; + /* Length of mapping required */ + uint64_t len; + /* Flags, FUSE_SETUPMAPPING_FLAG_* */ + uint64_t flags; + /* Offset in Memory Window */ + uint64_t moffset; +}; + +struct fuse_removemapping_in { + /* number of fuse_removemapping_one follows */ + uint32_t count; +}; + +struct fuse_removemapping_one { + /* Offset into the dax window start the unmapping */ + uint64_t moffset; + /* Length of mapping required */ + uint64_t len; +}; + +#define FUSE_REMOVEMAPPING_MAX_ENTRY \ + (PAGE_SIZE / sizeof(struct fuse_removemapping_one)) + +/* + * Fuse BPF Args + * + * Used to communicate with bpf programs to allow checking or altering certain values. + * The end_offset allows the bpf verifier to check boundaries statically. This reflects + * the ends of the buffer. size shows the length that was actually used. + * + */ + +/** One input argument of a request */ +struct fuse_bpf_in_arg { + uint32_t size; + const void *value; + const void *end_offset; +}; + +/** One output argument of a request */ +struct fuse_bpf_arg { + uint32_t size; + void *value; + void *end_offset; +}; + +#define FUSE_MAX_IN_ARGS 5 +#define FUSE_MAX_OUT_ARGS 3 + +#define FUSE_BPF_FORCE (1 << 0) +#define FUSE_BPF_OUT_ARGVAR (1 << 6) + +struct fuse_bpf_args { + uint64_t nodeid; + uint32_t opcode; + uint32_t error_in; + uint32_t in_numargs; + uint32_t out_numargs; + uint32_t flags; + struct fuse_bpf_in_arg in_args[FUSE_MAX_IN_ARGS]; + struct fuse_bpf_arg out_args[FUSE_MAX_OUT_ARGS]; +}; + +#define FUSE_BPF_USER_FILTER 1 +#define FUSE_BPF_BACKING 2 +#define FUSE_BPF_POST_FILTER 4 + +#define FUSE_OPCODE_FILTER 0x0ffff +#define FUSE_PREFILTER 0x10000 +#define FUSE_POSTFILTER 0x20000 + #endif /* _LINUX_FUSE_H */ diff --git a/include/uapi/linux/virtio_fs.h b/include/uapi/linux/virtio_fs.h index b02eb2ac3d99..2f64abce781f 100644 --- a/include/uapi/linux/virtio_fs.h +++ b/include/uapi/linux/virtio_fs.h @@ -16,4 +16,7 @@ struct virtio_fs_config { __u32 num_request_queues; } __attribute__((packed)); +/* For the id field in virtio_pci_shm_cap */ +#define VIRTIO_FS_SHMCAP_ID_CACHE 0 + #endif /* _UAPI_LINUX_VIRTIO_FS_H */ diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index c1b9f71ee6aa..bfdc63dc74db 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -36,3 +36,6 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o obj-${CONFIG_BPF_LSM} += bpf_lsm.o endif obj-$(CONFIG_BPF_PRELOAD) += preload/ +ifeq ($(CONFIG_FUSE_BPF),y) +obj-$(CONFIG_BPF_SYSCALL) += bpf_fuse.o +endif diff --git a/kernel/bpf/bpf_fuse.c b/kernel/bpf/bpf_fuse.c new file mode 100644 index 000000000000..00738c84b76e --- /dev/null +++ b/kernel/bpf/bpf_fuse.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2021 Google LLC + +#include +#include + +static const struct bpf_func_proto * +fuse_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_trace_printk: + return bpf_get_trace_printk_proto(); + + case BPF_FUNC_get_current_uid_gid: + return &bpf_get_current_uid_gid_proto; + + case BPF_FUNC_get_current_pid_tgid: + return &bpf_get_current_pid_tgid_proto; + + case BPF_FUNC_map_lookup_elem: + return &bpf_map_lookup_elem_proto; + + case BPF_FUNC_map_update_elem: + return &bpf_map_update_elem_proto; + + default: + pr_debug("Invalid fuse bpf func %d\n", func_id); + return NULL; + } +} + +static bool fuse_prog_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + int i; + + if (off < 0 || off > offsetofend(struct fuse_bpf_args, out_args)) + return false; + + /* TODO This is garbage. Do it properly */ + for (i = 0; i < 5; i++) { + if (off == offsetof(struct fuse_bpf_args, in_args[i].value)) { + info->reg_type = PTR_TO_RDONLY_BUF; + info->ctx_field_size = 256; + if (type != BPF_READ) + return false; + return true; + } + } + for (i = 0; i < 3; i++) { + if (off == offsetof(struct fuse_bpf_args, out_args[i].value)) { + info->reg_type = PTR_TO_RDWR_BUF; + info->ctx_field_size = 256; + return true; + } + } + if (type != BPF_READ) + return false; + + return true; +} + +const struct bpf_verifier_ops fuse_verifier_ops = { + .get_func_proto = fuse_prog_func_proto, + .is_valid_access = fuse_prog_is_valid_access, +}; + +const struct bpf_prog_ops fuse_prog_ops = { +}; + diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 61759369d7de..face6bda1c93 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3,6 +3,7 @@ #include #include +#include #include #include #include diff --git a/kernel/seccomp.c b/kernel/seccomp.c index 455330c8b9ba..1ec9bdc1e208 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -2035,7 +2035,7 @@ static bool seccomp_actions_logged_from_names(u32 *actions_logged, char *names) return true; } -static int read_actions_logged(struct ctl_table *ro_table, void __user *buffer, +static int read_actions_logged(struct ctl_table *ro_table, void *buffer, size_t *lenp, loff_t *ppos) { char names[sizeof(seccomp_actions_avail)]; @@ -2053,7 +2053,7 @@ static int read_actions_logged(struct ctl_table *ro_table, void __user *buffer, return proc_dostring(&table, 0, buffer, lenp, ppos); } -static int write_actions_logged(struct ctl_table *ro_table, void __user *buffer, +static int write_actions_logged(struct ctl_table *ro_table, void *buffer, size_t *lenp, loff_t *ppos, u32 *actions_logged) { char names[sizeof(seccomp_actions_avail)]; diff --git a/mm/backing-dev.c b/mm/backing-dev.c index 3954ef0cc770..1dcde0685a5f 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -878,6 +878,8 @@ struct backing_dev_info *bdi_alloc_node(gfp_t gfp_mask, int node_id) kfree(bdi); return NULL; } + bdi->ra_pages = VM_READAHEAD_PAGES; + bdi->io_pages = VM_READAHEAD_PAGES; return bdi; } EXPORT_SYMBOL(bdi_alloc_node); diff --git a/net/rfkill/core.c b/net/rfkill/core.c index 5bba7c36ac74..97101c55763d 100644 --- a/net/rfkill/core.c +++ b/net/rfkill/core.c @@ -1317,7 +1317,7 @@ static const struct file_operations rfkill_fops = { .release = rfkill_fop_release, #ifdef CONFIG_RFKILL_INPUT .unlocked_ioctl = rfkill_fop_ioctl, - .compat_ioctl = rfkill_fop_ioctl, + .compat_ioctl = compat_ptr_ioctl, #endif .llseek = no_llseek, };