diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig index 858b3339f381..9d634d3a1845 100644 --- a/fs/erofs/Kconfig +++ b/fs/erofs/Kconfig @@ -3,7 +3,6 @@ config EROFS_FS tristate "EROFS filesystem support" depends on BLOCK - select LIBCRC32C help EROFS (Enhanced Read-Only File System) is a lightweight read-only file system with modern designs (eg. page-sized @@ -76,3 +75,17 @@ config EROFS_FS_ZIP If you don't want to enable compression feature, say N. +config EROFS_FS_CLUSTER_PAGE_LIMIT + int "EROFS Cluster Pages Hard Limit" + depends on EROFS_FS_ZIP + range 1 256 + default "1" + help + Indicates maximum # of pages of a compressed + physical cluster. + + For example, if files in a image were compressed + into 8k-unit, hard limit should not be configured + less than 2. Otherwise, the image will be refused + to mount on this kernel. + diff --git a/fs/erofs/Makefile b/fs/erofs/Makefile index 1f9aced49070..46f2aa4ba46c 100644 --- a/fs/erofs/Makefile +++ b/fs/erofs/Makefile @@ -1,6 +1,11 @@ # SPDX-License-Identifier: GPL-2.0-only +EROFS_VERSION = "1.0" + +ccflags-y += -DEROFS_VERSION=\"$(EROFS_VERSION)\" + obj-$(CONFIG_EROFS_FS) += erofs.o -erofs-objs := super.o inode.o data.o namei.o dir.o utils.o pcpubuf.o +erofs-objs := super.o inode.o data.o namei.o dir.o utils.o erofs-$(CONFIG_EROFS_FS_XATTR) += xattr.o erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o + diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h index aea129ddda74..07d279fd5d67 100644 --- a/fs/erofs/compress.h +++ b/fs/erofs/compress.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2019 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #ifndef __EROFS_FS_COMPRESS_H @@ -26,58 +26,30 @@ struct z_erofs_decompress_req { bool inplace_io, partial_decoding; }; -/* some special page->private (unsigned long, see below) */ -#define Z_EROFS_SHORTLIVED_PAGE (-1UL << 2) -#define Z_EROFS_PREALLOCATED_PAGE (-2UL << 2) - /* - * For all pages in a pcluster, page->private should be one of - * Type Last 2bits page->private - * short-lived page 00 Z_EROFS_SHORTLIVED_PAGE - * preallocated page (tryalloc) 00 Z_EROFS_PREALLOCATED_PAGE - * cached/managed page 00 pointer to z_erofs_pcluster - * online page (file-backed, 01/10/11 sub-index << 2 | count - * some pages can be used for inplace I/O) - * - * page->mapping should be one of - * Type page->mapping - * short-lived page NULL - * preallocated page NULL - * cached/managed page non-NULL or NULL (invalidated/truncated page) - * online page non-NULL - * - * For all managed pages, PG_private should be set with 1 extra refcount, - * which is used for page reclaim / migration. + * - 0x5A110C8D ('sallocated', Z_EROFS_MAPPING_STAGING) - + * used to mark temporary allocated pages from other + * file/cached pages and NULL mapping pages. */ +#define Z_EROFS_MAPPING_STAGING ((void *)0x5A110C8D) -/* - * short-lived pages are pages directly from buddy system with specific - * page->private (no need to set PagePrivate since these are non-LRU / - * non-movable pages and bypass reclaim / migration code). - */ -static inline bool z_erofs_is_shortlived_page(struct page *page) +/* check if a page is marked as staging */ +static inline bool z_erofs_page_is_staging(struct page *page) { - if (page->private != Z_EROFS_SHORTLIVED_PAGE) - return false; - - DBG_BUGON(page->mapping); - return true; + return page->mapping == Z_EROFS_MAPPING_STAGING; } -static inline bool z_erofs_put_shortlivedpage(struct list_head *pagepool, - struct page *page) +static inline bool z_erofs_put_stagingpage(struct list_head *pagepool, + struct page *page) { - if (!z_erofs_is_shortlived_page(page)) + if (!z_erofs_page_is_staging(page)) return false; - /* short-lived pages should not be used by others at the same time */ - if (page_ref_count(page) > 1) { + /* staging pages should not be used by others at the same time */ + if (page_ref_count(page) > 1) put_page(page); - } else { - /* follow the pcluster rule above. */ - set_page_private(page, 0); + else list_add(&page->lru, pagepool); - } return true; } diff --git a/fs/erofs/data.c b/fs/erofs/data.c index b336a6201bca..b22a08ac53a2 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2017-2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include "internal.h" @@ -109,6 +109,21 @@ err_out: return err; } +int erofs_map_blocks(struct inode *inode, + struct erofs_map_blocks *map, int flags) +{ + if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) { + int err = z_erofs_map_blocks_iter(inode, map, flags); + + if (map->mpage) { + put_page(map->mpage); + map->mpage = NULL; + } + return err; + } + return erofs_map_blocks_flatmode(inode, map, flags); +} + static inline struct bio *erofs_read_raw_page(struct bio *bio, struct address_space *mapping, struct page *page, @@ -144,7 +159,7 @@ submit_bio_retry: erofs_blk_t blknr; unsigned int blkoff; - err = erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW); + err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW); if (err) goto err_out; @@ -209,7 +224,7 @@ submit_bio_retry: bio_set_dev(bio, sb->s_bdev); bio->bi_iter.bi_sector = (sector_t)blknr << LOG_SECTORS_PER_BLOCK; - bio->bi_opf = REQ_OP_READ | (ra ? REQ_RAHEAD : 0); + bio->bi_opf = REQ_OP_READ; } err = bio_add_page(bio, page, PAGE_SIZE, 0); @@ -322,7 +337,7 @@ static sector_t erofs_bmap(struct address_space *mapping, sector_t block) return 0; } - if (!erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW)) + if (!erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW)) return erofs_blknr(map.m_pa); return 0; diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index 6f6e3d0af2fd..38eeec5e3032 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2019 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include "compress.h" @@ -28,42 +28,6 @@ struct z_erofs_decompressor { char *name; }; -int z_erofs_load_lz4_config(struct super_block *sb, - struct erofs_super_block *dsb, - struct z_erofs_lz4_cfgs *lz4, int size) -{ - struct erofs_sb_info *sbi = EROFS_SB(sb); - u16 distance; - - if (lz4) { - if (size < sizeof(struct z_erofs_lz4_cfgs)) { - erofs_err(sb, "invalid lz4 cfgs, size=%u", size); - return -EINVAL; - } - distance = le16_to_cpu(lz4->max_distance); - - sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks); - if (!sbi->lz4.max_pclusterblks) { - sbi->lz4.max_pclusterblks = 1; /* reserved case */ - } else if (sbi->lz4.max_pclusterblks > - Z_EROFS_PCLUSTER_MAX_SIZE / EROFS_BLKSIZ) { - erofs_err(sb, "too large lz4 pclusterblks %u", - sbi->lz4.max_pclusterblks); - return -EINVAL; - } else if (sbi->lz4.max_pclusterblks >= 2) { - erofs_info(sb, "EXPERIMENTAL big pcluster feature in use. Use at your own risk!"); - } - } else { - distance = le16_to_cpu(dsb->u1.lz4_max_distance); - sbi->lz4.max_pclusterblks = 1; - } - - sbi->lz4.max_distance_pages = distance ? - DIV_ROUND_UP(distance, PAGE_SIZE) + 1 : - LZ4_MAX_DISTANCE_PAGES; - return erofs_pcpubuf_growsize(sbi->lz4.max_pclusterblks); -} - static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, struct list_head *pagepool) { @@ -72,8 +36,6 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL }; unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES, BITS_PER_LONG)] = { 0 }; - unsigned int lz4_max_distance_pages = - EROFS_SB(rq->sb)->lz4.max_distance_pages; void *kaddr = NULL; unsigned int i, j, top; @@ -82,14 +44,14 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, struct page *const page = rq->out[i]; struct page *victim; - if (j >= lz4_max_distance_pages) + if (j >= LZ4_MAX_DISTANCE_PAGES) j = 0; /* 'valid' bounced can only be tested after a complete round */ if (test_bit(j, bounced)) { - DBG_BUGON(i < lz4_max_distance_pages); - DBG_BUGON(top >= lz4_max_distance_pages); - availables[top++] = rq->out[i - lz4_max_distance_pages]; + DBG_BUGON(i < LZ4_MAX_DISTANCE_PAGES); + DBG_BUGON(top >= LZ4_MAX_DISTANCE_PAGES); + availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES]; } if (page) { @@ -115,149 +77,108 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, victim = availables[--top]; get_page(victim); } else { - victim = erofs_allocpage(pagepool, - GFP_KERNEL | __GFP_NOFAIL); - set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); + victim = erofs_allocpage(pagepool, GFP_KERNEL, false); + if (!victim) + return -ENOMEM; + victim->mapping = Z_EROFS_MAPPING_STAGING; } rq->out[i] = victim; } return kaddr ? 1 : 0; } -static void *z_erofs_handle_inplace_io(struct z_erofs_decompress_req *rq, - void *inpage, unsigned int *inputmargin, int *maptype, - bool support_0padding) +static void *generic_copy_inplace_data(struct z_erofs_decompress_req *rq, + u8 *src, unsigned int pageofs_in) { - unsigned int nrpages_in, nrpages_out; - unsigned int ofull, oend, inputsize, total, i, j; - struct page **in; - void *src, *tmp; + /* + * if in-place decompression is ongoing, those decompressed + * pages should be copied in order to avoid being overlapped. + */ + struct page **in = rq->in; + u8 *const tmp = erofs_get_pcpubuf(0); + u8 *tmpp = tmp; + unsigned int inlen = rq->inputsize - pageofs_in; + unsigned int count = min_t(uint, inlen, PAGE_SIZE - pageofs_in); - inputsize = rq->inputsize; - nrpages_in = PAGE_ALIGN(inputsize) >> PAGE_SHIFT; - oend = rq->pageofs_out + rq->outputsize; - ofull = PAGE_ALIGN(oend); - nrpages_out = ofull >> PAGE_SHIFT; - - if (rq->inplace_io) { - if (rq->partial_decoding || !support_0padding || - ofull - oend < LZ4_DECOMPRESS_INPLACE_MARGIN(inputsize)) - goto docopy; - - for (i = 0; i < nrpages_in; ++i) { - DBG_BUGON(rq->in[i] == NULL); - for (j = 0; j < nrpages_out - nrpages_in + i; ++j) - if (rq->out[j] == rq->in[i]) - goto docopy; - } - } - - if (nrpages_in <= 1) { - *maptype = 0; - return inpage; - } - kunmap_atomic(inpage); - might_sleep(); - src = erofs_vm_map_ram(rq->in, nrpages_in); - if (!src) - return ERR_PTR(-ENOMEM); - *maptype = 1; - return src; - -docopy: - /* Or copy compressed data which can be overlapped to per-CPU buffer */ - in = rq->in; - src = erofs_get_pcpubuf(nrpages_in); - if (!src) { - DBG_BUGON(1); - kunmap_atomic(inpage); - return ERR_PTR(-EFAULT); - } - - tmp = src; - total = rq->inputsize; - while (total) { - unsigned int page_copycnt = - min_t(unsigned int, total, PAGE_SIZE - *inputmargin); - - if (!inpage) - inpage = kmap_atomic(*in); - memcpy(tmp, inpage + *inputmargin, page_copycnt); - kunmap_atomic(inpage); - inpage = NULL; - tmp += page_copycnt; - total -= page_copycnt; + while (tmpp < tmp + inlen) { + if (!src) + src = kmap_atomic(*in); + memcpy(tmpp, src + pageofs_in, count); + kunmap_atomic(src); + src = NULL; + tmpp += count; + pageofs_in = 0; + count = PAGE_SIZE; ++in; - *inputmargin = 0; } - *maptype = 2; - return src; + return tmp; } static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out) { - unsigned int inputmargin; - u8 *headpage, *src; - bool support_0padding; - int ret, maptype; + unsigned int inputmargin, inlen; + u8 *src; + bool copied, support_0padding; + int ret; - DBG_BUGON(*rq->in == NULL); - headpage = kmap_atomic(*rq->in); + if (rq->inputsize > PAGE_SIZE) + return -EOPNOTSUPP; + + src = kmap_atomic(*rq->in); inputmargin = 0; support_0padding = false; /* decompression inplace is only safe when 0padding is enabled */ - if (erofs_sb_has_lz4_0padding(EROFS_SB(rq->sb))) { + if (EROFS_SB(rq->sb)->feature_incompat & + EROFS_FEATURE_INCOMPAT_LZ4_0PADDING) { support_0padding = true; - while (!headpage[inputmargin & ~PAGE_MASK]) + while (!src[inputmargin & ~PAGE_MASK]) if (!(++inputmargin & ~PAGE_MASK)) break; if (inputmargin >= rq->inputsize) { - kunmap_atomic(headpage); + kunmap_atomic(src); return -EIO; } } - rq->inputsize -= inputmargin; - src = z_erofs_handle_inplace_io(rq, headpage, &inputmargin, &maptype, - support_0padding); - if (IS_ERR(src)) - return PTR_ERR(src); + copied = false; + inlen = rq->inputsize - inputmargin; + if (rq->inplace_io) { + const uint oend = (rq->pageofs_out + + rq->outputsize) & ~PAGE_MASK; + const uint nr = PAGE_ALIGN(rq->pageofs_out + + rq->outputsize) >> PAGE_SHIFT; - /* legacy format could compress extra data in a pcluster. */ - if (rq->partial_decoding || !support_0padding) - ret = LZ4_decompress_safe_partial(src + inputmargin, out, - rq->inputsize, rq->outputsize, rq->outputsize); - else - ret = LZ4_decompress_safe(src + inputmargin, out, - rq->inputsize, rq->outputsize); - - if (ret != rq->outputsize) { - erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]", - ret, rq->inputsize, inputmargin, rq->outputsize); + if (rq->partial_decoding || !support_0padding || + rq->out[nr - 1] != rq->in[0] || + rq->inputsize - oend < + LZ4_DECOMPRESS_INPLACE_MARGIN(inlen)) { + src = generic_copy_inplace_data(rq, src, inputmargin); + inputmargin = 0; + copied = true; + } + } + ret = LZ4_decompress_safe_partial(src + inputmargin, out, + inlen, rq->outputsize, + rq->outputsize); + if (ret < 0) { + erofs_err(rq->sb, "failed to decompress, in[%u, %u] out[%u]", + inlen, inputmargin, rq->outputsize); + WARN_ON(1); print_hex_dump(KERN_DEBUG, "[ in]: ", DUMP_PREFIX_OFFSET, - 16, 1, src + inputmargin, rq->inputsize, true); + 16, 1, src + inputmargin, inlen, true); print_hex_dump(KERN_DEBUG, "[out]: ", DUMP_PREFIX_OFFSET, 16, 1, out, rq->outputsize, true); - - if (ret >= 0) - memset(out + ret, 0, rq->outputsize - ret); ret = -EIO; } - if (maptype == 0) { - kunmap_atomic(src); - } else if (maptype == 1) { - vm_unmap_ram(src, PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT); - } else if (maptype == 2) { + if (copied) erofs_put_pcpubuf(src); - } else { - DBG_BUGON(1); - return -EFAULT; - } + else + kunmap_atomic(src); return ret; } @@ -307,51 +228,57 @@ static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq, const struct z_erofs_decompressor *alg = decompressors + rq->alg; unsigned int dst_maptype; void *dst; - int ret; + int ret, i; - /* two optimized fast paths only for non bigpcluster cases yet */ - if (rq->inputsize <= PAGE_SIZE) { - if (nrpages_out == 1 && !rq->inplace_io) { - DBG_BUGON(!*rq->out); - dst = kmap_atomic(*rq->out); - dst_maptype = 0; - goto dstmap_out; - } - - /* - * For the case of small output size (especially much less - * than PAGE_SIZE), memcpy the decompressed data rather than - * compressed data is preferred. - */ - if (rq->outputsize <= PAGE_SIZE * 7 / 8) { - dst = erofs_get_pcpubuf(1); - if (IS_ERR(dst)) - return PTR_ERR(dst); - - rq->inplace_io = false; - ret = alg->decompress(rq, dst); - if (!ret) - copy_from_pcpubuf(rq->out, dst, rq->pageofs_out, - rq->outputsize); - - erofs_put_pcpubuf(dst); - return ret; - } + if (nrpages_out == 1 && !rq->inplace_io) { + DBG_BUGON(!*rq->out); + dst = kmap_atomic(*rq->out); + dst_maptype = 0; + goto dstmap_out; } - /* general decoding path which can be used for all cases */ - ret = alg->prepare_destpages(rq, pagepool); - if (ret < 0) + /* + * For the case of small output size (especially much less + * than PAGE_SIZE), memcpy the decompressed data rather than + * compressed data is preferred. + */ + if (rq->outputsize <= PAGE_SIZE * 7 / 8) { + dst = erofs_get_pcpubuf(0); + if (IS_ERR(dst)) + return PTR_ERR(dst); + + rq->inplace_io = false; + ret = alg->decompress(rq, dst); + if (!ret) + copy_from_pcpubuf(rq->out, dst, rq->pageofs_out, + rq->outputsize); + + erofs_put_pcpubuf(dst); return ret; - if (ret) { + } + + ret = alg->prepare_destpages(rq, pagepool); + if (ret < 0) { + return ret; + } else if (ret) { dst = page_address(*rq->out); dst_maptype = 1; goto dstmap_out; } - dst = erofs_vm_map_ram(rq->out, nrpages_out); + i = 0; + while (1) { + dst = vm_map_ram(rq->out, nrpages_out, -1, PAGE_KERNEL); + + /* retry two more times (totally 3 times) */ + if (dst || ++i >= 3) + break; + vm_unmap_aliases(); + } + if (!dst) return -ENOMEM; + dst_maptype = 2; dstmap_out: diff --git a/fs/erofs/dir.c b/fs/erofs/dir.c index 2776bb832127..d28c623dfef9 100644 --- a/fs/erofs/dir.c +++ b/fs/erofs/dir.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2017-2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include "internal.h" diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h index 8739d3adf51f..52ab96154467 100644 --- a/fs/erofs/erofs_fs.h +++ b/fs/erofs/erofs_fs.h @@ -3,7 +3,7 @@ * EROFS (Enhanced ROM File System) on-disk format definition * * Copyright (C) 2017-2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #ifndef __EROFS_FS_H @@ -11,29 +11,20 @@ #define EROFS_SUPER_OFFSET 1024 -#define EROFS_FEATURE_COMPAT_SB_CHKSUM 0x00000001 - /* * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should * be incompatible with this kernel version. */ #define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001 -#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002 -#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 -#define EROFS_ALL_FEATURE_INCOMPAT \ - (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \ - EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ - EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER) +#define EROFS_ALL_FEATURE_INCOMPAT EROFS_FEATURE_INCOMPAT_LZ4_0PADDING -#define EROFS_SB_EXTSLOT_SIZE 16 - -/* erofs on-disk super block (currently 128 bytes) */ +/* 128-byte erofs on-disk super block */ struct erofs_super_block { __le32 magic; /* file system magic number */ __le32 checksum; /* crc32c(super_block) */ __le32 feature_compat; __u8 blkszbits; /* support block_size == PAGE_SIZE only */ - __u8 sb_extslots; /* superblock size = 128 + sb_extslots * 16 */ + __u8 reserved; __le16 root_nid; /* nid of root directory */ __le64 inos; /* total valid ino # (== f_files - f_favail) */ @@ -46,13 +37,8 @@ struct erofs_super_block { __u8 uuid[16]; /* 128-bit uuid for volume */ __u8 volume_name[16]; /* volume name */ __le32 feature_incompat; - union { - /* bitmap for available compression algorithms */ - __le16 available_compr_algs; - /* customized sliding window size instead of 64k by default */ - __le16 lz4_max_distance; - } __packed u1; - __u8 reserved2[42]; + + __u8 reserved2[44]; }; /* @@ -205,33 +191,20 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e) e->e_name_len + le16_to_cpu(e->e_value_size)); } -/* maximum supported size of a physical compression cluster */ -#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) - /* available compression algorithm types (for h_algorithmtype) */ enum { Z_EROFS_COMPRESSION_LZ4 = 0, Z_EROFS_COMPRESSION_MAX }; -#define Z_EROFS_ALL_COMPR_ALGS (1 << (Z_EROFS_COMPRESSION_MAX - 1)) - -/* 14 bytes (+ length field = 16 bytes) */ -struct z_erofs_lz4_cfgs { - __le16 max_distance; - __le16 max_pclusterblks; - u8 reserved[10]; -} __packed; /* * bit 0 : COMPACTED_2B indexes (0 - off; 1 - on) * e.g. for 4k logical cluster size, 4B if compacted 2B is off; * (4B) + 2B + (4B) if compacted 2B is on. - * bit 1 : HEAD1 big pcluster (0 - off; 1 - on) - * bit 2 : HEAD2 big pcluster (0 - off; 1 - on) */ -#define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 -#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 -#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_COMPACTED_2B_BIT 0 + +#define Z_EROFS_ADVISE_COMPACTED_2B (1 << Z_EROFS_ADVISE_COMPACTED_2B_BIT) struct z_erofs_map_header { __le32 h_reserved1; @@ -243,7 +216,9 @@ struct z_erofs_map_header { __u8 h_algorithmtype; /* * bit 0-2 : logical cluster bits - 12, e.g. 0 for 4096; - * bit 3-7 : reserved. + * bit 3-4 : (physical - logical) cluster bits of head 1: + * For example, if logical clustersize = 4096, 1 for 8192. + * bit 5-7 : (physical - logical) cluster bits of head 2. */ __u8 h_clusterbits; }; @@ -286,13 +261,6 @@ enum { #define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS 2 #define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT 0 -/* - * D0_CBLKCNT will be marked _only_ at the 1st non-head lcluster to store the - * compressed block count of a compressed extent (in logical clusters, aka. - * block count of a pcluster). - */ -#define Z_EROFS_VLE_DI_D0_CBLKCNT (1 << 11) - struct z_erofs_vle_decompressed_index { __le16 di_advise; /* where to decompress in the head cluster */ diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index 0a94a52a119f..0dbeaf68e1d6 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2017-2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include "xattr.h" @@ -356,21 +356,27 @@ int erofs_getattr(const struct path *path, struct kstat *stat, const struct inode_operations erofs_generic_iops = { .getattr = erofs_getattr, +#ifdef CONFIG_EROFS_FS_XATTR .listxattr = erofs_listxattr, +#endif .get_acl = erofs_get_acl, }; const struct inode_operations erofs_symlink_iops = { .get_link = page_get_link, .getattr = erofs_getattr, +#ifdef CONFIG_EROFS_FS_XATTR .listxattr = erofs_listxattr, +#endif .get_acl = erofs_get_acl, }; const struct inode_operations erofs_fast_symlink_iops = { .get_link = simple_get_link, .getattr = erofs_getattr, +#ifdef CONFIG_EROFS_FS_XATTR .listxattr = erofs_listxattr, +#endif .get_acl = erofs_get_acl, }; diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index c18b0553cb98..cc7a42682814 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2017-2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #ifndef __EROFS_INTERNAL_H @@ -46,43 +46,25 @@ typedef u64 erofs_off_t; /* data type for filesystem-wide blocks number */ typedef u32 erofs_blk_t; -struct erofs_fs_context { -#ifdef CONFIG_EROFS_FS_ZIP - /* current strategy of how to use managed cache */ - unsigned char cache_strategy; - /* strategy of sync decompression (false - auto, true - force on) */ - bool readahead_sync_decompress; - - /* threshold for decompression synchronously */ - unsigned int max_sync_decompress_pages; -#endif - unsigned int mount_opt; -}; - -/* all filesystem-wide lz4 configurations */ -struct erofs_sb_lz4_info { - /* # of pages needed for EROFS lz4 rolling decompression */ - u16 max_distance_pages; - /* maximum possible blocks for pclusters in the filesystem */ - u16 max_pclusterblks; -}; - struct erofs_sb_info { #ifdef CONFIG_EROFS_FS_ZIP /* list for all registered superblocks, mainly for shrinker */ struct list_head list; struct mutex umount_mutex; - /* managed XArray arranged in physical block number */ - struct xarray managed_pslots; + /* the dedicated workstation for compression */ + struct radix_tree_root workstn_tree; + + /* threshold for decompression synchronously */ + unsigned int max_sync_decompress_pages; unsigned int shrinker_run_no; - u16 available_compr_algs; + + /* current strategy of how to use managed cache */ + unsigned char cache_strategy; /* pseudo inode to manage cached pages */ struct inode *managed_cache; - - struct erofs_sb_lz4_info lz4; #endif /* CONFIG_EROFS_FS_ZIP */ u32 blocks; u32 meta_blkaddr; @@ -93,7 +75,6 @@ struct erofs_sb_info { /* inode slot unit size in bit shift */ unsigned char islotbits; - u32 sb_size; /* total superblock size */ u32 build_time_nsec; u64 build_time; @@ -104,10 +85,9 @@ struct erofs_sb_info { u8 uuid[16]; /* 128-bit uuid for volume */ u8 volume_name[16]; /* volume name */ - u32 feature_compat; u32 feature_incompat; - struct erofs_fs_context ctx; /* options */ + unsigned int mount_opt; }; #define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info) @@ -117,17 +97,17 @@ struct erofs_sb_info { #define EROFS_MOUNT_XATTR_USER 0x00000010 #define EROFS_MOUNT_POSIX_ACL 0x00000020 -#define clear_opt(ctx, option) ((ctx)->mount_opt &= ~EROFS_MOUNT_##option) -#define set_opt(ctx, option) ((ctx)->mount_opt |= EROFS_MOUNT_##option) -#define test_opt(ctx, option) ((ctx)->mount_opt & EROFS_MOUNT_##option) +#define clear_opt(sbi, option) ((sbi)->mount_opt &= ~EROFS_MOUNT_##option) +#define set_opt(sbi, option) ((sbi)->mount_opt |= EROFS_MOUNT_##option) +#define test_opt(sbi, option) ((sbi)->mount_opt & EROFS_MOUNT_##option) +#ifdef CONFIG_EROFS_FS_ZIP enum { EROFS_ZIP_CACHE_DISABLED, EROFS_ZIP_CACHE_READAHEAD, EROFS_ZIP_CACHE_READAROUND }; -#ifdef CONFIG_EROFS_FS_ZIP #define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL) /* basic unit of the workstation of a super_block */ @@ -196,6 +176,12 @@ static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp) return v; } #endif /* !CONFIG_SMP */ + +/* hard limit of pages per compressed cluster */ +#define Z_EROFS_CLUSTER_MAX_PAGES (CONFIG_EROFS_FS_CLUSTER_PAGE_LIMIT) +#define EROFS_PCPUBUF_NR_PAGES Z_EROFS_CLUSTER_MAX_PAGES +#else +#define EROFS_PCPUBUF_NR_PAGES 0 #endif /* !CONFIG_EROFS_FS_ZIP */ /* we strictly follow PAGE_SIZE and no buffer head yet */ @@ -224,17 +210,6 @@ static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid) return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits); } -#define EROFS_FEATURE_FUNCS(name, compat, feature) \ -static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \ -{ \ - return sbi->feature_##compat & EROFS_FEATURE_##feature; \ -} - -EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING) -EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) -EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) -EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) - /* atomic flag definitions */ #define EROFS_I_EA_INITED_BIT 0 #define EROFS_I_Z_INITED_BIT 1 @@ -263,6 +238,7 @@ struct erofs_inode { unsigned short z_advise; unsigned char z_algorithmtype[2]; unsigned char z_logical_clusterbits; + unsigned char z_physical_clusterbits[2]; }; #endif /* CONFIG_EROFS_FS_ZIP */ }; @@ -302,10 +278,12 @@ static inline unsigned int erofs_inode_datalayout(unsigned int value) extern const struct super_operations erofs_sops; extern const struct address_space_operations erofs_raw_access_aops; -extern const struct address_space_operations z_erofs_aops; +#ifdef CONFIG_EROFS_FS_ZIP +extern const struct address_space_operations z_erofs_vle_normalaccess_aops; +#endif /* - * Logical to physical block mapping + * Logical to physical block mapping, used by erofs_map_blocks() * * Different with other file systems, it is used for 2 access modes: * @@ -352,7 +330,7 @@ struct erofs_map_blocks { struct page *mpage; }; -/* Flags used by erofs_map_blocks_flatmode() */ +/* Flags used by erofs_map_blocks() */ #define EROFS_GET_BLOCKS_RAW 0x0001 /* zmap.c */ @@ -374,6 +352,8 @@ static inline int z_erofs_map_blocks_iter(struct inode *inode, /* data.c */ struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr); +int erofs_map_blocks(struct inode *, struct erofs_map_blocks *, int); + /* inode.c */ static inline unsigned long erofs_inode_hash(erofs_nid_t nid) { @@ -401,37 +381,30 @@ int erofs_namei(struct inode *dir, struct qstr *name, /* dir.c */ extern const struct file_operations erofs_dir_fops; -static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count) +/* utils.c / zdata.c */ +struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp, bool nofail); + +#if (EROFS_PCPUBUF_NR_PAGES > 0) +void *erofs_get_pcpubuf(unsigned int pagenr); +#define erofs_put_pcpubuf(buf) do { \ + (void)&(buf); \ + preempt_enable(); \ +} while (0) +#else +static inline void *erofs_get_pcpubuf(unsigned int pagenr) { - int retried = 0; - - while (1) { - void *p = vm_map_ram(pages, count, -1, PAGE_KERNEL); - - /* retry two more times (totally 3 times) */ - if (p || ++retried >= 3) - return p; - vm_unmap_aliases(); - } - return NULL; + return ERR_PTR(-EOPNOTSUPP); } -/* pcpubuf.c */ -void *erofs_get_pcpubuf(unsigned int requiredpages); -void erofs_put_pcpubuf(void *ptr); -int erofs_pcpubuf_growsize(unsigned int nrpages); -void erofs_pcpubuf_init(void); -void erofs_pcpubuf_exit(void); - -/* utils.c / zdata.c */ -struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp); +#define erofs_put_pcpubuf(buf) do {} while (0) +#endif #ifdef CONFIG_EROFS_FS_ZIP int erofs_workgroup_put(struct erofs_workgroup *grp); struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb, - pgoff_t index); -struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb, - struct erofs_workgroup *grp); + pgoff_t index, bool *tag); +int erofs_register_workgroup(struct super_block *sb, + struct erofs_workgroup *grp, bool tag); void erofs_workgroup_free_rcu(struct erofs_workgroup *grp); void erofs_shrinker_register(struct super_block *sb); void erofs_shrinker_unregister(struct super_block *sb); @@ -443,9 +416,6 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, struct erofs_workgroup *egrp); int erofs_try_to_free_cached_page(struct address_space *mapping, struct page *page); -int z_erofs_load_lz4_config(struct super_block *sb, - struct erofs_super_block *dsb, - struct z_erofs_lz4_cfgs *lz4, int len); #else static inline void erofs_shrinker_register(struct super_block *sb) {} static inline void erofs_shrinker_unregister(struct super_block *sb) {} @@ -453,16 +423,6 @@ static inline int erofs_init_shrinker(void) { return 0; } static inline void erofs_exit_shrinker(void) {} static inline int z_erofs_init_zip_subsystem(void) { return 0; } static inline void z_erofs_exit_zip_subsystem(void) {} -static inline int z_erofs_load_lz4_config(struct super_block *sb, - struct erofs_super_block *dsb, - struct z_erofs_lz4_cfgs *lz4, int len) -{ - if (lz4 || dsb->u1.lz4_max_distance) { - erofs_err(sb, "lz4 algorithm isn't enabled"); - return -EINVAL; - } - return 0; -} #endif /* !CONFIG_EROFS_FS_ZIP */ #define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */ diff --git a/fs/erofs/namei.c b/fs/erofs/namei.c index 5f8cc7346c69..3abbecbf73de 100644 --- a/fs/erofs/namei.c +++ b/fs/erofs/namei.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2017-2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include "xattr.h" @@ -244,7 +244,9 @@ static struct dentry *erofs_lookup(struct inode *dir, const struct inode_operations erofs_dir_iops = { .lookup = erofs_lookup, .getattr = erofs_getattr, +#ifdef CONFIG_EROFS_FS_XATTR .listxattr = erofs_listxattr, +#endif .get_acl = erofs_get_acl, }; diff --git a/fs/erofs/pcpubuf.c b/fs/erofs/pcpubuf.c deleted file mode 100644 index 6c885575128a..000000000000 --- a/fs/erofs/pcpubuf.c +++ /dev/null @@ -1,148 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) Gao Xiang - * - * For low-latency decompression algorithms (e.g. lz4), reserve consecutive - * per-CPU virtual memory (in pages) in advance to store such inplace I/O - * data if inplace decompression is failed (due to unmet inplace margin for - * example). - */ -#include "internal.h" - -struct erofs_pcpubuf { - raw_spinlock_t lock; - void *ptr; - struct page **pages; - unsigned int nrpages; -}; - -static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb); - -void *erofs_get_pcpubuf(unsigned int requiredpages) - __acquires(pcb->lock) -{ - struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb); - - raw_spin_lock(&pcb->lock); - /* check if the per-CPU buffer is too small */ - if (requiredpages > pcb->nrpages) { - raw_spin_unlock(&pcb->lock); - put_cpu_var(erofs_pcb); - /* (for sparse checker) pretend pcb->lock is still taken */ - __acquire(pcb->lock); - return NULL; - } - return pcb->ptr; -} - -void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock) -{ - struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id()); - - DBG_BUGON(pcb->ptr != ptr); - raw_spin_unlock(&pcb->lock); - put_cpu_var(erofs_pcb); -} - -/* the next step: support per-CPU page buffers hotplug */ -int erofs_pcpubuf_growsize(unsigned int nrpages) -{ - static DEFINE_MUTEX(pcb_resize_mutex); - static unsigned int pcb_nrpages; - LIST_HEAD(pagepool); - int delta, cpu, ret, i; - - mutex_lock(&pcb_resize_mutex); - delta = nrpages - pcb_nrpages; - ret = 0; - /* avoid shrinking pcpubuf, since no idea how many fses rely on */ - if (delta <= 0) - goto out; - - for_each_possible_cpu(cpu) { - struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); - struct page **pages, **oldpages; - void *ptr, *old_ptr; - - pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL); - if (!pages) { - ret = -ENOMEM; - break; - } - - for (i = 0; i < nrpages; ++i) { - pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL); - if (!pages[i]) { - ret = -ENOMEM; - oldpages = pages; - goto free_pagearray; - } - } - ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL); - if (!ptr) { - ret = -ENOMEM; - oldpages = pages; - goto free_pagearray; - } - raw_spin_lock(&pcb->lock); - old_ptr = pcb->ptr; - pcb->ptr = ptr; - oldpages = pcb->pages; - pcb->pages = pages; - i = pcb->nrpages; - pcb->nrpages = nrpages; - raw_spin_unlock(&pcb->lock); - - if (!oldpages) { - DBG_BUGON(old_ptr); - continue; - } - - if (old_ptr) - vunmap(old_ptr); -free_pagearray: - while (i) - list_add(&oldpages[--i]->lru, &pagepool); - kfree(oldpages); - if (ret) - break; - } - pcb_nrpages = nrpages; - put_pages_list(&pagepool); -out: - mutex_unlock(&pcb_resize_mutex); - return ret; -} - -void erofs_pcpubuf_init(void) -{ - int cpu; - - for_each_possible_cpu(cpu) { - struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); - - raw_spin_lock_init(&pcb->lock); - } -} - -void erofs_pcpubuf_exit(void) -{ - int cpu, i; - - for_each_possible_cpu(cpu) { - struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); - - if (pcb->ptr) { - vunmap(pcb->ptr); - pcb->ptr = NULL; - } - if (!pcb->pages) - continue; - - for (i = 0; i < pcb->nrpages; ++i) - if (pcb->pages[i]) - put_page(pcb->pages[i]); - kfree(pcb->pages); - pcb->pages = NULL; - } -} diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 975c8a18f0ed..22e059b4f745 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2017-2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include @@ -9,9 +9,6 @@ #include #include #include -#include -#include -#include #include "xattr.h" #define CREATE_TRACE_POINTS @@ -49,30 +46,6 @@ void _erofs_info(struct super_block *sb, const char *function, va_end(args); } -static int erofs_superblock_csum_verify(struct super_block *sb, void *sbdata) -{ - struct erofs_super_block *dsb; - u32 expected_crc, crc; - - dsb = kmemdup(sbdata + EROFS_SUPER_OFFSET, - EROFS_BLKSIZ - EROFS_SUPER_OFFSET, GFP_KERNEL); - if (!dsb) - return -ENOMEM; - - expected_crc = le32_to_cpu(dsb->checksum); - dsb->checksum = 0; - /* to allow for x86 boot sectors and other oddities. */ - crc = crc32c(~0, dsb, EROFS_BLKSIZ - EROFS_SUPER_OFFSET); - kfree(dsb); - - if (crc != expected_crc) { - erofs_err(sb, "invalid checksum 0x%08x, 0x%08x expected", - crc, expected_crc); - return -EBADMSG; - } - return 0; -} - static void erofs_inode_init_once(void *ptr) { struct erofs_inode *vi = ptr; @@ -122,136 +95,6 @@ static bool check_layout_compatibility(struct super_block *sb, return true; } -#ifdef CONFIG_EROFS_FS_ZIP -/* read variable-sized metadata, offset will be aligned by 4-byte */ -static void *erofs_read_metadata(struct super_block *sb, struct page **pagep, - erofs_off_t *offset, int *lengthp) -{ - struct page *page = *pagep; - u8 *buffer, *ptr; - int len, i, cnt; - erofs_blk_t blk; - - *offset = round_up(*offset, 4); - blk = erofs_blknr(*offset); - - if (!page || page->index != blk) { - if (page) { - unlock_page(page); - put_page(page); - } - page = erofs_get_meta_page(sb, blk); - if (IS_ERR(page)) - goto err_nullpage; - } - - ptr = kmap(page); - len = le16_to_cpu(*(__le16 *)&ptr[erofs_blkoff(*offset)]); - if (!len) - len = U16_MAX + 1; - buffer = kmalloc(len, GFP_KERNEL); - if (!buffer) { - buffer = ERR_PTR(-ENOMEM); - goto out; - } - *offset += sizeof(__le16); - *lengthp = len; - - for (i = 0; i < len; i += cnt) { - cnt = min(EROFS_BLKSIZ - (int)erofs_blkoff(*offset), len - i); - blk = erofs_blknr(*offset); - - if (!page || page->index != blk) { - if (page) { - kunmap(page); - unlock_page(page); - put_page(page); - } - page = erofs_get_meta_page(sb, blk); - if (IS_ERR(page)) { - kfree(buffer); - goto err_nullpage; - } - ptr = kmap(page); - } - memcpy(buffer + i, ptr + erofs_blkoff(*offset), cnt); - *offset += cnt; - } -out: - kunmap(page); - *pagep = page; - return buffer; -err_nullpage: - *pagep = NULL; - return page; -} - -static int erofs_load_compr_cfgs(struct super_block *sb, - struct erofs_super_block *dsb) -{ - struct erofs_sb_info *sbi; - struct page *page; - unsigned int algs, alg; - erofs_off_t offset; - int size, ret; - - sbi = EROFS_SB(sb); - sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs); - - if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) { - erofs_err(sb, "try to load compressed fs with unsupported algorithms %x", - sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS); - return -EINVAL; - } - - offset = EROFS_SUPER_OFFSET + sbi->sb_size; - page = NULL; - alg = 0; - ret = 0; - - for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) { - void *data; - - if (!(algs & 1)) - continue; - - data = erofs_read_metadata(sb, &page, &offset, &size); - if (IS_ERR(data)) { - ret = PTR_ERR(data); - goto err; - } - - switch (alg) { - case Z_EROFS_COMPRESSION_LZ4: - ret = z_erofs_load_lz4_config(sb, dsb, data, size); - break; - default: - DBG_BUGON(1); - ret = -EFAULT; - } - kfree(data); - if (ret) - goto err; - } -err: - if (page) { - unlock_page(page); - put_page(page); - } - return ret; -} -#else -static int erofs_load_compr_cfgs(struct super_block *sb, - struct erofs_super_block *dsb) -{ - if (dsb->u1.available_compr_algs) { - erofs_err(sb, "try to load compressed fs when compression is disabled"); - return -EINVAL; - } - return 0; -} -#endif - static int erofs_read_superblock(struct super_block *sb) { struct erofs_sb_info *sbi; @@ -269,7 +112,7 @@ static int erofs_read_superblock(struct super_block *sb) sbi = EROFS_SB(sb); - data = kmap(page); + data = kmap_atomic(page); dsb = (struct erofs_super_block *)(data + EROFS_SUPER_OFFSET); ret = -EINVAL; @@ -278,14 +121,6 @@ static int erofs_read_superblock(struct super_block *sb) goto out; } - sbi->feature_compat = le32_to_cpu(dsb->feature_compat); - if (erofs_sb_has_sb_chksum(sbi)) { - ret = erofs_superblock_csum_verify(sb, data); - if (ret) - goto out; - } - - ret = -EINVAL; blkszbits = dsb->blkszbits; /* 9(512 bytes) + LOG_SECTORS_PER_BLOCK == LOG_BLOCK_SIZE */ if (blkszbits != LOG_BLOCK_SIZE) { @@ -297,12 +132,6 @@ static int erofs_read_superblock(struct super_block *sb) if (!check_layout_compatibility(sb, dsb)) goto out; - sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE; - if (sbi->sb_size > EROFS_BLKSIZ) { - erofs_err(sb, "invalid sb_extslots %u (more than a fs block)", - sbi->sb_size); - goto out; - } sbi->blocks = le32_to_cpu(dsb->blocks); sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr); #ifdef CONFIG_EROFS_FS_XATTR @@ -324,101 +153,139 @@ static int erofs_read_superblock(struct super_block *sb) ret = -EFSCORRUPTED; goto out; } - - /* parse on-disk compression configurations */ - if (erofs_sb_has_compr_cfgs(sbi)) - ret = erofs_load_compr_cfgs(sb, dsb); - else - ret = z_erofs_load_lz4_config(sb, dsb, NULL, 0); + ret = 0; out: - kunmap(page); + kunmap_atomic(data); put_page(page); return ret; } +#ifdef CONFIG_EROFS_FS_ZIP +static int erofs_build_cache_strategy(struct super_block *sb, + substring_t *args) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + const char *cs = match_strdup(args); + int err = 0; + + if (!cs) { + erofs_err(sb, "Not enough memory to store cache strategy"); + return -ENOMEM; + } + + if (!strcmp(cs, "disabled")) { + sbi->cache_strategy = EROFS_ZIP_CACHE_DISABLED; + } else if (!strcmp(cs, "readahead")) { + sbi->cache_strategy = EROFS_ZIP_CACHE_READAHEAD; + } else if (!strcmp(cs, "readaround")) { + sbi->cache_strategy = EROFS_ZIP_CACHE_READAROUND; + } else { + erofs_err(sb, "Unrecognized cache strategy \"%s\"", cs); + err = -EINVAL; + } + kfree(cs); + return err; +} +#else +static int erofs_build_cache_strategy(struct super_block *sb, + substring_t *args) +{ + erofs_info(sb, "EROFS compression is disabled, so cache strategy is ignored"); + return 0; +} +#endif + /* set up default EROFS parameters */ -static void erofs_default_options(struct erofs_fs_context *ctx) +static void erofs_default_options(struct erofs_sb_info *sbi) { #ifdef CONFIG_EROFS_FS_ZIP - ctx->cache_strategy = EROFS_ZIP_CACHE_READAROUND; - ctx->max_sync_decompress_pages = 3; - ctx->readahead_sync_decompress = false; + sbi->cache_strategy = EROFS_ZIP_CACHE_READAROUND; + sbi->max_sync_decompress_pages = 3; #endif #ifdef CONFIG_EROFS_FS_XATTR - set_opt(ctx, XATTR_USER); + set_opt(sbi, XATTR_USER); #endif #ifdef CONFIG_EROFS_FS_POSIX_ACL - set_opt(ctx, POSIX_ACL); + set_opt(sbi, POSIX_ACL); #endif } enum { Opt_user_xattr, + Opt_nouser_xattr, Opt_acl, + Opt_noacl, Opt_cache_strategy, Opt_err }; -static const struct fs_parameter_enum erofs_param_cache_strategy[] ={ - {Opt_cache_strategy, "disabled", EROFS_ZIP_CACHE_DISABLED}, - {Opt_cache_strategy, "readahead", EROFS_ZIP_CACHE_READAHEAD}, - {Opt_cache_strategy, "readaround", EROFS_ZIP_CACHE_READAROUND}, - {} +static match_table_t erofs_tokens = { + {Opt_user_xattr, "user_xattr"}, + {Opt_nouser_xattr, "nouser_xattr"}, + {Opt_acl, "acl"}, + {Opt_noacl, "noacl"}, + {Opt_cache_strategy, "cache_strategy=%s"}, + {Opt_err, NULL} }; -static const struct fs_parameter_spec erofs_fs_param_specs[] = { - fsparam_flag_no("user_xattr", Opt_user_xattr), - fsparam_flag_no("acl", Opt_acl), - fsparam_enum("cache_strategy", Opt_cache_strategy), - {} -}; -const struct fs_parameter_description erofs_fs_parameters = { - .name = "erofs", - .specs = erofs_fs_param_specs, - .enums = erofs_param_cache_strategy, -}; - -static int erofs_fc_parse_param(struct fs_context *fc, - struct fs_parameter *param) +static int erofs_parse_options(struct super_block *sb, char *options) { - struct erofs_fs_context *ctx __maybe_unused = fc->fs_private; - struct fs_parse_result result; - int opt; + substring_t args[MAX_OPT_ARGS]; + char *p; + int err; - opt = fs_parse(fc, &erofs_fs_parameters, param, &result); - if (opt < 0) - return opt; + if (!options) + return 0; - switch (opt) { - case Opt_user_xattr: + while ((p = strsep(&options, ","))) { + int token; + + if (!*p) + continue; + + args[0].to = args[0].from = NULL; + token = match_token(p, erofs_tokens, args); + + switch (token) { #ifdef CONFIG_EROFS_FS_XATTR - if (result.boolean) - set_opt(ctx, XATTR_USER); - else - clear_opt(ctx, XATTR_USER); + case Opt_user_xattr: + set_opt(EROFS_SB(sb), XATTR_USER); + break; + case Opt_nouser_xattr: + clear_opt(EROFS_SB(sb), XATTR_USER); + break; #else - errorfc(fc, "{,no}user_xattr options not supported"); + case Opt_user_xattr: + erofs_info(sb, "user_xattr options not supported"); + break; + case Opt_nouser_xattr: + erofs_info(sb, "nouser_xattr options not supported"); + break; #endif - break; - case Opt_acl: #ifdef CONFIG_EROFS_FS_POSIX_ACL - if (result.boolean) - set_opt(ctx, POSIX_ACL); - else - clear_opt(ctx, POSIX_ACL); + case Opt_acl: + set_opt(EROFS_SB(sb), POSIX_ACL); + break; + case Opt_noacl: + clear_opt(EROFS_SB(sb), POSIX_ACL); + break; #else - errorfc(fc, "{,no}acl options not supported"); + case Opt_acl: + erofs_info(sb, "acl options not supported"); + break; + case Opt_noacl: + erofs_info(sb, "noacl options not supported"); + break; #endif - break; - case Opt_cache_strategy: -#ifdef CONFIG_EROFS_FS_ZIP - ctx->cache_strategy = result.uint_32; -#else - errorfc(fc, "compression not supported, cache_strategy ignored"); -#endif - break; - default: - return -ENOPARAM; + case Opt_cache_strategy: + err = erofs_build_cache_strategy(sb, args); + if (err) + return err; + break; + default: + erofs_err(sb, "Unrecognized mount option \"%s\" or missing value", p); + return -EINVAL; + } } return 0; } @@ -482,11 +349,10 @@ static int erofs_init_managed_cache(struct super_block *sb) static int erofs_init_managed_cache(struct super_block *sb) { return 0; } #endif -static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) +static int erofs_fill_super(struct super_block *sb, void *data, int silent) { struct inode *inode; struct erofs_sb_info *sbi; - struct erofs_fs_context *ctx = fc->fs_private; int err; sb->s_magic = EROFS_SUPER_MAGIC; @@ -510,17 +376,24 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_time_gran = 1; sb->s_op = &erofs_sops; - sb->s_xattr = erofs_xattr_handlers; - if (test_opt(ctx, POSIX_ACL)) +#ifdef CONFIG_EROFS_FS_XATTR + sb->s_xattr = erofs_xattr_handlers; +#endif + /* set erofs default mount options */ + erofs_default_options(sbi); + + err = erofs_parse_options(sb, data); + if (err) + return err; + + if (test_opt(sbi, POSIX_ACL)) sb->s_flags |= SB_POSIXACL; else sb->s_flags &= ~SB_POSIXACL; - sbi->ctx = *ctx; - #ifdef CONFIG_EROFS_FS_ZIP - xa_init(&sbi->managed_pslots); + INIT_RADIX_TREE(&sbi->workstn_tree, GFP_ATOMIC); #endif /* get the root inode */ @@ -545,58 +418,15 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) if (err) return err; - erofs_info(sb, "mounted with root inode @ nid %llu.", ROOT_NID(sbi)); + erofs_info(sb, "mounted with opts: %s, root inode @ nid %llu.", + (char *)data, ROOT_NID(sbi)); return 0; } -static int erofs_fc_get_tree(struct fs_context *fc) +static struct dentry *erofs_mount(struct file_system_type *fs_type, int flags, + const char *dev_name, void *data) { - return get_tree_bdev(fc, erofs_fc_fill_super); -} - -static int erofs_fc_reconfigure(struct fs_context *fc) -{ - struct super_block *sb = fc->root->d_sb; - struct erofs_sb_info *sbi = EROFS_SB(sb); - struct erofs_fs_context *ctx = fc->fs_private; - - DBG_BUGON(!sb_rdonly(sb)); - - if (test_opt(ctx, POSIX_ACL)) - fc->sb_flags |= SB_POSIXACL; - else - fc->sb_flags &= ~SB_POSIXACL; - - sbi->ctx = *ctx; - - fc->sb_flags |= SB_RDONLY; - return 0; -} - -static void erofs_fc_free(struct fs_context *fc) -{ - kfree(fc->fs_private); -} - -static const struct fs_context_operations erofs_context_ops = { - .parse_param = erofs_fc_parse_param, - .get_tree = erofs_fc_get_tree, - .reconfigure = erofs_fc_reconfigure, - .free = erofs_fc_free, -}; - -static int erofs_init_fs_context(struct fs_context *fc) -{ - fc->fs_private = kzalloc(sizeof(struct erofs_fs_context), GFP_KERNEL); - if (!fc->fs_private) - return -ENOMEM; - - /* set default mount options */ - erofs_default_options(fc->fs_private); - - fc->ops = &erofs_context_ops; - - return 0; + return mount_bdev(fs_type, flags, dev_name, data, erofs_fill_super); } /* @@ -635,7 +465,7 @@ static void erofs_put_super(struct super_block *sb) static struct file_system_type erofs_fs_type = { .owner = THIS_MODULE, .name = "erofs", - .init_fs_context = erofs_init_fs_context, + .mount = erofs_mount, .kill_sb = erofs_kill_sb, .fs_flags = FS_REQUIRES_DEV, }; @@ -660,7 +490,6 @@ static int __init erofs_module_init(void) if (err) goto shrinker_err; - erofs_pcpubuf_init(); err = z_erofs_init_zip_subsystem(); if (err) goto zip_err; @@ -690,7 +519,6 @@ static void __exit erofs_module_exit(void) /* Ensure all RCU free inodes are safe before cache is destroyed. */ rcu_barrier(); kmem_cache_destroy(erofs_inode_cachep); - erofs_pcpubuf_exit(); } /* get filesystem statistics */ @@ -710,44 +538,72 @@ static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf) buf->f_namelen = EROFS_NAME_LEN; - buf->f_fsid = u64_to_fsid(id); + buf->f_fsid.val[0] = (u32)id; + buf->f_fsid.val[1] = (u32)(id >> 32); return 0; } static int erofs_show_options(struct seq_file *seq, struct dentry *root) { struct erofs_sb_info *sbi __maybe_unused = EROFS_SB(root->d_sb); - struct erofs_fs_context *ctx __maybe_unused = &sbi->ctx; #ifdef CONFIG_EROFS_FS_XATTR - if (test_opt(ctx, XATTR_USER)) + if (test_opt(sbi, XATTR_USER)) seq_puts(seq, ",user_xattr"); else seq_puts(seq, ",nouser_xattr"); #endif #ifdef CONFIG_EROFS_FS_POSIX_ACL - if (test_opt(ctx, POSIX_ACL)) + if (test_opt(sbi, POSIX_ACL)) seq_puts(seq, ",acl"); else seq_puts(seq, ",noacl"); #endif #ifdef CONFIG_EROFS_FS_ZIP - if (ctx->cache_strategy == EROFS_ZIP_CACHE_DISABLED) + if (sbi->cache_strategy == EROFS_ZIP_CACHE_DISABLED) { seq_puts(seq, ",cache_strategy=disabled"); - else if (ctx->cache_strategy == EROFS_ZIP_CACHE_READAHEAD) + } else if (sbi->cache_strategy == EROFS_ZIP_CACHE_READAHEAD) { seq_puts(seq, ",cache_strategy=readahead"); - else if (ctx->cache_strategy == EROFS_ZIP_CACHE_READAROUND) + } else if (sbi->cache_strategy == EROFS_ZIP_CACHE_READAROUND) { seq_puts(seq, ",cache_strategy=readaround"); + } else { + seq_puts(seq, ",cache_strategy=(unknown)"); + DBG_BUGON(1); + } #endif return 0; } +static int erofs_remount(struct super_block *sb, int *flags, char *data) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + unsigned int org_mnt_opt = sbi->mount_opt; + int err; + + DBG_BUGON(!sb_rdonly(sb)); + err = erofs_parse_options(sb, data); + if (err) + goto out; + + if (test_opt(sbi, POSIX_ACL)) + sb->s_flags |= SB_POSIXACL; + else + sb->s_flags &= ~SB_POSIXACL; + + *flags |= SB_RDONLY; + return 0; +out: + sbi->mount_opt = org_mnt_opt; + return err; +} + const struct super_operations erofs_sops = { .put_super = erofs_put_super, .alloc_inode = erofs_alloc_inode, .free_inode = erofs_free_inode, .statfs = erofs_statfs, .show_options = erofs_show_options, + .remount_fs = erofs_remount, }; module_init(erofs_module_init); @@ -756,5 +612,4 @@ module_exit(erofs_module_exit); MODULE_DESCRIPTION("Enhanced ROM File System"); MODULE_AUTHOR("Gao Xiang, Chao Yu, Miao Xie, CONSUMER BG, HUAWEI Inc."); MODULE_LICENSE("GPL"); -MODULE_IMPORT_NS(ANDROID_GKI_VFS_EXPORT_ONLY); diff --git a/fs/erofs/utils.c b/fs/erofs/utils.c index 9d9c2c73bebd..3e28fd082df0 100644 --- a/fs/erofs/utils.c +++ b/fs/erofs/utils.c @@ -1,13 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include "internal.h" #include -struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp) +struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp, bool nofail) { struct page *page; @@ -16,15 +16,30 @@ struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp) DBG_BUGON(page_ref_count(page) != 1); list_del(&page->lru); } else { - page = alloc_page(gfp); + page = alloc_pages(gfp | (nofail ? __GFP_NOFAIL : 0), 0); } return page; } +#if (EROFS_PCPUBUF_NR_PAGES > 0) +static struct { + u8 data[PAGE_SIZE * EROFS_PCPUBUF_NR_PAGES]; +} ____cacheline_aligned_in_smp erofs_pcpubuf[NR_CPUS]; + +void *erofs_get_pcpubuf(unsigned int pagenr) +{ + preempt_disable(); + return &erofs_pcpubuf[smp_processor_id()].data[pagenr * PAGE_SIZE]; +} +#endif + #ifdef CONFIG_EROFS_FS_ZIP /* global shrink count (for all mounted EROFS instances) */ static atomic_long_t erofs_global_shrink_cnt; +#define __erofs_workgroup_get(grp) atomic_inc(&(grp)->refcount) +#define __erofs_workgroup_put(grp) atomic_dec(&(grp)->refcount) + static int erofs_workgroup_get(struct erofs_workgroup *grp) { int o; @@ -44,15 +59,18 @@ repeat: } struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb, - pgoff_t index) + pgoff_t index, bool *tag) { struct erofs_sb_info *sbi = EROFS_SB(sb); struct erofs_workgroup *grp; repeat: rcu_read_lock(); - grp = xa_load(&sbi->managed_pslots, index); + grp = radix_tree_lookup(&sbi->workstn_tree, index); if (grp) { + *tag = xa_pointer_tag(grp); + grp = xa_untag_pointer(grp); + if (erofs_workgroup_get(grp)) { /* prefer to relax rcu read side */ rcu_read_unlock(); @@ -65,37 +83,46 @@ repeat: return grp; } -struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb, - struct erofs_workgroup *grp) +int erofs_register_workgroup(struct super_block *sb, + struct erofs_workgroup *grp, + bool tag) { - struct erofs_sb_info *const sbi = EROFS_SB(sb); - struct erofs_workgroup *pre; + struct erofs_sb_info *sbi; + int err; + + /* grp shouldn't be broken or used before */ + if (atomic_read(&grp->refcount) != 1) { + DBG_BUGON(1); + return -EINVAL; + } + + err = radix_tree_preload(GFP_NOFS); + if (err) + return err; + + sbi = EROFS_SB(sb); + xa_lock(&sbi->workstn_tree); + + grp = xa_tag_pointer(grp, tag); /* - * Bump up a reference count before making this visible - * to others for the XArray in order to avoid potential - * UAF without serialized by xa_lock. + * Bump up reference count before making this workgroup + * visible to other users in order to avoid potential UAF + * without serialized by workstn_lock. */ - atomic_inc(&grp->refcount); + __erofs_workgroup_get(grp); -repeat: - xa_lock(&sbi->managed_pslots); - pre = __xa_cmpxchg(&sbi->managed_pslots, grp->index, - NULL, grp, GFP_NOFS); - if (pre) { - if (xa_is_err(pre)) { - pre = ERR_PTR(xa_err(pre)); - } else if (erofs_workgroup_get(pre)) { - /* try to legitimize the current in-tree one */ - xa_unlock(&sbi->managed_pslots); - cond_resched(); - goto repeat; - } - atomic_dec(&grp->refcount); - grp = pre; - } - xa_unlock(&sbi->managed_pslots); - return grp; + err = radix_tree_insert(&sbi->workstn_tree, grp->index, grp); + if (err) + /* + * it's safe to decrease since the workgroup isn't visible + * and refcount >= 2 (cannot be freezed). + */ + __erofs_workgroup_put(grp); + + xa_unlock(&sbi->workstn_tree); + radix_tree_preload_end(); + return err; } static void __erofs_workgroup_free(struct erofs_workgroup *grp) @@ -115,8 +142,15 @@ int erofs_workgroup_put(struct erofs_workgroup *grp) return count; } +static void erofs_workgroup_unfreeze_final(struct erofs_workgroup *grp) +{ + erofs_workgroup_unfreeze(grp, 0); + __erofs_workgroup_free(grp); +} + static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi, - struct erofs_workgroup *grp) + struct erofs_workgroup *grp, + bool cleanup) { /* * If managed cache is on, refcount of workgroups @@ -128,7 +162,7 @@ static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi, /* * Note that all cached pages should be unattached - * before deleted from the XArray. Otherwise some + * before deleted from the radix tree. Otherwise some * cached pages could be still attached to the orphan * old workgroup when the new one is available in the tree. */ @@ -142,34 +176,49 @@ static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi, * however in order to avoid some race conditions, add a * DBG_BUGON to observe this in advance. */ - DBG_BUGON(__xa_erase(&sbi->managed_pslots, grp->index) != grp); + DBG_BUGON(xa_untag_pointer(radix_tree_delete(&sbi->workstn_tree, + grp->index)) != grp); - /* last refcount should be connected with its managed pslot. */ - erofs_workgroup_unfreeze(grp, 0); - __erofs_workgroup_free(grp); + /* + * If managed cache is on, last refcount should indicate + * the related workstation. + */ + erofs_workgroup_unfreeze_final(grp); return true; } static unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi, - unsigned long nr_shrink) + unsigned long nr_shrink, + bool cleanup) { - struct erofs_workgroup *grp; + pgoff_t first_index = 0; + void *batch[PAGEVEC_SIZE]; unsigned int freed = 0; - unsigned long index; - xa_lock(&sbi->managed_pslots); - xa_for_each(&sbi->managed_pslots, index, grp) { + int i, found; +repeat: + xa_lock(&sbi->workstn_tree); + + found = radix_tree_gang_lookup(&sbi->workstn_tree, + batch, first_index, PAGEVEC_SIZE); + + for (i = 0; i < found; ++i) { + struct erofs_workgroup *grp = xa_untag_pointer(batch[i]); + + first_index = grp->index + 1; + /* try to shrink each valid workgroup */ - if (!erofs_try_to_release_workgroup(sbi, grp)) + if (!erofs_try_to_release_workgroup(sbi, grp, cleanup)) continue; - xa_unlock(&sbi->managed_pslots); ++freed; if (!--nr_shrink) - return freed; - xa_lock(&sbi->managed_pslots); + break; } - xa_unlock(&sbi->managed_pslots); + xa_unlock(&sbi->workstn_tree); + + if (i && nr_shrink) + goto repeat; return freed; } @@ -196,8 +245,7 @@ void erofs_shrinker_unregister(struct super_block *sb) struct erofs_sb_info *const sbi = EROFS_SB(sb); mutex_lock(&sbi->umount_mutex); - /* clean up all remaining workgroups in memory */ - erofs_shrink_workstation(sbi, ~0UL); + erofs_shrink_workstation(sbi, ~0UL, true); spin_lock(&erofs_sb_list_lock); list_del(&sbi->list); @@ -246,7 +294,7 @@ static unsigned long erofs_shrink_scan(struct shrinker *shrink, spin_unlock(&erofs_sb_list_lock); sbi->shrinker_run_no = run_no; - freed += erofs_shrink_workstation(sbi, nr - freed); + freed += erofs_shrink_workstation(sbi, nr - freed, false); spin_lock(&erofs_sb_list_lock); /* Get the next list element before we move this one */ diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c index 6330bca9d91d..7d6e38116254 100644 --- a/fs/erofs/xattr.c +++ b/fs/erofs/xattr.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2017-2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include @@ -430,7 +430,7 @@ static int shared_getxattr(struct inode *inode, struct getxattr_iter *it) static bool erofs_xattr_user_list(struct dentry *dentry) { - return test_opt(&EROFS_SB(dentry->d_sb)->ctx, XATTR_USER); + return test_opt(EROFS_SB(dentry->d_sb), XATTR_USER); } static bool erofs_xattr_trusted_list(struct dentry *dentry) @@ -478,7 +478,7 @@ static int erofs_xattr_generic_get(const struct xattr_handler *handler, switch (handler->flags) { case EROFS_XATTR_INDEX_USER: - if (!test_opt(&sbi->ctx, XATTR_USER)) + if (!test_opt(sbi, XATTR_USER)) return -EOPNOTSUPP; break; case EROFS_XATTR_INDEX_TRUSTED: diff --git a/fs/erofs/xattr.h b/fs/erofs/xattr.h index 815304bd335f..3585b84d2f20 100644 --- a/fs/erofs/xattr.h +++ b/fs/erofs/xattr.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2017-2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #ifndef __EROFS_XATTR_H @@ -46,19 +46,18 @@ extern const struct xattr_handler erofs_xattr_security_handler; static inline const struct xattr_handler *erofs_xattr_handler(unsigned int idx) { - static const struct xattr_handler *xattr_handler_map[] = { - [EROFS_XATTR_INDEX_USER] = &erofs_xattr_user_handler, +static const struct xattr_handler *xattr_handler_map[] = { + [EROFS_XATTR_INDEX_USER] = &erofs_xattr_user_handler, #ifdef CONFIG_EROFS_FS_POSIX_ACL - [EROFS_XATTR_INDEX_POSIX_ACL_ACCESS] = - &posix_acl_access_xattr_handler, - [EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT] = - &posix_acl_default_xattr_handler, + [EROFS_XATTR_INDEX_POSIX_ACL_ACCESS] = &posix_acl_access_xattr_handler, + [EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT] = + &posix_acl_default_xattr_handler, #endif - [EROFS_XATTR_INDEX_TRUSTED] = &erofs_xattr_trusted_handler, + [EROFS_XATTR_INDEX_TRUSTED] = &erofs_xattr_trusted_handler, #ifdef CONFIG_EROFS_FS_SECURITY - [EROFS_XATTR_INDEX_SECURITY] = &erofs_xattr_security_handler, + [EROFS_XATTR_INDEX_SECURITY] = &erofs_xattr_security_handler, #endif - }; +}; return idx && idx < ARRAY_SIZE(xattr_handler_map) ? xattr_handler_map[idx] : NULL; @@ -76,8 +75,11 @@ static inline int erofs_getxattr(struct inode *inode, int index, return -EOPNOTSUPP; } -#define erofs_listxattr (NULL) -#define erofs_xattr_handlers (NULL) +static inline ssize_t erofs_listxattr(struct dentry *dentry, + char *buffer, size_t buffer_size) +{ + return -EOPNOTSUPP; +} #endif /* !CONFIG_EROFS_FS_XATTR */ #ifdef CONFIG_EROFS_FS_POSIX_ACL diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 15e03d0f60a0..fb718c3e3ebd 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include "zdata.h" @@ -10,93 +10,6 @@ #include -/* - * since pclustersize is variable for big pcluster feature, introduce slab - * pools implementation for different pcluster sizes. - */ -struct z_erofs_pcluster_slab { - struct kmem_cache *slab; - unsigned int maxpages; - char name[48]; -}; - -#define _PCLP(n) { .maxpages = n } - -static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { - _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), - _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES) -}; - -static void z_erofs_destroy_pcluster_pool(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { - if (!pcluster_pool[i].slab) - continue; - kmem_cache_destroy(pcluster_pool[i].slab); - pcluster_pool[i].slab = NULL; - } -} - -static int z_erofs_create_pcluster_pool(void) -{ - struct z_erofs_pcluster_slab *pcs; - struct z_erofs_pcluster *a; - unsigned int size; - - for (pcs = pcluster_pool; - pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) { - size = struct_size(a, compressed_pages, pcs->maxpages); - - sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages); - pcs->slab = kmem_cache_create(pcs->name, size, 0, - SLAB_RECLAIM_ACCOUNT, NULL); - if (pcs->slab) - continue; - - z_erofs_destroy_pcluster_pool(); - return -ENOMEM; - } - return 0; -} - -static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { - struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; - struct z_erofs_pcluster *pcl; - - if (nrpages > pcs->maxpages) - continue; - - pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS); - if (!pcl) - return ERR_PTR(-ENOMEM); - pcl->pclusterpages = nrpages; - return pcl; - } - return ERR_PTR(-EINVAL); -} - -static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { - struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; - - if (pcl->pclusterpages > pcs->maxpages) - continue; - - kmem_cache_free(pcs->slab, pcl); - return; - } - DBG_BUGON(1); -} - /* * a compressed_pages[] placeholder in order to avoid * being filled with file pages for in-place decompression. @@ -107,11 +20,6 @@ static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) enum z_erofs_cache_alloctype { DONTALLOC, /* don't allocate any cached pages */ DELAYEDALLOC, /* delayed allocation (at the time of submitting io) */ - /* - * try to use cached I/O if page allocation succeeds or fallback - * to in-place I/O instead to avoid any direct reclaim. - */ - TRYALLOC, }; /* @@ -124,37 +32,64 @@ typedef tagptr1_t compressed_page_t; tagptr_fold(compressed_page_t, page, 1) static struct workqueue_struct *z_erofs_workqueue __read_mostly; +static struct kmem_cache *pcluster_cachep __read_mostly; void z_erofs_exit_zip_subsystem(void) { destroy_workqueue(z_erofs_workqueue); - z_erofs_destroy_pcluster_pool(); + kmem_cache_destroy(pcluster_cachep); } static inline int z_erofs_init_workqueue(void) { const unsigned int onlinecpus = num_possible_cpus(); + const unsigned int flags = WQ_UNBOUND | WQ_HIGHPRI | WQ_CPU_INTENSIVE; /* * no need to spawn too many threads, limiting threads could minimum * scheduling overhead, perhaps per-CPU threads should be better? */ - z_erofs_workqueue = alloc_workqueue("erofs_unzipd", - WQ_UNBOUND | WQ_HIGHPRI, + z_erofs_workqueue = alloc_workqueue("erofs_unzipd", flags, onlinecpus + onlinecpus / 4); return z_erofs_workqueue ? 0 : -ENOMEM; } +static void z_erofs_pcluster_init_once(void *ptr) +{ + struct z_erofs_pcluster *pcl = ptr; + struct z_erofs_collection *cl = z_erofs_primarycollection(pcl); + unsigned int i; + + mutex_init(&cl->lock); + cl->nr_pages = 0; + cl->vcnt = 0; + for (i = 0; i < Z_EROFS_CLUSTER_MAX_PAGES; ++i) + pcl->compressed_pages[i] = NULL; +} + +static void z_erofs_pcluster_init_always(struct z_erofs_pcluster *pcl) +{ + struct z_erofs_collection *cl = z_erofs_primarycollection(pcl); + + atomic_set(&pcl->obj.refcount, 1); + + DBG_BUGON(cl->nr_pages); + DBG_BUGON(cl->vcnt); +} + int __init z_erofs_init_zip_subsystem(void) { - int err = z_erofs_create_pcluster_pool(); + pcluster_cachep = kmem_cache_create("erofs_compress", + Z_EROFS_WORKGROUP_SIZE, 0, + SLAB_RECLAIM_ACCOUNT, + z_erofs_pcluster_init_once); + if (pcluster_cachep) { + if (!z_erofs_init_workqueue()) + return 0; - if (err) - return err; - err = z_erofs_init_workqueue(); - if (err) - z_erofs_destroy_pcluster_pool(); - return err; + kmem_cache_destroy(pcluster_cachep); + } + return -ENOMEM; } enum z_erofs_collectmode { @@ -174,12 +109,6 @@ enum z_erofs_collectmode { * |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________| */ COLLECT_PRIMARY_HOOKED, - /* - * a weak form of COLLECT_PRIMARY_FOLLOWED, the difference is that it - * could be dispatched into bypass queue later due to uptodated managed - * pages. All related online pages cannot be reused for inplace I/O (or - * pagevec) since it can be directly decoded without I/O submission. - */ COLLECT_PRIMARY_FOLLOWED_NOINPLACE, /* * The current collection has been linked with the owned chain, and @@ -204,8 +133,7 @@ struct z_erofs_collector { struct z_erofs_pcluster *pcl, *tailpcl; struct z_erofs_collection *cl; - /* a pointer used to pick up inplace I/O pages */ - struct page **icpage_ptr; + struct page **compressedpages; z_erofs_next_pcluster_t owned_head; enum z_erofs_collectmode mode; @@ -217,7 +145,6 @@ struct z_erofs_decompress_frontend { struct z_erofs_collector clt; struct erofs_map_blocks map; - bool readahead; /* used for applying cache strategy on the fly */ bool backmost; erofs_off_t headoffset; @@ -239,22 +166,18 @@ static void preload_compressed_pages(struct z_erofs_collector *clt, enum z_erofs_cache_alloctype type, struct list_head *pagepool) { - struct z_erofs_pcluster *pcl = clt->pcl; + const struct z_erofs_pcluster *pcl = clt->pcl; + const unsigned int clusterpages = BIT(pcl->clusterbits); + struct page **pages = clt->compressedpages; + pgoff_t index = pcl->obj.index + (pages - pcl->compressed_pages); bool standalone = true; - gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) | - __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; - struct page **pages; - pgoff_t index; if (clt->mode < COLLECT_PRIMARY_FOLLOWED) return; - pages = pcl->compressed_pages; - index = pcl->obj.index; - for (; index < pcl->obj.index + pcl->pclusterpages; ++index, ++pages) { + for (; pages < pcl->compressed_pages + clusterpages; ++pages) { struct page *page; compressed_page_t t; - struct page *newpage = NULL; /* the compressed page was loaded before */ if (READ_ONCE(*pages)) @@ -264,43 +187,23 @@ static void preload_compressed_pages(struct z_erofs_collector *clt, if (page) { t = tag_compressed_page_justfound(page); - } else { - /* I/O is needed, no possible to decompress directly */ + } else if (type == DELAYEDALLOC) { + t = tagptr_init(compressed_page_t, PAGE_UNALLOCATED); + } else { /* DONTALLOC */ + if (standalone) + clt->compressedpages = pages; standalone = false; - switch (type) { - case DELAYEDALLOC: - t = tagptr_init(compressed_page_t, - PAGE_UNALLOCATED); - break; - case TRYALLOC: - newpage = erofs_allocpage(pagepool, gfp); - if (!newpage) - continue; - set_page_private(newpage, - Z_EROFS_PREALLOCATED_PAGE); - t = tag_compressed_page_justfound(newpage); - break; - default: /* DONTALLOC */ - continue; - } + continue; } if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t))) continue; - if (page) { + if (page) put_page(page); - } else if (newpage) { - set_page_private(newpage, 0); - list_add(&newpage->lru, pagepool); - } } - /* - * don't do inplace I/O if all compressed pages are available in - * managed cache since it can be moved to the bypass queue instead. - */ - if (standalone) + if (standalone) /* downgrade to PRIMARY_FOLLOWED_NOINPLACE */ clt->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE; } @@ -311,13 +214,14 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, struct z_erofs_pcluster *const pcl = container_of(grp, struct z_erofs_pcluster, obj); struct address_space *const mapping = MNGD_MAPPING(sbi); + const unsigned int clusterpages = BIT(pcl->clusterbits); int i; /* * refcount of workgroup is now freezed as 1, * therefore no need to worry about available decompression users. */ - for (i = 0; i < pcl->pclusterpages; ++i) { + for (i = 0; i < clusterpages; ++i) { struct page *page = pcl->compressed_pages[i]; if (!page) @@ -332,8 +236,11 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, /* barrier is implied in the following 'unlock_page' */ WRITE_ONCE(pcl->compressed_pages[i], NULL); - detach_page_private(page); + set_page_private(page, 0); + ClearPagePrivate(page); + unlock_page(page); + put_page(page); } return 0; } @@ -342,12 +249,13 @@ int erofs_try_to_free_cached_page(struct address_space *mapping, struct page *page) { struct z_erofs_pcluster *const pcl = (void *)page_private(page); + const unsigned int clusterpages = BIT(pcl->clusterbits); int ret = 0; /* 0 - busy */ if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) { unsigned int i; - for (i = 0; i < pcl->pclusterpages; ++i) { + for (i = 0; i < clusterpages; ++i) { if (pcl->compressed_pages[i] == page) { WRITE_ONCE(pcl->compressed_pages[i], NULL); ret = 1; @@ -356,21 +264,25 @@ int erofs_try_to_free_cached_page(struct address_space *mapping, } erofs_workgroup_unfreeze(&pcl->obj, 1); - if (ret) - detach_page_private(page); + if (ret) { + ClearPagePrivate(page); + put_page(page); + } } return ret; } /* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */ -static bool z_erofs_try_inplace_io(struct z_erofs_collector *clt, - struct page *page) +static inline bool z_erofs_try_inplace_io(struct z_erofs_collector *clt, + struct page *page) { struct z_erofs_pcluster *const pcl = clt->pcl; + const unsigned int clusterpages = BIT(pcl->clusterbits); - while (clt->icpage_ptr > pcl->compressed_pages) - if (!cmpxchg(--clt->icpage_ptr, NULL, page)) + while (clt->compressedpages < pcl->compressed_pages + clusterpages) { + if (!cmpxchg(clt->compressedpages++, NULL, page)) return true; + } return false; } @@ -423,31 +335,40 @@ retry: return COLLECT_PRIMARY; /* :( better luck next time */ } -static int z_erofs_lookup_collection(struct z_erofs_collector *clt, - struct inode *inode, - struct erofs_map_blocks *map) +static struct z_erofs_collection *cllookup(struct z_erofs_collector *clt, + struct inode *inode, + struct erofs_map_blocks *map) { - struct z_erofs_pcluster *pcl = clt->pcl; + struct erofs_workgroup *grp; + struct z_erofs_pcluster *pcl; struct z_erofs_collection *cl; unsigned int length; + bool tag; - /* to avoid unexpected loop formed by corrupted images */ + grp = erofs_find_workgroup(inode->i_sb, map->m_pa >> PAGE_SHIFT, &tag); + if (!grp) + return NULL; + + pcl = container_of(grp, struct z_erofs_pcluster, obj); if (clt->owned_head == &pcl->next || pcl == clt->tailpcl) { DBG_BUGON(1); - return -EFSCORRUPTED; + erofs_workgroup_put(grp); + return ERR_PTR(-EFSCORRUPTED); } cl = z_erofs_primarycollection(pcl); if (cl->pageofs != (map->m_la & ~PAGE_MASK)) { DBG_BUGON(1); - return -EFSCORRUPTED; + erofs_workgroup_put(grp); + return ERR_PTR(-EFSCORRUPTED); } length = READ_ONCE(pcl->length); if (length & Z_EROFS_PCLUSTER_FULL_LENGTH) { if ((map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) > length) { DBG_BUGON(1); - return -EFSCORRUPTED; + erofs_workgroup_put(grp); + return ERR_PTR(-EFSCORRUPTED); } } else { unsigned int llen = map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT; @@ -469,25 +390,25 @@ static int z_erofs_lookup_collection(struct z_erofs_collector *clt, /* clean tailpcl if the current owned_head is Z_EROFS_PCLUSTER_TAIL */ if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL) clt->tailpcl = NULL; + clt->pcl = pcl; clt->cl = cl; - return 0; + return cl; } -static int z_erofs_register_collection(struct z_erofs_collector *clt, - struct inode *inode, - struct erofs_map_blocks *map) +static struct z_erofs_collection *clregister(struct z_erofs_collector *clt, + struct inode *inode, + struct erofs_map_blocks *map) { struct z_erofs_pcluster *pcl; struct z_erofs_collection *cl; - struct erofs_workgroup *grp; int err; - /* no available pcluster, let's allocate one */ - pcl = z_erofs_alloc_pcluster(map->m_plen >> PAGE_SHIFT); - if (IS_ERR(pcl)) - return PTR_ERR(pcl); + /* no available workgroup, let's allocate one */ + pcl = kmem_cache_alloc(pcluster_cachep, GFP_NOFS); + if (!pcl) + return ERR_PTR(-ENOMEM); - atomic_set(&pcl->obj.refcount, 1); + z_erofs_pcluster_init_always(pcl); pcl->obj.index = map->m_pa >> PAGE_SHIFT; pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) | @@ -499,6 +420,9 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt, else pcl->algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; + pcl->clusterbits = EROFS_I(inode)->z_physical_clusterbits[0]; + pcl->clusterbits -= PAGE_SHIFT; + /* new pclusters should be claimed as type 1, primary and followed */ pcl->next = clt->owned_head; clt->mode = COLLECT_PRIMARY_FOLLOWED; @@ -510,19 +434,13 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt, * lock all primary followed works before visible to others * and mutex_trylock *never* fails for a new pcluster. */ - mutex_init(&cl->lock); - DBG_BUGON(!mutex_trylock(&cl->lock)); + mutex_trylock(&cl->lock); - grp = erofs_insert_workgroup(inode->i_sb, &pcl->obj); - if (IS_ERR(grp)) { - err = PTR_ERR(grp); - goto err_out; - } - - if (grp != &pcl->obj) { - clt->pcl = container_of(grp, struct z_erofs_pcluster, obj); - err = -EEXIST; - goto err_out; + err = erofs_register_workgroup(inode->i_sb, &pcl->obj, 0); + if (err) { + mutex_unlock(&cl->lock); + kmem_cache_free(pcluster_cachep, pcl); + return ERR_PTR(-EAGAIN); } /* used to check tail merging loop due to corrupted images */ if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL) @@ -530,20 +448,14 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt, clt->owned_head = &pcl->next; clt->pcl = pcl; clt->cl = cl; - return 0; - -err_out: - mutex_unlock(&cl->lock); - z_erofs_free_pcluster(pcl); - return err; + return cl; } static int z_erofs_collector_begin(struct z_erofs_collector *clt, struct inode *inode, struct erofs_map_blocks *map) { - struct erofs_workgroup *grp; - int ret; + struct z_erofs_collection *cl; DBG_BUGON(clt->cl); @@ -556,30 +468,24 @@ static int z_erofs_collector_begin(struct z_erofs_collector *clt, return -EINVAL; } - grp = erofs_find_workgroup(inode->i_sb, map->m_pa >> PAGE_SHIFT); - if (grp) { - clt->pcl = container_of(grp, struct z_erofs_pcluster, obj); - } else { - ret = z_erofs_register_collection(clt, inode, map); +repeat: + cl = cllookup(clt, inode, map); + if (!cl) { + cl = clregister(clt, inode, map); - if (!ret) - goto out; - if (ret != -EEXIST) - return ret; + if (cl == ERR_PTR(-EAGAIN)) + goto repeat; } - ret = z_erofs_lookup_collection(clt, inode, map); - if (ret) { - erofs_workgroup_put(&clt->pcl->obj); - return ret; - } + if (IS_ERR(cl)) + return PTR_ERR(cl); -out: z_erofs_pagevec_ctor_init(&clt->vector, Z_EROFS_NR_INLINE_PAGEVECS, - clt->cl->pagevec, clt->cl->vcnt); + cl->pagevec, cl->vcnt); - /* since file-backed online pages are traversed in reverse order */ - clt->icpage_ptr = clt->pcl->compressed_pages + clt->pcl->pclusterpages; + clt->compressedpages = clt->pcl->compressed_pages; + if (clt->mode <= COLLECT_PRIMARY) /* cannot do in-place I/O */ + clt->compressedpages += Z_EROFS_CLUSTER_MAX_PAGES; return 0; } @@ -592,8 +498,9 @@ static void z_erofs_rcu_callback(struct rcu_head *head) struct z_erofs_collection *const cl = container_of(head, struct z_erofs_collection, rcu); - z_erofs_free_pcluster(container_of(cl, struct z_erofs_pcluster, - primary_collection)); + kmem_cache_free(pcluster_cachep, + container_of(cl, struct z_erofs_pcluster, + primary_collection)); } void erofs_workgroup_free_rcu(struct erofs_workgroup *grp) @@ -634,6 +541,15 @@ static bool z_erofs_collector_end(struct z_erofs_collector *clt) return true; } +static inline struct page *__stagingpage_alloc(struct list_head *pagepool, + gfp_t gfp) +{ + struct page *page = erofs_allocpage(pagepool, gfp, true); + + page->mapping = Z_EROFS_MAPPING_STAGING; + return page; +} + static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe, unsigned int cachestrategy, erofs_off_t la) @@ -649,10 +565,11 @@ static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe, } static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, - struct page *page, struct list_head *pagepool) + struct page *page, + struct list_head *pagepool) { struct inode *const inode = fe->inode; - struct erofs_sb_info *const sbi = EROFS_I_SB(inode); + struct erofs_sb_info *const sbi __maybe_unused = EROFS_I_SB(inode); struct erofs_map_blocks *const map = &fe->map; struct z_erofs_collector *const clt = &fe->clt; const loff_t offset = page_offset(page); @@ -701,8 +618,8 @@ restart_now: goto err_out; /* preload all compressed pages (maybe downgrade role if necessary) */ - if (should_alloc_managed_pages(fe, sbi->ctx.cache_strategy, map->m_la)) - cache_strategy = TRYALLOC; + if (should_alloc_managed_pages(fe, sbi->cache_strategy, map->m_la)) + cache_strategy = DELAYEDALLOC; else cache_strategy = DONTALLOC; @@ -742,9 +659,8 @@ retry: /* should allocate an additional staging page for pagevec */ if (err == -EAGAIN) { struct page *const newpage = - alloc_page(GFP_NOFS | __GFP_NOFAIL); + __stagingpage_alloc(pagepool, GFP_NOFS); - set_page_private(newpage, Z_EROFS_SHORTLIVED_PAGE); err = z_erofs_attach_page(clt, newpage, Z_EROFS_PAGE_TYPE_EXCLUSIVE, true); if (!err) @@ -783,14 +699,13 @@ err_out: goto out; } -static void z_erofs_decompressqueue_work(struct work_struct *work); -static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, - bool sync, int bios) +static void z_erofs_vle_unzip_kickoff(void *ptr, int bios) { - struct erofs_sb_info *const sbi = EROFS_SB(io->sb); + tagptr1_t t = tagptr_init(tagptr1_t, ptr); + struct z_erofs_unzip_io *io = tagptr_unfold_ptr(t); + bool background = tagptr_unfold_tags(t); - /* wake up the caller thread for sync decompression */ - if (sync) { + if (!background) { unsigned long flags; spin_lock_irqsave(&io->u.wait.lock, flags); @@ -800,46 +715,41 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, return; } - if (atomic_add_return(bios, &io->pending_bios)) - return; - /* Use workqueue and sync decompression for atomic contexts only */ - if (in_atomic() || irqs_disabled()) { + if (!atomic_add_return(bios, &io->pending_bios)) queue_work(z_erofs_workqueue, &io->u.work); - sbi->ctx.readahead_sync_decompress = true; - return; - } - z_erofs_decompressqueue_work(&io->u.work); } -static bool z_erofs_page_is_invalidated(struct page *page) +static inline void z_erofs_vle_read_endio(struct bio *bio) { - return !page->mapping && !z_erofs_is_shortlived_page(page); -} - -static void z_erofs_decompressqueue_endio(struct bio *bio) -{ - tagptr1_t t = tagptr_init(tagptr1_t, bio->bi_private); - struct z_erofs_decompressqueue *q = tagptr_unfold_ptr(t); + struct erofs_sb_info *sbi = NULL; blk_status_t err = bio->bi_status; struct bio_vec *bvec; struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) { struct page *page = bvec->bv_page; + bool cachemngd = false; DBG_BUGON(PageUptodate(page)); - DBG_BUGON(z_erofs_page_is_invalidated(page)); + DBG_BUGON(!page->mapping); + + if (!sbi && !z_erofs_page_is_staging(page)) + sbi = EROFS_SB(page->mapping->host->i_sb); + + /* sbi should already be gotten if the page is managed */ + if (sbi) + cachemngd = erofs_page_is_managed(sbi, page); if (err) SetPageError(page); + else if (cachemngd) + SetPageUptodate(page); - if (erofs_page_is_managed(EROFS_SB(q->sb), page)) { - if (!err) - SetPageUptodate(page); + if (cachemngd) unlock_page(page); - } } - z_erofs_decompress_kickoff(q, tagptr_unfold_tags(t), -1); + + z_erofs_vle_unzip_kickoff(bio->bi_private, -1); bio_put(bio); } @@ -848,8 +758,9 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, struct list_head *pagepool) { struct erofs_sb_info *const sbi = EROFS_SB(sb); + const unsigned int clusterpages = BIT(pcl->clusterbits); struct z_erofs_pagevec_ctor ctor; - unsigned int i, inputsize, outputsize, llen, nr_pages; + unsigned int i, outputsize, llen, nr_pages; struct page *pages_onstack[Z_EROFS_VMAP_ONSTACK_PAGES]; struct page **pages, **compressed_pages, *page; @@ -900,9 +811,9 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, /* all pages in pagevec ought to be valid */ DBG_BUGON(!page); - DBG_BUGON(z_erofs_page_is_invalidated(page)); + DBG_BUGON(!page->mapping); - if (z_erofs_put_shortlivedpage(pagepool, page)) + if (z_erofs_put_stagingpage(pagepool, page)) continue; if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD) @@ -929,16 +840,16 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, overlapped = false; compressed_pages = pcl->compressed_pages; - for (i = 0; i < pcl->pclusterpages; ++i) { + for (i = 0; i < clusterpages; ++i) { unsigned int pagenr; page = compressed_pages[i]; /* all compressed pages ought to be valid */ DBG_BUGON(!page); - DBG_BUGON(z_erofs_page_is_invalidated(page)); + DBG_BUGON(!page->mapping); - if (!z_erofs_is_shortlived_page(page)) { + if (!z_erofs_page_is_staging(page)) { if (erofs_page_is_managed(sbi, page)) { if (!PageUptodate(page)) err = -EIO; @@ -963,7 +874,7 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, overlapped = true; } - /* PG_error needs checking for all non-managed pages */ + /* PG_error needs checking for inplaced and staging pages */ if (PageError(page)) { DBG_BUGON(PageUptodate(page)); err = -EIO; @@ -982,13 +893,12 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, partial = true; } - inputsize = pcl->pclusterpages * PAGE_SIZE; err = z_erofs_decompress(&(struct z_erofs_decompress_req) { .sb = sb, .in = compressed_pages, .out = pages, .pageofs_out = cl->pageofs, - .inputsize = inputsize, + .inputsize = PAGE_SIZE, .outputsize = outputsize, .alg = pcl->algorithmformat, .inplace_io = overlapped, @@ -996,15 +906,15 @@ static int z_erofs_decompress_pcluster(struct super_block *sb, }, pagepool); out: - /* must handle all compressed pages before ending pages */ - for (i = 0; i < pcl->pclusterpages; ++i) { + /* must handle all compressed pages before endding pages */ + for (i = 0; i < clusterpages; ++i) { page = compressed_pages[i]; if (erofs_page_is_managed(sbi, page)) continue; - /* recycle all individual short-lived pages */ - (void)z_erofs_put_shortlivedpage(pagepool, page); + /* recycle all individual staging pages */ + (void)z_erofs_put_stagingpage(pagepool, page); WRITE_ONCE(compressed_pages[i], NULL); } @@ -1014,10 +924,10 @@ out: if (!page) continue; - DBG_BUGON(z_erofs_page_is_invalidated(page)); + DBG_BUGON(!page->mapping); - /* recycle all individual short-lived pages */ - if (z_erofs_put_shortlivedpage(pagepool, page)) + /* recycle all individual staging pages */ + if (z_erofs_put_stagingpage(pagepool, page)) continue; if (err < 0) @@ -1044,8 +954,9 @@ out: return err; } -static void z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io, - struct list_head *pagepool) +static void z_erofs_vle_unzip_all(struct super_block *sb, + struct z_erofs_unzip_io *io, + struct list_head *pagepool) { z_erofs_next_pcluster_t owned = io->head; @@ -1061,21 +972,21 @@ static void z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io, pcl = container_of(owned, struct z_erofs_pcluster, next); owned = READ_ONCE(pcl->next); - z_erofs_decompress_pcluster(io->sb, pcl, pagepool); + z_erofs_decompress_pcluster(sb, pcl, pagepool); } } -static void z_erofs_decompressqueue_work(struct work_struct *work) +static void z_erofs_vle_unzip_wq(struct work_struct *work) { - struct z_erofs_decompressqueue *bgq = - container_of(work, struct z_erofs_decompressqueue, u.work); + struct z_erofs_unzip_io_sb *iosb = + container_of(work, struct z_erofs_unzip_io_sb, io.u.work); LIST_HEAD(pagepool); - DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL_CLOSED); - z_erofs_decompress_queue(bgq, &pagepool); + DBG_BUGON(iosb->io.head == Z_EROFS_PCLUSTER_TAIL_CLOSED); + z_erofs_vle_unzip_all(iosb->sb, &iosb->io, &pagepool); put_pages_list(&pagepool); - kvfree(bgq); + kvfree(iosb); } static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, @@ -1084,6 +995,8 @@ static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, struct address_space *mc, gfp_t gfp) { + /* determined at compile time to avoid too many #ifdefs */ + const bool nocache = __builtin_constant_p(mc) ? !mc : false; const pgoff_t index = pcl->obj.index; bool tocache = false; @@ -1104,7 +1017,7 @@ repeat: * the cached page has not been allocated and * an placeholder is out there, prepare it now. */ - if (page == PAGE_UNALLOCATED) { + if (!nocache && page == PAGE_UNALLOCATED) { tocache = true; goto out_allocpage; } @@ -1114,30 +1027,31 @@ repeat: justfound = tagptr_unfold_tags(t); page = tagptr_unfold_ptr(t); - /* - * preallocated cached pages, which is used to avoid direct reclaim - * otherwise, it will go inplace I/O path instead. - */ - if (page->private == Z_EROFS_PREALLOCATED_PAGE) { - WRITE_ONCE(pcl->compressed_pages[nr], page); - set_page_private(page, 0); - tocache = true; - goto out_tocache; - } mapping = READ_ONCE(page->mapping); /* - * file-backed online pages in plcuster are all locked steady, + * if managed cache is disabled, it's no way to + * get such a cached-like page. + */ + if (nocache) { + /* if managed cache is disabled, it is impossible `justfound' */ + DBG_BUGON(justfound); + + /* and it should be locked, not uptodate, and not truncated */ + DBG_BUGON(!PageLocked(page)); + DBG_BUGON(PageUptodate(page)); + DBG_BUGON(!mapping); + goto out; + } + + /* + * unmanaged (file) pages are all locked solidly, * therefore it is impossible for `mapping' to be NULL. */ if (mapping && mapping != mc) /* ought to be unmanaged pages */ goto out; - /* directly return for shortlived page as well */ - if (z_erofs_is_shortlived_page(page)) - goto out; - lock_page(page); /* only true if page reclaim goes wrong, should never happen */ @@ -1180,48 +1094,50 @@ repeat: unlock_page(page); put_page(page); out_allocpage: - page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL); + page = __stagingpage_alloc(pagepool, gfp); if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) { list_add(&page->lru, pagepool); - cond_resched(); + cpu_relax(); goto repeat; } -out_tocache: - if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) { - /* turn into temporary page if fails (1 ref) */ - set_page_private(page, Z_EROFS_SHORTLIVED_PAGE); + if (nocache || !tocache) + goto out; + if (add_to_page_cache_lru(page, mc, index + nr, gfp)) { + page->mapping = Z_EROFS_MAPPING_STAGING; goto out; } - attach_page_private(page, pcl); - /* drop a refcount added by allocpage (then we have 2 refs here) */ - put_page(page); + set_page_private(page, (unsigned long)pcl); + SetPagePrivate(page); out: /* the only exit (for tracing and debugging) */ return page; } -static struct z_erofs_decompressqueue * -jobqueue_init(struct super_block *sb, - struct z_erofs_decompressqueue *fgq, bool *fg) +static struct z_erofs_unzip_io *jobqueue_init(struct super_block *sb, + struct z_erofs_unzip_io *io, + bool foreground) { - struct z_erofs_decompressqueue *q; + struct z_erofs_unzip_io_sb *iosb; - if (fg && !*fg) { - q = kvzalloc(sizeof(*q), GFP_KERNEL | __GFP_NOWARN); - if (!q) { - *fg = true; - goto fg_out; - } - INIT_WORK(&q->u.work, z_erofs_decompressqueue_work); - } else { -fg_out: - q = fgq; - init_waitqueue_head(&fgq->u.wait); - atomic_set(&fgq->pending_bios, 0); + if (foreground) { + /* waitqueue available for foreground io */ + DBG_BUGON(!io); + + init_waitqueue_head(&io->u.wait); + atomic_set(&io->pending_bios, 0); + goto out; } - q->sb = sb; - q->head = Z_EROFS_PCLUSTER_TAIL_CLOSED; - return q; + + iosb = kvzalloc(sizeof(*iosb), GFP_KERNEL | __GFP_NOFAIL); + DBG_BUGON(!iosb); + + /* initialize fields in the allocated descriptor */ + io = &iosb->io; + iosb->sb = sb; + INIT_WORK(&io->u.work, z_erofs_vle_unzip_wq); +out: + io->head = Z_EROFS_PCLUSTER_TAIL_CLOSED; + return io; } /* define decompression jobqueue types */ @@ -1232,17 +1148,22 @@ enum { }; static void *jobqueueset_init(struct super_block *sb, - struct z_erofs_decompressqueue *q[], - struct z_erofs_decompressqueue *fgq, bool *fg) + z_erofs_next_pcluster_t qtail[], + struct z_erofs_unzip_io *q[], + struct z_erofs_unzip_io *fgq, + bool forcefg) { /* * if managed cache is enabled, bypass jobqueue is needed, * no need to read from device for all pclusters in this queue. */ - q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL); - q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, fg); + q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, true); + qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; - return tagptr_cast_ptr(tagptr_fold(tagptr1_t, q[JQ_SUBMIT], *fg)); + q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, forcefg); + qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head; + + return tagptr_cast_ptr(tagptr_fold(tagptr1_t, q[JQ_SUBMIT], !forcefg)); } static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl, @@ -1264,34 +1185,55 @@ static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl, qtail[JQ_BYPASS] = &pcl->next; } -static void z_erofs_submit_queue(struct super_block *sb, - struct z_erofs_decompress_frontend *f, - struct list_head *pagepool, - struct z_erofs_decompressqueue *fgq, - bool *force_fg) +static bool postsubmit_is_all_bypassed(struct z_erofs_unzip_io *q[], + unsigned int nr_bios, + bool force_fg) { - struct erofs_sb_info *const sbi = EROFS_SB(sb); - z_erofs_next_pcluster_t qtail[NR_JOBQUEUES]; - struct z_erofs_decompressqueue *q[NR_JOBQUEUES]; - void *bi_private; - z_erofs_next_pcluster_t owned_head = f->clt.owned_head; - /* since bio will be NULL, no need to initialize last_index */ - pgoff_t last_index; - unsigned int nr_bios = 0; - struct bio *bio = NULL; + /* + * although background is preferred, no one is pending for submission. + * don't issue workqueue for decompression but drop it directly instead. + */ + if (force_fg || nr_bios) + return false; - bi_private = jobqueueset_init(sb, q, fgq, force_fg); - qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; - qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head; + kvfree(container_of(q[JQ_SUBMIT], struct z_erofs_unzip_io_sb, io)); + return true; +} + +static bool z_erofs_vle_submit_all(struct super_block *sb, + z_erofs_next_pcluster_t owned_head, + struct list_head *pagepool, + struct z_erofs_unzip_io *fgq, + bool force_fg) +{ + struct erofs_sb_info *const sbi __maybe_unused = EROFS_SB(sb); + z_erofs_next_pcluster_t qtail[NR_JOBQUEUES]; + struct z_erofs_unzip_io *q[NR_JOBQUEUES]; + struct bio *bio; + void *bi_private; + /* since bio will be NULL, no need to initialize last_index */ + pgoff_t uninitialized_var(last_index); + bool force_submit = false; + unsigned int nr_bios; + + if (owned_head == Z_EROFS_PCLUSTER_TAIL) + return false; + + force_submit = false; + bio = NULL; + nr_bios = 0; + bi_private = jobqueueset_init(sb, qtail, q, fgq, force_fg); /* by default, all need io submission */ q[JQ_SUBMIT]->head = owned_head; do { struct z_erofs_pcluster *pcl; - pgoff_t cur, end; - unsigned int i = 0; - bool bypass = true; + unsigned int clusterpages; + pgoff_t first_index; + struct page *page; + unsigned int i = 0, bypass = 0; + int err; /* no possible 'owned_head' equals the following */ DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); @@ -1299,50 +1241,55 @@ static void z_erofs_submit_queue(struct super_block *sb, pcl = container_of(owned_head, struct z_erofs_pcluster, next); - cur = pcl->obj.index; - end = cur + pcl->pclusterpages; + clusterpages = BIT(pcl->clusterbits); /* close the main owned chain at first */ owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, Z_EROFS_PCLUSTER_TAIL_CLOSED); - do { - struct page *page; + first_index = pcl->obj.index; + force_submit |= (first_index != last_index + 1); - page = pickup_page_for_submission(pcl, i++, pagepool, - MNGD_MAPPING(sbi), - GFP_NOFS); - if (!page) - continue; +repeat: + page = pickup_page_for_submission(pcl, i, pagepool, + MNGD_MAPPING(sbi), + GFP_NOFS); + if (!page) { + force_submit = true; + ++bypass; + goto skippage; + } - if (bio && cur != last_index + 1) { + if (bio && force_submit) { submit_bio_retry: - submit_bio(bio); - bio = NULL; - } + submit_bio(bio); + bio = NULL; + } - if (!bio) { - bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); + if (!bio) { + bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); - bio->bi_end_io = z_erofs_decompressqueue_endio; - bio_set_dev(bio, sb->s_bdev); - bio->bi_iter.bi_sector = (sector_t)cur << - LOG_SECTORS_PER_BLOCK; - bio->bi_private = bi_private; - bio->bi_opf = REQ_OP_READ; - if (f->readahead) - bio->bi_opf |= REQ_RAHEAD; - ++nr_bios; - } + bio->bi_end_io = z_erofs_vle_read_endio; + bio_set_dev(bio, sb->s_bdev); + bio->bi_iter.bi_sector = (sector_t)(first_index + i) << + LOG_SECTORS_PER_BLOCK; + bio->bi_private = bi_private; + bio->bi_opf = REQ_OP_READ; - if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) - goto submit_bio_retry; + ++nr_bios; + } - last_index = cur; - bypass = false; - } while (++cur < end); + err = bio_add_page(bio, page, PAGE_SIZE, 0); + if (err < PAGE_SIZE) + goto submit_bio_retry; - if (!bypass) + force_submit = false; + last_index = first_index + i; +skippage: + if (++i < clusterpages) + goto repeat; + + if (bypass < clusterpages) qtail[JQ_SUBMIT] = &pcl->next; else move_to_bypass_jobqueue(pcl, qtail, owned_head); @@ -1351,42 +1298,40 @@ submit_bio_retry: if (bio) submit_bio(bio); - /* - * although background is preferred, no one is pending for submission. - * don't issue workqueue for decompression but drop it directly instead. - */ - if (!*force_fg && !nr_bios) { - kvfree(q[JQ_SUBMIT]); - return; - } - z_erofs_decompress_kickoff(q[JQ_SUBMIT], *force_fg, nr_bios); + if (postsubmit_is_all_bypassed(q, nr_bios, force_fg)) + return true; + + z_erofs_vle_unzip_kickoff(bi_private, nr_bios); + return true; } -static void z_erofs_runqueue(struct super_block *sb, - struct z_erofs_decompress_frontend *f, - struct list_head *pagepool, bool force_fg) +static void z_erofs_submit_and_unzip(struct super_block *sb, + struct z_erofs_collector *clt, + struct list_head *pagepool, + bool force_fg) { - struct z_erofs_decompressqueue io[NR_JOBQUEUES]; + struct z_erofs_unzip_io io[NR_JOBQUEUES]; - if (f->clt.owned_head == Z_EROFS_PCLUSTER_TAIL) + if (!z_erofs_vle_submit_all(sb, clt->owned_head, + pagepool, io, force_fg)) return; - z_erofs_submit_queue(sb, f, pagepool, io, &force_fg); - /* handle bypass queue (no i/o pclusters) immediately */ - z_erofs_decompress_queue(&io[JQ_BYPASS], pagepool); + /* decompress no I/O pclusters immediately */ + z_erofs_vle_unzip_all(sb, &io[JQ_BYPASS], pagepool); if (!force_fg) return; /* wait until all bios are completed */ - io_wait_event(io[JQ_SUBMIT].u.wait, - !atomic_read(&io[JQ_SUBMIT].pending_bios)); + wait_event(io[JQ_SUBMIT].u.wait, + !atomic_read(&io[JQ_SUBMIT].pending_bios)); - /* handle synchronous decompress queue in the caller context */ - z_erofs_decompress_queue(&io[JQ_SUBMIT], pagepool); + /* let's synchronous decompression */ + z_erofs_vle_unzip_all(sb, &io[JQ_SUBMIT], pagepool); } -static int z_erofs_readpage(struct file *file, struct page *page) +static int z_erofs_vle_normalaccess_readpage(struct file *file, + struct page *page) { struct inode *const inode = page->mapping->host; struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); @@ -1401,7 +1346,7 @@ static int z_erofs_readpage(struct file *file, struct page *page) (void)z_erofs_collector_end(&f.clt); /* if some compressed cluster ready, need submit them anyway */ - z_erofs_runqueue(inode->i_sb, &f, &pagepool, true); + z_erofs_submit_and_unzip(inode->i_sb, &f.clt, &pagepool, true); if (err) erofs_err(inode->i_sb, "failed to read, err [%d]", err); @@ -1417,7 +1362,7 @@ static int z_erofs_readpage(struct file *file, struct page *page) static bool should_decompress_synchronously(struct erofs_sb_info *sbi, unsigned int nr) { - return nr <= 3; + return nr <= sbi->max_sync_decompress_pages; } static int z_erofs_vle_normalaccess_readpages(struct file *filp, @@ -1478,7 +1423,7 @@ static int z_erofs_vle_normalaccess_readpages(struct file *filp, (void)z_erofs_collector_end(&f.clt); - z_erofs_runqueue(inode->i_sb, &f, &pagepool, sync); + z_erofs_submit_and_unzip(inode->i_sb, &f.clt, &pagepool, sync); if (f.map.mpage) put_page(f.map.mpage); @@ -1488,9 +1433,8 @@ static int z_erofs_vle_normalaccess_readpages(struct file *filp, return 0; } - -const struct address_space_operations z_erofs_aops = { - .readpage = z_erofs_readpage, +const struct address_space_operations z_erofs_vle_normalaccess_aops = { + .readpage = z_erofs_vle_normalaccess_readpage, .readpages = z_erofs_vle_normalaccess_readpages, }; diff --git a/fs/erofs/zdata.h b/fs/erofs/zdata.h index 942ee69dff6a..568d5a493876 100644 --- a/fs/erofs/zdata.h +++ b/fs/erofs/zdata.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #ifndef __EROFS_FS_ZDATA_H @@ -10,7 +10,6 @@ #include "internal.h" #include "zpvec.h" -#define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE) #define Z_EROFS_NR_INLINE_PAGEVECS 3 /* @@ -60,17 +59,16 @@ struct z_erofs_pcluster { /* A: point to next chained pcluster or TAILs */ z_erofs_next_pcluster_t next; + /* A: compressed pages (including multi-usage pages) */ + struct page *compressed_pages[Z_EROFS_CLUSTER_MAX_PAGES]; + /* A: lower limit of decompressed length and if full length or not */ unsigned int length; - /* I: physical cluster size in pages */ - unsigned short pclusterpages; - /* I: compression algorithm format */ unsigned char algorithmformat; - - /* A: compressed pages (can be cached or inplaced pages) */ - struct page *compressed_pages[]; + /* I: bit shift of physical cluster size */ + unsigned char clusterbits; }; #define z_erofs_primarycollection(pcluster) (&(pcluster)->primary_collection) @@ -84,8 +82,9 @@ struct z_erofs_pcluster { #define Z_EROFS_PCLUSTER_NIL (NULL) -struct z_erofs_decompressqueue { - struct super_block *sb; +#define Z_EROFS_WORKGROUP_SIZE sizeof(struct z_erofs_pcluster) + +struct z_erofs_unzip_io { atomic_t pending_bios; z_erofs_next_pcluster_t head; @@ -95,6 +94,11 @@ struct z_erofs_decompressqueue { } u; }; +struct z_erofs_unzip_io_sb { + struct z_erofs_unzip_io io; + struct super_block *sb; +}; + #define MNGD_MAPPING(sbi) ((sbi)->managed_cache->i_mapping) static inline bool erofs_page_is_managed(const struct erofs_sb_info *sbi, struct page *page) @@ -173,7 +177,6 @@ static inline void z_erofs_onlinepage_endio(struct page *page) v = atomic_dec_return(u.o); if (!(v & Z_EROFS_ONLINEPAGE_COUNT_MASK)) { - set_page_private(page, 0); ClearPagePrivate(page); if (!PageError(page)) SetPageUptodate(page); diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index adc4b4bcf51e..6553f58fb289 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2018-2019 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #include "internal.h" @@ -11,21 +11,22 @@ int z_erofs_fill_inode(struct inode *inode) { struct erofs_inode *const vi = EROFS_I(inode); - struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); - if (!erofs_sb_has_big_pcluster(sbi) && - vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { + if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { vi->z_advise = 0; vi->z_algorithmtype[0] = 0; vi->z_algorithmtype[1] = 0; vi->z_logical_clusterbits = LOG_BLOCK_SIZE; + vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits; + vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits; set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); } - inode->i_mapping->a_ops = &z_erofs_aops; + + inode->i_mapping->a_ops = &z_erofs_vle_normalaccess_aops; return 0; } -static int z_erofs_fill_inode_lazy(struct inode *inode) +static int fill_inode_lazy(struct inode *inode) { struct erofs_inode *const vi = EROFS_I(inode); struct super_block *const sb = inode->i_sb; @@ -51,8 +52,7 @@ static int z_erofs_fill_inode_lazy(struct inode *inode) if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) goto out_unlock; - DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && - vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY); + DBG_BUGON(vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY); pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize + vi->xattr_isize, 8); @@ -77,22 +77,18 @@ static int z_erofs_fill_inode_lazy(struct inode *inode) } vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7); - if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && - vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | - Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { - erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", - vi->nid); - err = -EFSCORRUPTED; - goto unmap_done; - } - if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION && - !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ - !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { - erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", - vi->nid); - err = -EFSCORRUPTED; + vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits + + ((h->h_clusterbits >> 3) & 3); + + if (vi->z_physical_clusterbits[0] != LOG_BLOCK_SIZE) { + erofs_err(sb, "unsupported physical clusterbits %u for nid %llu, please upgrade kernel", + vi->z_physical_clusterbits[0], vi->nid); + err = -EOPNOTSUPP; goto unmap_done; } + + vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits + + ((h->h_clusterbits >> 5) & 7); /* paired with smp_mb() at the beginning of the function */ smp_mb(); set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); @@ -115,7 +111,7 @@ struct z_erofs_maprecorder { u8 type; u16 clusterofs; u16 delta[2]; - erofs_blk_t pblk, compressedlcs; + erofs_blk_t pblk; }; static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, @@ -150,8 +146,8 @@ static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, return 0; } -static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, - unsigned long lcn) +static int vle_legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, + unsigned long lcn) { struct inode *const inode = m->inode; struct erofs_inode *const vi = EROFS_I(inode); @@ -178,15 +174,6 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: m->clusterofs = 1 << vi->z_logical_clusterbits; m->delta[0] = le16_to_cpu(di->di_u.delta[0]); - if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) { - if (!(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) { - DBG_BUGON(1); - return -EFSCORRUPTED; - } - m->compressedlcs = m->delta[0] & - ~Z_EROFS_VLE_DI_D0_CBLKCNT; - m->delta[0] = 1; - } m->delta[1] = le16_to_cpu(di->di_u.delta[1]); break; case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: @@ -227,7 +214,6 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m, unsigned int vcnt, base, lo, encodebits, nblk; int i; u8 *in, type; - bool big_pcluster; if (1 << amortizedshift == 4 && lclusterbits <= 14) vcnt = 2; @@ -236,7 +222,6 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m, else return -EOPNOTSUPP; - big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; base = round_down(eofs, vcnt << amortizedshift); in = m->kaddr + base; @@ -248,15 +233,7 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m, m->type = type; if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { m->clusterofs = 1 << lclusterbits; - if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { - if (!big_pcluster) { - DBG_BUGON(1); - return -EFSCORRUPTED; - } - m->compressedlcs = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; - m->delta[0] = 1; - return 0; - } else if (i + 1 != (int)vcnt) { + if (i + 1 != vcnt) { m->delta[0] = lo; return 0; } @@ -269,48 +246,22 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m, in, encodebits * (i - 1), &type); if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) lo = 0; - else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) - lo = 1; m->delta[0] = lo + 1; return 0; } m->clusterofs = lo; m->delta[0] = 0; /* figout out blkaddr (pblk) for HEAD lclusters */ - if (!big_pcluster) { - nblk = 1; - while (i > 0) { - --i; - lo = decode_compactedbits(lclusterbits, lomask, - in, encodebits * i, &type); - if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) - i -= lo; + nblk = 1; + while (i > 0) { + --i; + lo = decode_compactedbits(lclusterbits, lomask, + in, encodebits * i, &type); + if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) + i -= lo; - if (i >= 0) - ++nblk; - } - } else { - nblk = 0; - while (i > 0) { - --i; - lo = decode_compactedbits(lclusterbits, lomask, - in, encodebits * i, &type); - if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { - if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { - --i; - nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; - continue; - } - /* bigpcluster shouldn't have plain d0 == 1 */ - if (lo <= 1) { - DBG_BUGON(1); - return -EFSCORRUPTED; - } - i -= lo - 2; - continue; - } + if (i >= 0) ++nblk; - } } in += (vcnt << amortizedshift) - sizeof(__le32); m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; @@ -368,13 +319,13 @@ out: return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos)); } -static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, - unsigned int lcn) +static int vle_load_cluster_from_disk(struct z_erofs_maprecorder *m, + unsigned int lcn) { const unsigned int datamode = EROFS_I(m->inode)->datalayout; if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY) - return legacy_load_cluster_from_disk(m, lcn); + return vle_legacy_load_cluster_from_disk(m, lcn); if (datamode == EROFS_INODE_FLAT_COMPRESSION) return compacted_load_cluster_from_disk(m, lcn); @@ -382,8 +333,8 @@ static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, return -EINVAL; } -static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, - unsigned int lookback_distance) +static int vle_extent_lookback(struct z_erofs_maprecorder *m, + unsigned int lookback_distance) { struct erofs_inode *const vi = EROFS_I(m->inode); struct erofs_map_blocks *const map = m->map; @@ -400,7 +351,7 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, /* load extent head logical cluster if needed */ lcn -= lookback_distance; - err = z_erofs_load_cluster_from_disk(m, lcn); + err = vle_load_cluster_from_disk(m, lcn); if (err) return err; @@ -413,10 +364,10 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, DBG_BUGON(1); return -EFSCORRUPTED; } - return z_erofs_extent_lookback(m, m->delta[0]); + return vle_extent_lookback(m, m->delta[0]); case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: map->m_flags &= ~EROFS_MAP_ZIPPED; - fallthrough; + /* fallthrough */ case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: map->m_la = (lcn << lclusterbits) | m->clusterofs; break; @@ -430,75 +381,6 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, return 0; } -static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, - unsigned int initial_lcn) -{ - struct erofs_inode *const vi = EROFS_I(m->inode); - struct erofs_map_blocks *const map = m->map; - const unsigned int lclusterbits = vi->z_logical_clusterbits; - unsigned long lcn; - int err; - - DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN && - m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD); - if (!(map->m_flags & EROFS_MAP_ZIPPED) || - !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) { - map->m_plen = 1 << lclusterbits; - return 0; - } - - lcn = m->lcn + 1; - if (m->compressedlcs) - goto out; - - err = z_erofs_load_cluster_from_disk(m, lcn); - if (err) - return err; - - /* - * If the 1st NONHEAD lcluster has already been handled initially w/o - * valid compressedlcs, which means at least it mustn't be CBLKCNT, or - * an internal implemenatation error is detected. - * - * The following code can also handle it properly anyway, but let's - * BUG_ON in the debugging mode only for developers to notice that. - */ - DBG_BUGON(lcn == initial_lcn && - m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD); - - switch (m->type) { - case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: - case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: - /* - * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type - * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. - */ - m->compressedlcs = 1; - break; - case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: - if (m->delta[0] != 1) - goto err_bonus_cblkcnt; - if (m->compressedlcs) - break; - fallthrough; - default: - erofs_err(m->inode->i_sb, - "cannot found CBLKCNT @ lcn %lu of nid %llu", - lcn, vi->nid); - DBG_BUGON(1); - return -EFSCORRUPTED; - } -out: - map->m_plen = m->compressedlcs << lclusterbits; - return 0; -err_bonus_cblkcnt: - erofs_err(m->inode->i_sb, - "bogus CBLKCNT @ lcn %lu of nid %llu", - lcn, vi->nid); - DBG_BUGON(1); - return -EFSCORRUPTED; -} - int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, int flags) @@ -510,7 +392,6 @@ int z_erofs_map_blocks_iter(struct inode *inode, }; int err = 0; unsigned int lclusterbits, endoff; - unsigned long initial_lcn; unsigned long long ofs, end; trace_z_erofs_map_blocks_iter_enter(inode, map, flags); @@ -523,16 +404,16 @@ int z_erofs_map_blocks_iter(struct inode *inode, goto out; } - err = z_erofs_fill_inode_lazy(inode); + err = fill_inode_lazy(inode); if (err) goto out; lclusterbits = vi->z_logical_clusterbits; ofs = map->m_la; - initial_lcn = ofs >> lclusterbits; + m.lcn = ofs >> lclusterbits; endoff = ofs & ((1 << lclusterbits) - 1); - err = z_erofs_load_cluster_from_disk(&m, initial_lcn); + err = vle_load_cluster_from_disk(&m, m.lcn); if (err) goto unmap_out; @@ -543,7 +424,7 @@ int z_erofs_map_blocks_iter(struct inode *inode, case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: if (endoff >= m.clusterofs) map->m_flags &= ~EROFS_MAP_ZIPPED; - fallthrough; + /* fallthrough */ case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: if (endoff >= m.clusterofs) { map->m_la = (m.lcn << lclusterbits) | m.clusterofs; @@ -560,10 +441,10 @@ int z_erofs_map_blocks_iter(struct inode *inode, end = (m.lcn << lclusterbits) | m.clusterofs; map->m_flags |= EROFS_MAP_FULL_MAPPED; m.delta[0] = 1; - fallthrough; + /* fallthrough */ case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: - /* get the corresponding first chunk */ - err = z_erofs_extent_lookback(&m, m.delta[0]); + /* get the correspoinding first chunk */ + err = vle_extent_lookback(&m, m.delta[0]); if (err) goto unmap_out; break; @@ -576,12 +457,10 @@ int z_erofs_map_blocks_iter(struct inode *inode, } map->m_llen = end - map->m_la; + map->m_plen = 1 << lclusterbits; map->m_pa = blknr_to_addr(m.pblk); map->m_flags |= EROFS_MAP_MAPPED; - err = z_erofs_get_extent_compressedlen(&m, initial_lcn); - if (err) - goto out; unmap_out: if (m.kaddr) kunmap_atomic(m.kaddr); diff --git a/fs/erofs/zpvec.h b/fs/erofs/zpvec.h index 52898176ef31..6a20b2c3a24c 100644 --- a/fs/erofs/zpvec.h +++ b/fs/erofs/zpvec.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2018 HUAWEI, Inc. - * https://www.huawei.com/ + * http://www.huawei.com/ * Created by Gao Xiang */ #ifndef __EROFS_FS_ZPVEC_H