A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-05 19:31:57 -04:00
Code Issues Projects Releases Packages Wiki Activity

fs:EROFS:Porting 5.10 erofs to 5.4

Browse source 
Porting erfos changes from 5.10 common kernel
And some glue code to make it pass build on 5.4

be3f6035831a erofs: fix unsafe pagevec reuse of hooked pclusters
6c1ad56b2d78 erofs: remove the occupied parameter from z_erofs_pagevec_enqueue()
ee75174f6ab9 erofs: don't trigger WARN() when decompression fails
8326be9e5121 erofs: fix up erofs_lookup tracepoint
ca9ee53cea1f18edf318a01a19653ae3847474a9
UPSTREAM: erofs: fix deadlock when shrink erofs slab
b943d328889d25cdb1aa397880c079b7d55a0901
UPSTREAM: erofs: fix unsafe pagevec reuse of hooked pclusters
028f7128c4fbb75ba113824f8600ca9a7097a0ed
UPSTREAM: erofs: remove the occupied parameter from z_erofs_pagevec_enqueue()
ee75174f6ab93611e1623f0ae5590beda008aef5
erofs: don't trigger WARN() when decompression fail
dcd77f0b7453 UPSTREAM: erofs: fix 1 lcluster-sized pcluster for big pcluster
e085d3f0d0dc UPSTREAM: erofs: enable big pcluster feature
ed0607cc5256 UPSTREAM: erofs: support decompress big pcluster for lz4 backend
d34cb6cdc044 UPSTREAM: erofs: support parsing big pcluster compact indexes
051d76b899bd UPSTREAM: erofs: support parsing big pcluster compress indexes
d14993160141 UPSTREAM: erofs: adjust per-CPU buffers according to max_pclusterblks
95a1d5df840a UPSTREAM: erofs: add big physical cluster definition
8043aaed1dad UPSTREAM: erofs: fix up inplace I/O pointer for big pcluster
6ad2f8f1699c UPSTREAM: erofs: introduce physical cluster slab pools
432f58b100f0 UPSTREAM: erofs: introduce multipage per-CPU buffers
571c9a0bd334 UPSTREAM: erofs: remove a void EROFS_VERSION macro set in Makefile
431d73396dad UPSTREAM: erofs: reserve physical_clusterbits[]
89dbc6246a86 UPSTREAM: erofs: Clean up spelling mistakes found in fs/erofs
ac1f14e9d5a8 UPSTREAM: erofs: add on-disk compression configurations
cd21e62366bb UPSTREAM: erofs: introduce on-disk lz4 fs configurations
e17fd2ac9dfc UPSTREAM: erofs: introduce erofs_sb_has_xxx() helpers
ba1a3d1fb2db UPSTREAM: erofs: don't use erofs_map_blocks() any more
384b2cdaf83e UPSTREAM: erofs: complete a missing case for inplace I/O
8326be9e51218d23ebe52c8d4c2d7c23302f6e44
erofs: fix up erofs_lookup tracepoint
51c19f4a62129bca096df760c7690833958ce9cf
erofs: fix error return code in erofs_read_superblock()
06229c49eb7f UPSTREAM: erofs: support adjust lz4 history window size
0a24d25f0882 UPSTREAM: erofs: use sync decompression for atomic contexts only
0ca4eafb3928 UPSTREAM: erofs: use workqueue decompression for atomic contexts only

4ae1c8a4d06a UPSTREAM: erofs: force inplace I/O under low memory scenario
46a00756cbf8 UPSTREAM: erofs: insert to managed cache after adding to pcl
b1c757466f42 UPSTREAM: erofs: get rid of magical Z_EROFS_MAPPING_STAGING

Change-Id: Ibc0d16af7424aa02806ec35b431d4faa28a126cd
Signed-off-by: William Liu <a17671@motorola.com>
Reviewed-on: https://gerrit.mot.com/2379154
SME-Granted: SME Approvals Granted
SLTApproved: Slta Waiver
Tested-by: Jira Key
Reviewed-by: Xiangpo Zhao <zhaoxp3@motorola.com>
Submit-Approved: Jira Key
This commit is contained in:
William Liu 2022-09-08 19:32:29 +08:00 • committed by Xiaojun Ji
commit fa3b0b74bb
19 changed files with 1454 additions and 905 deletions
Download patch file Download diff file Expand all files Collapse all files

15
fs/erofs/Kconfig
View file

@ -3,6 +3,7 @@
config EROFS_FS
tristate "EROFS filesystem support"
depends on BLOCK
select LIBCRC32C
help
EROFS (Enhanced Read-Only File System) is a lightweight
read-only file system with modern designs (eg. page-sized
@ -75,17 +76,3 @@ config EROFS_FS_ZIP
If you don't want to enable compression feature, say N.
config EROFS_FS_CLUSTER_PAGE_LIMIT
int "EROFS Cluster Pages Hard Limit"
depends on EROFS_FS_ZIP
range 1 256
default "1"
help
Indicates maximum # of pages of a compressed
physical cluster.
For example, if files in a image were compressed
into 8k-unit, hard limit should not be configured
less than 2. Otherwise, the image will be refused
to mount on this kernel.

7
fs/erofs/Makefile
View file

@ -1,11 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
EROFS_VERSION = "1.0"
ccflags-y += -DEROFS_VERSION=\"$(EROFS_VERSION)\"
obj-$(CONFIG_EROFS_FS) += erofs.o
erofs-objs := super.o inode.o data.o namei.o dir.o utils.o
erofs-objs := super.o inode.o data.o namei.o dir.o utils.o pcpubuf.o
erofs-$(CONFIG_EROFS_FS_XATTR) += xattr.o
erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o

62
fs/erofs/compress.h
View file

@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2019 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_FS_COMPRESS_H
@ -26,30 +26,58 @@ struct z_erofs_decompress_req {
bool inplace_io, partial_decoding;
};
/* some special page->private (unsigned long, see below) */
#define Z_EROFS_SHORTLIVED_PAGE (-1UL << 2)
#define Z_EROFS_PREALLOCATED_PAGE (-2UL << 2)
/*
* - 0x5A110C8D ('sallocated', Z_EROFS_MAPPING_STAGING) -
* used to mark temporary allocated pages from other
* file/cached pages and NULL mapping pages.
* For all pages in a pcluster, page->private should be one of
* Type Last 2bits page->private
* short-lived page 00 Z_EROFS_SHORTLIVED_PAGE
* preallocated page (tryalloc) 00 Z_EROFS_PREALLOCATED_PAGE
* cached/managed page 00 pointer to z_erofs_pcluster
* online page (file-backed, 01/10/11 sub-index << 2 | count
* some pages can be used for inplace I/O)
*
* page->mapping should be one of
* Type page->mapping
* short-lived page NULL
* preallocated page NULL
* cached/managed page non-NULL or NULL (invalidated/truncated page)
* online page non-NULL
*
* For all managed pages, PG_private should be set with 1 extra refcount,
* which is used for page reclaim / migration.
*/
#define Z_EROFS_MAPPING_STAGING ((void *)0x5A110C8D)
/* check if a page is marked as staging */
static inline bool z_erofs_page_is_staging(struct page *page)
/*
* short-lived pages are pages directly from buddy system with specific
* page->private (no need to set PagePrivate since these are non-LRU /
* non-movable pages and bypass reclaim / migration code).
*/
static inline bool z_erofs_is_shortlived_page(struct page *page)
{
return page->mapping == Z_EROFS_MAPPING_STAGING;
}
static inline bool z_erofs_put_stagingpage(struct list_head *pagepool,
struct page *page)
{
if (!z_erofs_page_is_staging(page))
if (page->private != Z_EROFS_SHORTLIVED_PAGE)
return false;
/* staging pages should not be used by others at the same time */
if (page_ref_count(page) > 1)
DBG_BUGON(page->mapping);
return true;
}
static inline bool z_erofs_put_shortlivedpage(struct list_head *pagepool,
struct page *page)
{
if (!z_erofs_is_shortlived_page(page))
return false;
/* short-lived pages should not be used by others at the same time */
if (page_ref_count(page) > 1) {
put_page(page);
else
} else {
/* follow the pcluster rule above. */
set_page_private(page, 0);
list_add(&page->lru, pagepool);
}
return true;
}

23
fs/erofs/data.c
View file

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "internal.h"
@ -109,21 +109,6 @@ err_out:
return err;
}
int erofs_map_blocks(struct inode *inode,
struct erofs_map_blocks *map, int flags)
{
if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) {
int err = z_erofs_map_blocks_iter(inode, map, flags);
if (map->mpage) {
put_page(map->mpage);
map->mpage = NULL;
}
return err;
}
return erofs_map_blocks_flatmode(inode, map, flags);
}
static inline struct bio *erofs_read_raw_page(struct bio *bio,
struct address_space *mapping,
struct page *page,
@ -159,7 +144,7 @@ submit_bio_retry:
erofs_blk_t blknr;
unsigned int blkoff;
err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
err = erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW);
if (err)
goto err_out;
@ -224,7 +209,7 @@ submit_bio_retry:
bio_set_dev(bio, sb->s_bdev);
bio->bi_iter.bi_sector = (sector_t)blknr <<
LOG_SECTORS_PER_BLOCK;
bio->bi_opf = REQ_OP_READ;
bio->bi_opf = REQ_OP_READ | (ra ? REQ_RAHEAD : 0);
}
err = bio_add_page(bio, page, PAGE_SIZE, 0);
@ -337,7 +322,7 @@ static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
return 0;
}
if (!erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW))
if (!erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW))
return erofs_blknr(map.m_pa);
return 0;

291
fs/erofs/decompressor.c
View file

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2019 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "compress.h"
@ -28,6 +28,42 @@ struct z_erofs_decompressor {
char *name;
};
int z_erofs_load_lz4_config(struct super_block *sb,
struct erofs_super_block *dsb,
struct z_erofs_lz4_cfgs *lz4, int size)
{
struct erofs_sb_info *sbi = EROFS_SB(sb);
u16 distance;
if (lz4) {
if (size < sizeof(struct z_erofs_lz4_cfgs)) {
erofs_err(sb, "invalid lz4 cfgs, size=%u", size);
return -EINVAL;
}
distance = le16_to_cpu(lz4->max_distance);
sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks);
if (!sbi->lz4.max_pclusterblks) {
sbi->lz4.max_pclusterblks = 1; /* reserved case */
} else if (sbi->lz4.max_pclusterblks >
Z_EROFS_PCLUSTER_MAX_SIZE / EROFS_BLKSIZ) {
erofs_err(sb, "too large lz4 pclusterblks %u",
sbi->lz4.max_pclusterblks);
return -EINVAL;
} else if (sbi->lz4.max_pclusterblks >= 2) {
erofs_info(sb, "EXPERIMENTAL big pcluster feature in use. Use at your own risk!");
}
} else {
distance = le16_to_cpu(dsb->u1.lz4_max_distance);
sbi->lz4.max_pclusterblks = 1;
}
sbi->lz4.max_distance_pages = distance ?
DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
LZ4_MAX_DISTANCE_PAGES;
return erofs_pcpubuf_growsize(sbi->lz4.max_pclusterblks);
}
static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
struct list_head *pagepool)
{
@ -36,6 +72,8 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
BITS_PER_LONG)] = { 0 };
unsigned int lz4_max_distance_pages =
EROFS_SB(rq->sb)->lz4.max_distance_pages;
void *kaddr = NULL;
unsigned int i, j, top;
@ -44,14 +82,14 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
struct page *const page = rq->out[i];
struct page *victim;
if (j >= LZ4_MAX_DISTANCE_PAGES)
if (j >= lz4_max_distance_pages)
j = 0;
/* 'valid' bounced can only be tested after a complete round */
if (test_bit(j, bounced)) {
DBG_BUGON(i < LZ4_MAX_DISTANCE_PAGES);
DBG_BUGON(top >= LZ4_MAX_DISTANCE_PAGES);
availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES];
DBG_BUGON(i < lz4_max_distance_pages);
DBG_BUGON(top >= lz4_max_distance_pages);
availables[top++] = rq->out[i - lz4_max_distance_pages];
}
if (page) {
@ -77,108 +115,149 @@ static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
victim = availables[--top];
get_page(victim);
} else {
victim = erofs_allocpage(pagepool, GFP_KERNEL, false);
if (!victim)
return -ENOMEM;
victim->mapping = Z_EROFS_MAPPING_STAGING;
victim = erofs_allocpage(pagepool,
GFP_KERNEL | __GFP_NOFAIL);
set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
}
rq->out[i] = victim;
}
return kaddr ? 1 : 0;
}
static void *generic_copy_inplace_data(struct z_erofs_decompress_req *rq,
u8 *src, unsigned int pageofs_in)
static void *z_erofs_handle_inplace_io(struct z_erofs_decompress_req *rq,
void *inpage, unsigned int *inputmargin, int *maptype,
bool support_0padding)
{
/*
* if in-place decompression is ongoing, those decompressed
* pages should be copied in order to avoid being overlapped.
*/
struct page **in = rq->in;
u8 *const tmp = erofs_get_pcpubuf(0);
u8 *tmpp = tmp;
unsigned int inlen = rq->inputsize - pageofs_in;
unsigned int count = min_t(uint, inlen, PAGE_SIZE - pageofs_in);
unsigned int nrpages_in, nrpages_out;
unsigned int ofull, oend, inputsize, total, i, j;
struct page **in;
void *src, *tmp;
while (tmpp < tmp + inlen) {
if (!src)
src = kmap_atomic(*in);
memcpy(tmpp, src + pageofs_in, count);
kunmap_atomic(src);
src = NULL;
tmpp += count;
pageofs_in = 0;
count = PAGE_SIZE;
++in;
inputsize = rq->inputsize;
nrpages_in = PAGE_ALIGN(inputsize) >> PAGE_SHIFT;
oend = rq->pageofs_out + rq->outputsize;
ofull = PAGE_ALIGN(oend);
nrpages_out = ofull >> PAGE_SHIFT;
if (rq->inplace_io) {
if (rq->partial_decoding || !support_0padding ||
ofull - oend < LZ4_DECOMPRESS_INPLACE_MARGIN(inputsize))
goto docopy;
for (i = 0; i < nrpages_in; ++i) {
DBG_BUGON(rq->in[i] == NULL);
for (j = 0; j < nrpages_out - nrpages_in + i; ++j)
if (rq->out[j] == rq->in[i])
goto docopy;
}
}
return tmp;
if (nrpages_in <= 1) {
*maptype = 0;
return inpage;
}
kunmap_atomic(inpage);
might_sleep();
src = erofs_vm_map_ram(rq->in, nrpages_in);
if (!src)
return ERR_PTR(-ENOMEM);
*maptype = 1;
return src;
docopy:
/* Or copy compressed data which can be overlapped to per-CPU buffer */
in = rq->in;
src = erofs_get_pcpubuf(nrpages_in);
if (!src) {
DBG_BUGON(1);
kunmap_atomic(inpage);
return ERR_PTR(-EFAULT);
}
tmp = src;
total = rq->inputsize;
while (total) {
unsigned int page_copycnt =
min_t(unsigned int, total, PAGE_SIZE - *inputmargin);
if (!inpage)
inpage = kmap_atomic(*in);
memcpy(tmp, inpage + *inputmargin, page_copycnt);
kunmap_atomic(inpage);
inpage = NULL;
tmp += page_copycnt;
total -= page_copycnt;
++in;
*inputmargin = 0;
}
*maptype = 2;
return src;
}
static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out)
{
unsigned int inputmargin, inlen;
u8 *src;
bool copied, support_0padding;
int ret;
unsigned int inputmargin;
u8 *headpage, *src;
bool support_0padding;
int ret, maptype;
if (rq->inputsize > PAGE_SIZE)
return -EOPNOTSUPP;
src = kmap_atomic(*rq->in);
DBG_BUGON(*rq->in == NULL);
headpage = kmap_atomic(*rq->in);
inputmargin = 0;
support_0padding = false;
/* decompression inplace is only safe when 0padding is enabled */
if (EROFS_SB(rq->sb)->feature_incompat &
EROFS_FEATURE_INCOMPAT_LZ4_0PADDING) {
if (erofs_sb_has_lz4_0padding(EROFS_SB(rq->sb))) {
support_0padding = true;
while (!src[inputmargin & ~PAGE_MASK])
while (!headpage[inputmargin & ~PAGE_MASK])
if (!(++inputmargin & ~PAGE_MASK))
break;
if (inputmargin >= rq->inputsize) {
kunmap_atomic(src);
kunmap_atomic(headpage);
return -EIO;
}
}
copied = false;
inlen = rq->inputsize - inputmargin;
if (rq->inplace_io) {
const uint oend = (rq->pageofs_out +
rq->outputsize) & ~PAGE_MASK;
const uint nr = PAGE_ALIGN(rq->pageofs_out +
rq->outputsize) >> PAGE_SHIFT;
rq->inputsize -= inputmargin;
src = z_erofs_handle_inplace_io(rq, headpage, &inputmargin, &maptype,
support_0padding);
if (IS_ERR(src))
return PTR_ERR(src);
if (rq->partial_decoding || !support_0padding ||
rq->out[nr - 1] != rq->in[0] ||
rq->inputsize - oend <
LZ4_DECOMPRESS_INPLACE_MARGIN(inlen)) {
src = generic_copy_inplace_data(rq, src, inputmargin);
inputmargin = 0;
copied = true;
}
}
/* legacy format could compress extra data in a pcluster. */
if (rq->partial_decoding || !support_0padding)
ret = LZ4_decompress_safe_partial(src + inputmargin, out,
rq->inputsize, rq->outputsize, rq->outputsize);
else
ret = LZ4_decompress_safe(src + inputmargin, out,
rq->inputsize, rq->outputsize);
if (ret != rq->outputsize) {
erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
ret, rq->inputsize, inputmargin, rq->outputsize);
ret = LZ4_decompress_safe_partial(src + inputmargin, out,
inlen, rq->outputsize,
rq->outputsize);
if (ret < 0) {
erofs_err(rq->sb, "failed to decompress, in[%u, %u] out[%u]",
inlen, inputmargin, rq->outputsize);
WARN_ON(1);
print_hex_dump(KERN_DEBUG, "[ in]: ", DUMP_PREFIX_OFFSET,
16, 1, src + inputmargin, inlen, true);
16, 1, src + inputmargin, rq->inputsize, true);
print_hex_dump(KERN_DEBUG, "[out]: ", DUMP_PREFIX_OFFSET,
16, 1, out, rq->outputsize, true);
if (ret >= 0)
memset(out + ret, 0, rq->outputsize - ret);
ret = -EIO;
}
if (copied)
erofs_put_pcpubuf(src);
else
if (maptype == 0) {
kunmap_atomic(src);
} else if (maptype == 1) {
vm_unmap_ram(src, PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT);
} else if (maptype == 2) {
erofs_put_pcpubuf(src);
} else {
DBG_BUGON(1);
return -EFAULT;
}
return ret;
}
@ -228,57 +307,51 @@ static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq,
const struct z_erofs_decompressor *alg = decompressors + rq->alg;
unsigned int dst_maptype;
void *dst;
int ret, i;
int ret;
if (nrpages_out == 1 && !rq->inplace_io) {
DBG_BUGON(!*rq->out);
dst = kmap_atomic(*rq->out);
dst_maptype = 0;
goto dstmap_out;
}
/*
* For the case of small output size (especially much less
* than PAGE_SIZE), memcpy the decompressed data rather than
* compressed data is preferred.
*/
if (rq->outputsize <= PAGE_SIZE * 7 / 8) {
dst = erofs_get_pcpubuf(0);
if (IS_ERR(dst))
return PTR_ERR(dst);
rq->inplace_io = false;
ret = alg->decompress(rq, dst);
if (!ret)
copy_from_pcpubuf(rq->out, dst, rq->pageofs_out,
rq->outputsize);
erofs_put_pcpubuf(dst);
return ret;
/* two optimized fast paths only for non bigpcluster cases yet */
if (rq->inputsize <= PAGE_SIZE) {
if (nrpages_out == 1 && !rq->inplace_io) {
DBG_BUGON(!*rq->out);
dst = kmap_atomic(*rq->out);
dst_maptype = 0;
goto dstmap_out;
}
/*
* For the case of small output size (especially much less
* than PAGE_SIZE), memcpy the decompressed data rather than
* compressed data is preferred.
*/
if (rq->outputsize <= PAGE_SIZE * 7 / 8) {
dst = erofs_get_pcpubuf(1);
if (IS_ERR(dst))
return PTR_ERR(dst);
rq->inplace_io = false;
ret = alg->decompress(rq, dst);
if (!ret)
copy_from_pcpubuf(rq->out, dst, rq->pageofs_out,
rq->outputsize);
erofs_put_pcpubuf(dst);
return ret;
}
}
/* general decoding path which can be used for all cases */
ret = alg->prepare_destpages(rq, pagepool);
if (ret < 0) {
if (ret < 0)
return ret;
} else if (ret) {
if (ret) {
dst = page_address(*rq->out);
dst_maptype = 1;
goto dstmap_out;
}
i = 0;
while (1) {
dst = vm_map_ram(rq->out, nrpages_out, -1, PAGE_KERNEL);
/* retry two more times (totally 3 times) */
if (dst || ++i >= 3)
break;
vm_unmap_aliases();
}
dst = erofs_vm_map_ram(rq->out, nrpages_out);
if (!dst)
return -ENOMEM;
dst_maptype = 2;
dstmap_out:

2
fs/erofs/dir.c
View file

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "internal.h"

56
fs/erofs/erofs_fs.h
View file

@ -3,7 +3,7 @@
* EROFS (Enhanced ROM File System) on-disk format definition
*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_FS_H
@ -11,20 +11,29 @@
#define EROFS_SUPER_OFFSET 1024
#define EROFS_FEATURE_COMPAT_SB_CHKSUM 0x00000001
/*
* Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should
* be incompatible with this kernel version.
*/
#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001
#define EROFS_ALL_FEATURE_INCOMPAT EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002
#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002
#define EROFS_ALL_FEATURE_INCOMPAT \
(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER)
/* 128-byte erofs on-disk super block */
#define EROFS_SB_EXTSLOT_SIZE 16
/* erofs on-disk super block (currently 128 bytes) */
struct erofs_super_block {
__le32 magic; /* file system magic number */
__le32 checksum; /* crc32c(super_block) */
__le32 feature_compat;
__u8 blkszbits; /* support block_size == PAGE_SIZE only */
__u8 reserved;
__u8 sb_extslots; /* superblock size = 128 + sb_extslots * 16 */
__le16 root_nid; /* nid of root directory */
__le64 inos; /* total valid ino # (== f_files - f_favail) */
@ -37,8 +46,13 @@ struct erofs_super_block {
__u8 uuid[16]; /* 128-bit uuid for volume */
__u8 volume_name[16]; /* volume name */
__le32 feature_incompat;
__u8 reserved2[44];
union {
/* bitmap for available compression algorithms */
__le16 available_compr_algs;
/* customized sliding window size instead of 64k by default */
__le16 lz4_max_distance;
} __packed u1;
__u8 reserved2[42];
};
/*
@ -191,20 +205,33 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)
e->e_name_len + le16_to_cpu(e->e_value_size));
}
/* maximum supported size of a physical compression cluster */
#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024)
/* available compression algorithm types (for h_algorithmtype) */
enum {
Z_EROFS_COMPRESSION_LZ4 = 0,
Z_EROFS_COMPRESSION_MAX
};
#define Z_EROFS_ALL_COMPR_ALGS (1 << (Z_EROFS_COMPRESSION_MAX - 1))
/* 14 bytes (+ length field = 16 bytes) */
struct z_erofs_lz4_cfgs {
__le16 max_distance;
__le16 max_pclusterblks;
u8 reserved[10];
} __packed;
/*
* bit 0 : COMPACTED_2B indexes (0 - off; 1 - on)
* e.g. for 4k logical cluster size, 4B if compacted 2B is off;
* (4B) + 2B + (4B) if compacted 2B is on.
* bit 1 : HEAD1 big pcluster (0 - off; 1 - on)
* bit 2 : HEAD2 big pcluster (0 - off; 1 - on)
*/
#define Z_EROFS_ADVISE_COMPACTED_2B_BIT 0
#define Z_EROFS_ADVISE_COMPACTED_2B (1 << Z_EROFS_ADVISE_COMPACTED_2B_BIT)
#define Z_EROFS_ADVISE_COMPACTED_2B 0x0001
#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002
#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004
struct z_erofs_map_header {
__le32 h_reserved1;
@ -216,9 +243,7 @@ struct z_erofs_map_header {
__u8 h_algorithmtype;
/*
* bit 0-2 : logical cluster bits - 12, e.g. 0 for 4096;
* bit 3-4 : (physical - logical) cluster bits of head 1:
* For example, if logical clustersize = 4096, 1 for 8192.
* bit 5-7 : (physical - logical) cluster bits of head 2.
* bit 3-7 : reserved.
*/
__u8 h_clusterbits;
};
@ -261,6 +286,13 @@ enum {
#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS 2
#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT 0
/*
* D0_CBLKCNT will be marked _only_ at the 1st non-head lcluster to store the
* compressed block count of a compressed extent (in logical clusters, aka.
* block count of a pcluster).
*/
#define Z_EROFS_VLE_DI_D0_CBLKCNT (1 << 11)
struct z_erofs_vle_decompressed_index {
__le16 di_advise;
/* where to decompress in the head cluster */

8
fs/erofs/inode.c
View file

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "xattr.h"
@ -356,27 +356,21 @@ int erofs_getattr(const struct path *path, struct kstat *stat,
const struct inode_operations erofs_generic_iops = {
.getattr = erofs_getattr,
#ifdef CONFIG_EROFS_FS_XATTR
.listxattr = erofs_listxattr,
#endif
.get_acl = erofs_get_acl,
};
const struct inode_operations erofs_symlink_iops = {
.get_link = page_get_link,
.getattr = erofs_getattr,
#ifdef CONFIG_EROFS_FS_XATTR
.listxattr = erofs_listxattr,
#endif
.get_acl = erofs_get_acl,
};
const struct inode_operations erofs_fast_symlink_iops = {
.get_link = simple_get_link,
.getattr = erofs_getattr,
#ifdef CONFIG_EROFS_FS_XATTR
.listxattr = erofs_listxattr,
#endif
.get_acl = erofs_get_acl,
};

130
fs/erofs/internal.h
View file

@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_INTERNAL_H
@ -46,25 +46,43 @@ typedef u64 erofs_off_t;
/* data type for filesystem-wide blocks number */
typedef u32 erofs_blk_t;
struct erofs_fs_context {
#ifdef CONFIG_EROFS_FS_ZIP
/* current strategy of how to use managed cache */
unsigned char cache_strategy;
/* strategy of sync decompression (false - auto, true - force on) */
bool readahead_sync_decompress;
/* threshold for decompression synchronously */
unsigned int max_sync_decompress_pages;
#endif
unsigned int mount_opt;
};
/* all filesystem-wide lz4 configurations */
struct erofs_sb_lz4_info {
/* # of pages needed for EROFS lz4 rolling decompression */
u16 max_distance_pages;
/* maximum possible blocks for pclusters in the filesystem */
u16 max_pclusterblks;
};
struct erofs_sb_info {
#ifdef CONFIG_EROFS_FS_ZIP
/* list for all registered superblocks, mainly for shrinker */
struct list_head list;
struct mutex umount_mutex;
/* the dedicated workstation for compression */
struct radix_tree_root workstn_tree;
/* threshold for decompression synchronously */
unsigned int max_sync_decompress_pages;
/* managed XArray arranged in physical block number */
struct xarray managed_pslots;
unsigned int shrinker_run_no;
/* current strategy of how to use managed cache */
unsigned char cache_strategy;
u16 available_compr_algs;
/* pseudo inode to manage cached pages */
struct inode *managed_cache;
struct erofs_sb_lz4_info lz4;
#endif /* CONFIG_EROFS_FS_ZIP */
u32 blocks;
u32 meta_blkaddr;
@ -75,6 +93,7 @@ struct erofs_sb_info {
/* inode slot unit size in bit shift */
unsigned char islotbits;
u32 sb_size; /* total superblock size */
u32 build_time_nsec;
u64 build_time;
@ -85,9 +104,10 @@ struct erofs_sb_info {
u8 uuid[16]; /* 128-bit uuid for volume */
u8 volume_name[16]; /* volume name */
u32 feature_compat;
u32 feature_incompat;
unsigned int mount_opt;
struct erofs_fs_context ctx; /* options */
};
#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)
@ -97,17 +117,17 @@ struct erofs_sb_info {
#define EROFS_MOUNT_XATTR_USER 0x00000010
#define EROFS_MOUNT_POSIX_ACL 0x00000020
#define clear_opt(sbi, option) ((sbi)->mount_opt &= ~EROFS_MOUNT_##option)
#define set_opt(sbi, option) ((sbi)->mount_opt |= EROFS_MOUNT_##option)
#define test_opt(sbi, option) ((sbi)->mount_opt & EROFS_MOUNT_##option)
#define clear_opt(ctx, option) ((ctx)->mount_opt &= ~EROFS_MOUNT_##option)
#define set_opt(ctx, option) ((ctx)->mount_opt |= EROFS_MOUNT_##option)
#define test_opt(ctx, option) ((ctx)->mount_opt & EROFS_MOUNT_##option)
#ifdef CONFIG_EROFS_FS_ZIP
enum {
EROFS_ZIP_CACHE_DISABLED,
EROFS_ZIP_CACHE_READAHEAD,
EROFS_ZIP_CACHE_READAROUND
};
#ifdef CONFIG_EROFS_FS_ZIP
#define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL)
/* basic unit of the workstation of a super_block */
@ -176,12 +196,6 @@ static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp)
return v;
}
#endif /* !CONFIG_SMP */
/* hard limit of pages per compressed cluster */
#define Z_EROFS_CLUSTER_MAX_PAGES (CONFIG_EROFS_FS_CLUSTER_PAGE_LIMIT)
#define EROFS_PCPUBUF_NR_PAGES Z_EROFS_CLUSTER_MAX_PAGES
#else
#define EROFS_PCPUBUF_NR_PAGES 0
#endif /* !CONFIG_EROFS_FS_ZIP */
/* we strictly follow PAGE_SIZE and no buffer head yet */
@ -210,6 +224,17 @@ static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid)
return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits);
}
#define EROFS_FEATURE_FUNCS(name, compat, feature) \
static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
{ \
return sbi->feature_##compat & EROFS_FEATURE_##feature; \
}
EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
/* atomic flag definitions */
#define EROFS_I_EA_INITED_BIT 0
#define EROFS_I_Z_INITED_BIT 1
@ -238,7 +263,6 @@ struct erofs_inode {
unsigned short z_advise;
unsigned char z_algorithmtype[2];
unsigned char z_logical_clusterbits;
unsigned char z_physical_clusterbits[2];
};
#endif /* CONFIG_EROFS_FS_ZIP */
};
@ -278,12 +302,10 @@ static inline unsigned int erofs_inode_datalayout(unsigned int value)
extern const struct super_operations erofs_sops;
extern const struct address_space_operations erofs_raw_access_aops;
#ifdef CONFIG_EROFS_FS_ZIP
extern const struct address_space_operations z_erofs_vle_normalaccess_aops;
#endif
extern const struct address_space_operations z_erofs_aops;
/*
* Logical to physical block mapping, used by erofs_map_blocks()
* Logical to physical block mapping
*
* Different with other file systems, it is used for 2 access modes:
*
@ -330,7 +352,7 @@ struct erofs_map_blocks {
struct page *mpage;
};
/* Flags used by erofs_map_blocks() */
/* Flags used by erofs_map_blocks_flatmode() */
#define EROFS_GET_BLOCKS_RAW 0x0001
/* zmap.c */
@ -352,8 +374,6 @@ static inline int z_erofs_map_blocks_iter(struct inode *inode,
/* data.c */
struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr);
int erofs_map_blocks(struct inode *, struct erofs_map_blocks *, int);
/* inode.c */
static inline unsigned long erofs_inode_hash(erofs_nid_t nid)
{
@ -381,30 +401,37 @@ int erofs_namei(struct inode *dir, struct qstr *name,
/* dir.c */
extern const struct file_operations erofs_dir_fops;
/* utils.c / zdata.c */
struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp, bool nofail);
#if (EROFS_PCPUBUF_NR_PAGES > 0)
void *erofs_get_pcpubuf(unsigned int pagenr);
#define erofs_put_pcpubuf(buf) do { \
(void)&(buf); \
preempt_enable(); \
} while (0)
#else
static inline void *erofs_get_pcpubuf(unsigned int pagenr)
static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count)
{
return ERR_PTR(-EOPNOTSUPP);
int retried = 0;
while (1) {
void *p = vm_map_ram(pages, count, -1, PAGE_KERNEL);
/* retry two more times (totally 3 times) */
if (p || ++retried >= 3)
return p;
vm_unmap_aliases();
}
return NULL;
}
#define erofs_put_pcpubuf(buf) do {} while (0)
#endif
/* pcpubuf.c */
void *erofs_get_pcpubuf(unsigned int requiredpages);
void erofs_put_pcpubuf(void *ptr);
int erofs_pcpubuf_growsize(unsigned int nrpages);
void erofs_pcpubuf_init(void);
void erofs_pcpubuf_exit(void);
/* utils.c / zdata.c */
struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp);
#ifdef CONFIG_EROFS_FS_ZIP
int erofs_workgroup_put(struct erofs_workgroup *grp);
struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb,
pgoff_t index, bool *tag);
int erofs_register_workgroup(struct super_block *sb,
struct erofs_workgroup *grp, bool tag);
pgoff_t index);
struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb,
struct erofs_workgroup *grp);
void erofs_workgroup_free_rcu(struct erofs_workgroup *grp);
void erofs_shrinker_register(struct super_block *sb);
void erofs_shrinker_unregister(struct super_block *sb);
@ -416,6 +443,9 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
struct erofs_workgroup *egrp);
int erofs_try_to_free_cached_page(struct address_space *mapping,
struct page *page);
int z_erofs_load_lz4_config(struct super_block *sb,
struct erofs_super_block *dsb,
struct z_erofs_lz4_cfgs *lz4, int len);
#else
static inline void erofs_shrinker_register(struct super_block *sb) {}
static inline void erofs_shrinker_unregister(struct super_block *sb) {}
@ -423,6 +453,16 @@ static inline int erofs_init_shrinker(void) { return 0; }
static inline void erofs_exit_shrinker(void) {}
static inline int z_erofs_init_zip_subsystem(void) { return 0; }
static inline void z_erofs_exit_zip_subsystem(void) {}
static inline int z_erofs_load_lz4_config(struct super_block *sb,
struct erofs_super_block *dsb,
struct z_erofs_lz4_cfgs *lz4, int len)
{
if (lz4 || dsb->u1.lz4_max_distance) {
erofs_err(sb, "lz4 algorithm isn't enabled");
return -EINVAL;
}
return 0;
}
#endif /* !CONFIG_EROFS_FS_ZIP */
#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */

4
fs/erofs/namei.c
View file

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "xattr.h"
@ -244,9 +244,7 @@ static struct dentry *erofs_lookup(struct inode *dir,
const struct inode_operations erofs_dir_iops = {
.lookup = erofs_lookup,
.getattr = erofs_getattr,
#ifdef CONFIG_EROFS_FS_XATTR
.listxattr = erofs_listxattr,
#endif
.get_acl = erofs_get_acl,
};

148
fs/erofs/pcpubuf.c Normal file
View file

@ -0,0 +1,148 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) Gao Xiang <xiang@kernel.org>
*
* For low-latency decompression algorithms (e.g. lz4), reserve consecutive
* per-CPU virtual memory (in pages) in advance to store such inplace I/O
* data if inplace decompression is failed (due to unmet inplace margin for
* example).
*/
#include "internal.h"
struct erofs_pcpubuf {
raw_spinlock_t lock;
void *ptr;
struct page **pages;
unsigned int nrpages;
};
static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
void *erofs_get_pcpubuf(unsigned int requiredpages)
__acquires(pcb->lock)
{
struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
raw_spin_lock(&pcb->lock);
/* check if the per-CPU buffer is too small */
if (requiredpages > pcb->nrpages) {
raw_spin_unlock(&pcb->lock);
put_cpu_var(erofs_pcb);
/* (for sparse checker) pretend pcb->lock is still taken */
__acquire(pcb->lock);
return NULL;
}
return pcb->ptr;
}
void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
{
struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
DBG_BUGON(pcb->ptr != ptr);
raw_spin_unlock(&pcb->lock);
put_cpu_var(erofs_pcb);
}
/* the next step: support per-CPU page buffers hotplug */
int erofs_pcpubuf_growsize(unsigned int nrpages)
{
static DEFINE_MUTEX(pcb_resize_mutex);
static unsigned int pcb_nrpages;
LIST_HEAD(pagepool);
int delta, cpu, ret, i;
mutex_lock(&pcb_resize_mutex);
delta = nrpages - pcb_nrpages;
ret = 0;
/* avoid shrinking pcpubuf, since no idea how many fses rely on */
if (delta <= 0)
goto out;
for_each_possible_cpu(cpu) {
struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
struct page **pages, **oldpages;
void *ptr, *old_ptr;
pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
if (!pages) {
ret = -ENOMEM;
break;
}
for (i = 0; i < nrpages; ++i) {
pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
if (!pages[i]) {
ret = -ENOMEM;
oldpages = pages;
goto free_pagearray;
}
}
ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
if (!ptr) {
ret = -ENOMEM;
oldpages = pages;
goto free_pagearray;
}
raw_spin_lock(&pcb->lock);
old_ptr = pcb->ptr;
pcb->ptr = ptr;
oldpages = pcb->pages;
pcb->pages = pages;
i = pcb->nrpages;
pcb->nrpages = nrpages;
raw_spin_unlock(&pcb->lock);
if (!oldpages) {
DBG_BUGON(old_ptr);
continue;
}
if (old_ptr)
vunmap(old_ptr);
free_pagearray:
while (i)
list_add(&oldpages[--i]->lru, &pagepool);
kfree(oldpages);
if (ret)
break;
}
pcb_nrpages = nrpages;
put_pages_list(&pagepool);
out:
mutex_unlock(&pcb_resize_mutex);
return ret;
}
void erofs_pcpubuf_init(void)
{
int cpu;
for_each_possible_cpu(cpu) {
struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
raw_spin_lock_init(&pcb->lock);
}
}
void erofs_pcpubuf_exit(void)
{
int cpu, i;
for_each_possible_cpu(cpu) {
struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
if (pcb->ptr) {
vunmap(pcb->ptr);
pcb->ptr = NULL;
}
if (!pcb->pages)
continue;
for (i = 0; i < pcb->nrpages; ++i)
if (pcb->pages[i])
put_page(pcb->pages[i]);
kfree(pcb->pages);
pcb->pages = NULL;
}
}

455
fs/erofs/super.c
View file

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include <linux/module.h>
@ -9,6 +9,9 @@
#include <linux/statfs.h>
#include <linux/parser.h>
#include <linux/seq_file.h>
#include <linux/crc32c.h>
#include <linux/fs_context.h>
#include <linux/fs_parser.h>
#include "xattr.h"
#define CREATE_TRACE_POINTS
@ -46,6 +49,30 @@ void _erofs_info(struct super_block *sb, const char *function,
va_end(args);
}
static int erofs_superblock_csum_verify(struct super_block *sb, void *sbdata)
{
struct erofs_super_block *dsb;
u32 expected_crc, crc;
dsb = kmemdup(sbdata + EROFS_SUPER_OFFSET,
EROFS_BLKSIZ - EROFS_SUPER_OFFSET, GFP_KERNEL);
if (!dsb)
return -ENOMEM;
expected_crc = le32_to_cpu(dsb->checksum);
dsb->checksum = 0;
/* to allow for x86 boot sectors and other oddities. */
crc = crc32c(~0, dsb, EROFS_BLKSIZ - EROFS_SUPER_OFFSET);
kfree(dsb);
if (crc != expected_crc) {
erofs_err(sb, "invalid checksum 0x%08x, 0x%08x expected",
crc, expected_crc);
return -EBADMSG;
}
return 0;
}
static void erofs_inode_init_once(void *ptr)
{
struct erofs_inode *vi = ptr;
@ -95,6 +122,136 @@ static bool check_layout_compatibility(struct super_block *sb,
return true;
}
#ifdef CONFIG_EROFS_FS_ZIP
/* read variable-sized metadata, offset will be aligned by 4-byte */
static void *erofs_read_metadata(struct super_block *sb, struct page **pagep,
erofs_off_t *offset, int *lengthp)
{
struct page *page = *pagep;
u8 *buffer, *ptr;
int len, i, cnt;
erofs_blk_t blk;
*offset = round_up(*offset, 4);
blk = erofs_blknr(*offset);
if (!page || page->index != blk) {
if (page) {
unlock_page(page);
put_page(page);
}
page = erofs_get_meta_page(sb, blk);
if (IS_ERR(page))
goto err_nullpage;
}
ptr = kmap(page);
len = le16_to_cpu(*(__le16 *)&ptr[erofs_blkoff(*offset)]);
if (!len)
len = U16_MAX + 1;
buffer = kmalloc(len, GFP_KERNEL);
if (!buffer) {
buffer = ERR_PTR(-ENOMEM);
goto out;
}
*offset += sizeof(__le16);
*lengthp = len;
for (i = 0; i < len; i += cnt) {
cnt = min(EROFS_BLKSIZ - (int)erofs_blkoff(*offset), len - i);
blk = erofs_blknr(*offset);
if (!page || page->index != blk) {
if (page) {
kunmap(page);
unlock_page(page);
put_page(page);
}
page = erofs_get_meta_page(sb, blk);
if (IS_ERR(page)) {
kfree(buffer);
goto err_nullpage;
}
ptr = kmap(page);
}
memcpy(buffer + i, ptr + erofs_blkoff(*offset), cnt);
*offset += cnt;
}
out:
kunmap(page);
*pagep = page;
return buffer;
err_nullpage:
*pagep = NULL;
return page;
}
static int erofs_load_compr_cfgs(struct super_block *sb,
struct erofs_super_block *dsb)
{
struct erofs_sb_info *sbi;
struct page *page;
unsigned int algs, alg;
erofs_off_t offset;
int size, ret;
sbi = EROFS_SB(sb);
sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs);
if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) {
erofs_err(sb, "try to load compressed fs with unsupported algorithms %x",
sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS);
return -EINVAL;
}
offset = EROFS_SUPER_OFFSET + sbi->sb_size;
page = NULL;
alg = 0;
ret = 0;
for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) {
void *data;
if (!(algs & 1))
continue;
data = erofs_read_metadata(sb, &page, &offset, &size);
if (IS_ERR(data)) {
ret = PTR_ERR(data);
goto err;
}
switch (alg) {
case Z_EROFS_COMPRESSION_LZ4:
ret = z_erofs_load_lz4_config(sb, dsb, data, size);
break;
default:
DBG_BUGON(1);
ret = -EFAULT;
}
kfree(data);
if (ret)
goto err;
}
err:
if (page) {
unlock_page(page);
put_page(page);
}
return ret;
}
#else
static int erofs_load_compr_cfgs(struct super_block *sb,
struct erofs_super_block *dsb)
{
if (dsb->u1.available_compr_algs) {
erofs_err(sb, "try to load compressed fs when compression is disabled");
return -EINVAL;
}
return 0;
}
#endif
static int erofs_read_superblock(struct super_block *sb)
{
struct erofs_sb_info *sbi;
@ -112,7 +269,7 @@ static int erofs_read_superblock(struct super_block *sb)
sbi = EROFS_SB(sb);
data = kmap_atomic(page);
data = kmap(page);
dsb = (struct erofs_super_block *)(data + EROFS_SUPER_OFFSET);
ret = -EINVAL;
@ -121,6 +278,14 @@ static int erofs_read_superblock(struct super_block *sb)
goto out;
}
sbi->feature_compat = le32_to_cpu(dsb->feature_compat);
if (erofs_sb_has_sb_chksum(sbi)) {
ret = erofs_superblock_csum_verify(sb, data);
if (ret)
goto out;
}
ret = -EINVAL;
blkszbits = dsb->blkszbits;
/* 9(512 bytes) + LOG_SECTORS_PER_BLOCK == LOG_BLOCK_SIZE */
if (blkszbits != LOG_BLOCK_SIZE) {
@ -132,6 +297,12 @@ static int erofs_read_superblock(struct super_block *sb)
if (!check_layout_compatibility(sb, dsb))
goto out;
sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE;
if (sbi->sb_size > EROFS_BLKSIZ) {
erofs_err(sb, "invalid sb_extslots %u (more than a fs block)",
sbi->sb_size);
goto out;
}
sbi->blocks = le32_to_cpu(dsb->blocks);
sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr);
#ifdef CONFIG_EROFS_FS_XATTR
@ -153,139 +324,101 @@ static int erofs_read_superblock(struct super_block *sb)
ret = -EFSCORRUPTED;
goto out;
}
ret = 0;
/* parse on-disk compression configurations */
if (erofs_sb_has_compr_cfgs(sbi))
ret = erofs_load_compr_cfgs(sb, dsb);
else
ret = z_erofs_load_lz4_config(sb, dsb, NULL, 0);
out:
kunmap_atomic(data);
kunmap(page);
put_page(page);
return ret;
}
#ifdef CONFIG_EROFS_FS_ZIP
static int erofs_build_cache_strategy(struct super_block *sb,
substring_t *args)
{
struct erofs_sb_info *sbi = EROFS_SB(sb);
const char *cs = match_strdup(args);
int err = 0;
if (!cs) {
erofs_err(sb, "Not enough memory to store cache strategy");
return -ENOMEM;
}
if (!strcmp(cs, "disabled")) {
sbi->cache_strategy = EROFS_ZIP_CACHE_DISABLED;
} else if (!strcmp(cs, "readahead")) {
sbi->cache_strategy = EROFS_ZIP_CACHE_READAHEAD;
} else if (!strcmp(cs, "readaround")) {
sbi->cache_strategy = EROFS_ZIP_CACHE_READAROUND;
} else {
erofs_err(sb, "Unrecognized cache strategy \"%s\"", cs);
err = -EINVAL;
}
kfree(cs);
return err;
}
#else
static int erofs_build_cache_strategy(struct super_block *sb,
substring_t *args)
{
erofs_info(sb, "EROFS compression is disabled, so cache strategy is ignored");
return 0;
}
#endif
/* set up default EROFS parameters */
static void erofs_default_options(struct erofs_sb_info *sbi)
static void erofs_default_options(struct erofs_fs_context *ctx)
{
#ifdef CONFIG_EROFS_FS_ZIP
sbi->cache_strategy = EROFS_ZIP_CACHE_READAROUND;
sbi->max_sync_decompress_pages = 3;
ctx->cache_strategy = EROFS_ZIP_CACHE_READAROUND;
ctx->max_sync_decompress_pages = 3;
ctx->readahead_sync_decompress = false;
#endif
#ifdef CONFIG_EROFS_FS_XATTR
set_opt(sbi, XATTR_USER);
set_opt(ctx, XATTR_USER);
#endif
#ifdef CONFIG_EROFS_FS_POSIX_ACL
set_opt(sbi, POSIX_ACL);
set_opt(ctx, POSIX_ACL);
#endif
}
enum {
Opt_user_xattr,
Opt_nouser_xattr,
Opt_acl,
Opt_noacl,
Opt_cache_strategy,
Opt_err
};
static match_table_t erofs_tokens = {
{Opt_user_xattr, "user_xattr"},
{Opt_nouser_xattr, "nouser_xattr"},
{Opt_acl, "acl"},
{Opt_noacl, "noacl"},
{Opt_cache_strategy, "cache_strategy=%s"},
{Opt_err, NULL}
static const struct fs_parameter_enum erofs_param_cache_strategy[] ={
{Opt_cache_strategy, "disabled", EROFS_ZIP_CACHE_DISABLED},
{Opt_cache_strategy, "readahead", EROFS_ZIP_CACHE_READAHEAD},
{Opt_cache_strategy, "readaround", EROFS_ZIP_CACHE_READAROUND},
{}
};
static int erofs_parse_options(struct super_block *sb, char *options)
static const struct fs_parameter_spec erofs_fs_param_specs[] = {
fsparam_flag_no("user_xattr", Opt_user_xattr),
fsparam_flag_no("acl", Opt_acl),
fsparam_enum("cache_strategy", Opt_cache_strategy),
{}
};
const struct fs_parameter_description erofs_fs_parameters = {
.name = "erofs",
.specs = erofs_fs_param_specs,
.enums = erofs_param_cache_strategy,
};
static int erofs_fc_parse_param(struct fs_context *fc,
struct fs_parameter *param)
{
substring_t args[MAX_OPT_ARGS];
char *p;
int err;
struct erofs_fs_context *ctx __maybe_unused = fc->fs_private;
struct fs_parse_result result;
int opt;
if (!options)
return 0;
opt = fs_parse(fc, &erofs_fs_parameters, param, &result);
if (opt < 0)
return opt;
while ((p = strsep(&options, ","))) {
int token;
if (!*p)
continue;
args[0].to = args[0].from = NULL;
token = match_token(p, erofs_tokens, args);
switch (token) {
switch (opt) {
case Opt_user_xattr:
#ifdef CONFIG_EROFS_FS_XATTR
case Opt_user_xattr:
set_opt(EROFS_SB(sb), XATTR_USER);
break;
case Opt_nouser_xattr:
clear_opt(EROFS_SB(sb), XATTR_USER);
break;
if (result.boolean)
set_opt(ctx, XATTR_USER);
else
clear_opt(ctx, XATTR_USER);
#else
case Opt_user_xattr:
erofs_info(sb, "user_xattr options not supported");
break;
case Opt_nouser_xattr:
erofs_info(sb, "nouser_xattr options not supported");
break;
errorfc(fc, "{,no}user_xattr options not supported");
#endif
break;
case Opt_acl:
#ifdef CONFIG_EROFS_FS_POSIX_ACL
case Opt_acl:
set_opt(EROFS_SB(sb), POSIX_ACL);
break;
case Opt_noacl:
clear_opt(EROFS_SB(sb), POSIX_ACL);
break;
if (result.boolean)
set_opt(ctx, POSIX_ACL);
else
clear_opt(ctx, POSIX_ACL);
#else
case Opt_acl:
erofs_info(sb, "acl options not supported");
break;
case Opt_noacl:
erofs_info(sb, "noacl options not supported");
break;
errorfc(fc, "{,no}acl options not supported");
#endif
case Opt_cache_strategy:
err = erofs_build_cache_strategy(sb, args);
if (err)
return err;
break;
default:
erofs_err(sb, "Unrecognized mount option \"%s\" or missing value", p);
return -EINVAL;
}
break;
case Opt_cache_strategy:
#ifdef CONFIG_EROFS_FS_ZIP
ctx->cache_strategy = result.uint_32;
#else
errorfc(fc, "compression not supported, cache_strategy ignored");
#endif
break;
default:
return -ENOPARAM;
}
return 0;
}
@ -349,10 +482,11 @@ static int erofs_init_managed_cache(struct super_block *sb)
static int erofs_init_managed_cache(struct super_block *sb) { return 0; }
#endif
static int erofs_fill_super(struct super_block *sb, void *data, int silent)
static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
{
struct inode *inode;
struct erofs_sb_info *sbi;
struct erofs_fs_context *ctx = fc->fs_private;
int err;
sb->s_magic = EROFS_SUPER_MAGIC;
@ -376,24 +510,17 @@ static int erofs_fill_super(struct super_block *sb, void *data, int silent)
sb->s_time_gran = 1;
sb->s_op = &erofs_sops;
#ifdef CONFIG_EROFS_FS_XATTR
sb->s_xattr = erofs_xattr_handlers;
#endif
/* set erofs default mount options */
erofs_default_options(sbi);
err = erofs_parse_options(sb, data);
if (err)
return err;
if (test_opt(sbi, POSIX_ACL))
if (test_opt(ctx, POSIX_ACL))
sb->s_flags |= SB_POSIXACL;
else
sb->s_flags &= ~SB_POSIXACL;
sbi->ctx = *ctx;
#ifdef CONFIG_EROFS_FS_ZIP
INIT_RADIX_TREE(&sbi->workstn_tree, GFP_ATOMIC);
xa_init(&sbi->managed_pslots);
#endif
/* get the root inode */
@ -418,15 +545,58 @@ static int erofs_fill_super(struct super_block *sb, void *data, int silent)
if (err)
return err;
erofs_info(sb, "mounted with opts: %s, root inode @ nid %llu.",
(char *)data, ROOT_NID(sbi));
erofs_info(sb, "mounted with root inode @ nid %llu.", ROOT_NID(sbi));
return 0;
}
static struct dentry *erofs_mount(struct file_system_type *fs_type, int flags,
const char *dev_name, void *data)
static int erofs_fc_get_tree(struct fs_context *fc)
{
return mount_bdev(fs_type, flags, dev_name, data, erofs_fill_super);
return get_tree_bdev(fc, erofs_fc_fill_super);
}
static int erofs_fc_reconfigure(struct fs_context *fc)
{
struct super_block *sb = fc->root->d_sb;
struct erofs_sb_info *sbi = EROFS_SB(sb);
struct erofs_fs_context *ctx = fc->fs_private;
DBG_BUGON(!sb_rdonly(sb));
if (test_opt(ctx, POSIX_ACL))
fc->sb_flags |= SB_POSIXACL;
else
fc->sb_flags &= ~SB_POSIXACL;
sbi->ctx = *ctx;
fc->sb_flags |= SB_RDONLY;
return 0;
}
static void erofs_fc_free(struct fs_context *fc)
{
kfree(fc->fs_private);
}
static const struct fs_context_operations erofs_context_ops = {
.parse_param = erofs_fc_parse_param,
.get_tree = erofs_fc_get_tree,
.reconfigure = erofs_fc_reconfigure,
.free = erofs_fc_free,
};
static int erofs_init_fs_context(struct fs_context *fc)
{
fc->fs_private = kzalloc(sizeof(struct erofs_fs_context), GFP_KERNEL);
if (!fc->fs_private)
return -ENOMEM;
/* set default mount options */
erofs_default_options(fc->fs_private);
fc->ops = &erofs_context_ops;
return 0;
}
/*
@ -465,7 +635,7 @@ static void erofs_put_super(struct super_block *sb)
static struct file_system_type erofs_fs_type = {
.owner = THIS_MODULE,
.name = "erofs",
.mount = erofs_mount,
.init_fs_context = erofs_init_fs_context,
.kill_sb = erofs_kill_sb,
.fs_flags = FS_REQUIRES_DEV,
};
@ -490,6 +660,7 @@ static int __init erofs_module_init(void)
if (err)
goto shrinker_err;
erofs_pcpubuf_init();
err = z_erofs_init_zip_subsystem();
if (err)
goto zip_err;
@ -519,6 +690,7 @@ static void __exit erofs_module_exit(void)
/* Ensure all RCU free inodes are safe before cache is destroyed. */
rcu_barrier();
kmem_cache_destroy(erofs_inode_cachep);
erofs_pcpubuf_exit();
}
/* get filesystem statistics */
@ -538,72 +710,44 @@ static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf)
buf->f_namelen = EROFS_NAME_LEN;
buf->f_fsid.val[0] = (u32)id;
buf->f_fsid.val[1] = (u32)(id >> 32);
buf->f_fsid = u64_to_fsid(id);
return 0;
}
static int erofs_show_options(struct seq_file *seq, struct dentry *root)
{
struct erofs_sb_info *sbi __maybe_unused = EROFS_SB(root->d_sb);
struct erofs_fs_context *ctx __maybe_unused = &sbi->ctx;
#ifdef CONFIG_EROFS_FS_XATTR
if (test_opt(sbi, XATTR_USER))
if (test_opt(ctx, XATTR_USER))
seq_puts(seq, ",user_xattr");
else
seq_puts(seq, ",nouser_xattr");
#endif
#ifdef CONFIG_EROFS_FS_POSIX_ACL
if (test_opt(sbi, POSIX_ACL))
if (test_opt(ctx, POSIX_ACL))
seq_puts(seq, ",acl");
else
seq_puts(seq, ",noacl");
#endif
#ifdef CONFIG_EROFS_FS_ZIP
if (sbi->cache_strategy == EROFS_ZIP_CACHE_DISABLED) {
if (ctx->cache_strategy == EROFS_ZIP_CACHE_DISABLED)
seq_puts(seq, ",cache_strategy=disabled");
} else if (sbi->cache_strategy == EROFS_ZIP_CACHE_READAHEAD) {
else if (ctx->cache_strategy == EROFS_ZIP_CACHE_READAHEAD)
seq_puts(seq, ",cache_strategy=readahead");
} else if (sbi->cache_strategy == EROFS_ZIP_CACHE_READAROUND) {
else if (ctx->cache_strategy == EROFS_ZIP_CACHE_READAROUND)
seq_puts(seq, ",cache_strategy=readaround");
} else {
seq_puts(seq, ",cache_strategy=(unknown)");
DBG_BUGON(1);
}
#endif
return 0;
}
static int erofs_remount(struct super_block *sb, int *flags, char *data)
{
struct erofs_sb_info *sbi = EROFS_SB(sb);
unsigned int org_mnt_opt = sbi->mount_opt;
int err;
DBG_BUGON(!sb_rdonly(sb));
err = erofs_parse_options(sb, data);
if (err)
goto out;
if (test_opt(sbi, POSIX_ACL))
sb->s_flags |= SB_POSIXACL;
else
sb->s_flags &= ~SB_POSIXACL;
*flags |= SB_RDONLY;
return 0;
out:
sbi->mount_opt = org_mnt_opt;
return err;
}
const struct super_operations erofs_sops = {
.put_super = erofs_put_super,
.alloc_inode = erofs_alloc_inode,
.free_inode = erofs_free_inode,
.statfs = erofs_statfs,
.show_options = erofs_show_options,
.remount_fs = erofs_remount,
};
module_init(erofs_module_init);
@ -612,4 +756,5 @@ module_exit(erofs_module_exit);
MODULE_DESCRIPTION("Enhanced ROM File System");
MODULE_AUTHOR("Gao Xiang, Chao Yu, Miao Xie, CONSUMER BG, HUAWEI Inc.");
MODULE_LICENSE("GPL");
MODULE_IMPORT_NS(ANDROID_GKI_VFS_EXPORT_ONLY);

148
fs/erofs/utils.c
View file

@ -1,13 +1,13 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "internal.h"
#include <linux/pagevec.h>
struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp, bool nofail)
struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp)
{
struct page *page;
@ -16,30 +16,15 @@ struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp, bool nofail)
DBG_BUGON(page_ref_count(page) != 1);
list_del(&page->lru);
} else {
page = alloc_pages(gfp | (nofail ? __GFP_NOFAIL : 0), 0);
page = alloc_page(gfp);
}
return page;
}
#if (EROFS_PCPUBUF_NR_PAGES > 0)
static struct {
u8 data[PAGE_SIZE * EROFS_PCPUBUF_NR_PAGES];
} ____cacheline_aligned_in_smp erofs_pcpubuf[NR_CPUS];
void *erofs_get_pcpubuf(unsigned int pagenr)
{
preempt_disable();
return &erofs_pcpubuf[smp_processor_id()].data[pagenr * PAGE_SIZE];
}
#endif
#ifdef CONFIG_EROFS_FS_ZIP
/* global shrink count (for all mounted EROFS instances) */
static atomic_long_t erofs_global_shrink_cnt;
#define __erofs_workgroup_get(grp) atomic_inc(&(grp)->refcount)
#define __erofs_workgroup_put(grp) atomic_dec(&(grp)->refcount)
static int erofs_workgroup_get(struct erofs_workgroup *grp)
{
int o;
@ -59,18 +44,15 @@ repeat:
}
struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb,
pgoff_t index, bool *tag)
pgoff_t index)
{
struct erofs_sb_info *sbi = EROFS_SB(sb);
struct erofs_workgroup *grp;
repeat:
rcu_read_lock();
grp = radix_tree_lookup(&sbi->workstn_tree, index);
grp = xa_load(&sbi->managed_pslots, index);
if (grp) {
*tag = xa_pointer_tag(grp);
grp = xa_untag_pointer(grp);
if (erofs_workgroup_get(grp)) {
/* prefer to relax rcu read side */
rcu_read_unlock();
@ -83,46 +65,37 @@ repeat:
return grp;
}
int erofs_register_workgroup(struct super_block *sb,
struct erofs_workgroup *grp,
bool tag)
struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb,
struct erofs_workgroup *grp)
{
struct erofs_sb_info *sbi;
int err;
/* grp shouldn't be broken or used before */
if (atomic_read(&grp->refcount) != 1) {
DBG_BUGON(1);
return -EINVAL;
}
err = radix_tree_preload(GFP_NOFS);
if (err)
return err;
sbi = EROFS_SB(sb);
xa_lock(&sbi->workstn_tree);
grp = xa_tag_pointer(grp, tag);
struct erofs_sb_info *const sbi = EROFS_SB(sb);
struct erofs_workgroup *pre;
/*
* Bump up reference count before making this workgroup
* visible to other users in order to avoid potential UAF
* without serialized by workstn_lock.
* Bump up a reference count before making this visible
* to others for the XArray in order to avoid potential
* UAF without serialized by xa_lock.
*/
__erofs_workgroup_get(grp);
atomic_inc(&grp->refcount);
err = radix_tree_insert(&sbi->workstn_tree, grp->index, grp);
if (err)
/*
* it's safe to decrease since the workgroup isn't visible
* and refcount >= 2 (cannot be freezed).
*/
__erofs_workgroup_put(grp);
xa_unlock(&sbi->workstn_tree);
radix_tree_preload_end();
return err;
repeat:
xa_lock(&sbi->managed_pslots);
pre = __xa_cmpxchg(&sbi->managed_pslots, grp->index,
NULL, grp, GFP_NOFS);
if (pre) {
if (xa_is_err(pre)) {
pre = ERR_PTR(xa_err(pre));
} else if (erofs_workgroup_get(pre)) {
/* try to legitimize the current in-tree one */
xa_unlock(&sbi->managed_pslots);
cond_resched();
goto repeat;
}
atomic_dec(&grp->refcount);
grp = pre;
}
xa_unlock(&sbi->managed_pslots);
return grp;
}
static void __erofs_workgroup_free(struct erofs_workgroup *grp)
@ -142,15 +115,8 @@ int erofs_workgroup_put(struct erofs_workgroup *grp)
return count;
}
static void erofs_workgroup_unfreeze_final(struct erofs_workgroup *grp)
{
erofs_workgroup_unfreeze(grp, 0);
__erofs_workgroup_free(grp);
}
static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi,
struct erofs_workgroup *grp,
bool cleanup)
struct erofs_workgroup *grp)
{
/*
* If managed cache is on, refcount of workgroups
@ -162,7 +128,7 @@ static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi,
/*
* Note that all cached pages should be unattached
* before deleted from the radix tree. Otherwise some
* before deleted from the XArray. Otherwise some
* cached pages could be still attached to the orphan
* old workgroup when the new one is available in the tree.
*/
@ -176,49 +142,34 @@ static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi,
* however in order to avoid some race conditions, add a
* DBG_BUGON to observe this in advance.
*/
DBG_BUGON(xa_untag_pointer(radix_tree_delete(&sbi->workstn_tree,
grp->index)) != grp);
DBG_BUGON(__xa_erase(&sbi->managed_pslots, grp->index) != grp);
/*
* If managed cache is on, last refcount should indicate
* the related workstation.
*/
erofs_workgroup_unfreeze_final(grp);
/* last refcount should be connected with its managed pslot. */
erofs_workgroup_unfreeze(grp, 0);
__erofs_workgroup_free(grp);
return true;
}
static unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi,
unsigned long nr_shrink,
bool cleanup)
unsigned long nr_shrink)
{
pgoff_t first_index = 0;
void *batch[PAGEVEC_SIZE];
struct erofs_workgroup *grp;
unsigned int freed = 0;
unsigned long index;
int i, found;
repeat:
xa_lock(&sbi->workstn_tree);
found = radix_tree_gang_lookup(&sbi->workstn_tree,
batch, first_index, PAGEVEC_SIZE);
for (i = 0; i < found; ++i) {
struct erofs_workgroup *grp = xa_untag_pointer(batch[i]);
first_index = grp->index + 1;
xa_lock(&sbi->managed_pslots);
xa_for_each(&sbi->managed_pslots, index, grp) {
/* try to shrink each valid workgroup */
if (!erofs_try_to_release_workgroup(sbi, grp, cleanup))
if (!erofs_try_to_release_workgroup(sbi, grp))
continue;
xa_unlock(&sbi->managed_pslots);
++freed;
if (!--nr_shrink)
break;
return freed;
xa_lock(&sbi->managed_pslots);
}
xa_unlock(&sbi->workstn_tree);
if (i && nr_shrink)
goto repeat;
xa_unlock(&sbi->managed_pslots);
return freed;
}
@ -245,7 +196,8 @@ void erofs_shrinker_unregister(struct super_block *sb)
struct erofs_sb_info *const sbi = EROFS_SB(sb);
mutex_lock(&sbi->umount_mutex);
erofs_shrink_workstation(sbi, ~0UL, true);
/* clean up all remaining workgroups in memory */
erofs_shrink_workstation(sbi, ~0UL);
spin_lock(&erofs_sb_list_lock);
list_del(&sbi->list);
@ -294,7 +246,7 @@ static unsigned long erofs_shrink_scan(struct shrinker *shrink,
spin_unlock(&erofs_sb_list_lock);
sbi->shrinker_run_no = run_no;
freed += erofs_shrink_workstation(sbi, nr - freed, false);
freed += erofs_shrink_workstation(sbi, nr - freed);
spin_lock(&erofs_sb_list_lock);
/* Get the next list element before we move this one */

6
fs/erofs/xattr.c
View file

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include <linux/security.h>
@ -430,7 +430,7 @@ static int shared_getxattr(struct inode *inode, struct getxattr_iter *it)
static bool erofs_xattr_user_list(struct dentry *dentry)
{
return test_opt(EROFS_SB(dentry->d_sb), XATTR_USER);
return test_opt(&EROFS_SB(dentry->d_sb)->ctx, XATTR_USER);
}
static bool erofs_xattr_trusted_list(struct dentry *dentry)
@ -478,7 +478,7 @@ static int erofs_xattr_generic_get(const struct xattr_handler *handler,
switch (handler->flags) {
case EROFS_XATTR_INDEX_USER:
if (!test_opt(sbi, XATTR_USER))
if (!test_opt(&sbi->ctx, XATTR_USER))
return -EOPNOTSUPP;
break;
case EROFS_XATTR_INDEX_TRUSTED:

26
fs/erofs/xattr.h
View file

@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_XATTR_H
@ -46,18 +46,19 @@ extern const struct xattr_handler erofs_xattr_security_handler;
static inline const struct xattr_handler *erofs_xattr_handler(unsigned int idx)
{
static const struct xattr_handler *xattr_handler_map[] = {
[EROFS_XATTR_INDEX_USER] = &erofs_xattr_user_handler,
static const struct xattr_handler *xattr_handler_map[] = {
[EROFS_XATTR_INDEX_USER] = &erofs_xattr_user_handler,
#ifdef CONFIG_EROFS_FS_POSIX_ACL
[EROFS_XATTR_INDEX_POSIX_ACL_ACCESS] = &posix_acl_access_xattr_handler,
[EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT] =
&posix_acl_default_xattr_handler,
[EROFS_XATTR_INDEX_POSIX_ACL_ACCESS] =
&posix_acl_access_xattr_handler,
[EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT] =
&posix_acl_default_xattr_handler,
#endif
[EROFS_XATTR_INDEX_TRUSTED] = &erofs_xattr_trusted_handler,
[EROFS_XATTR_INDEX_TRUSTED] = &erofs_xattr_trusted_handler,
#ifdef CONFIG_EROFS_FS_SECURITY
[EROFS_XATTR_INDEX_SECURITY] = &erofs_xattr_security_handler,
[EROFS_XATTR_INDEX_SECURITY] = &erofs_xattr_security_handler,
#endif
};
};
return idx && idx < ARRAY_SIZE(xattr_handler_map) ?
xattr_handler_map[idx] : NULL;
@ -75,11 +76,8 @@ static inline int erofs_getxattr(struct inode *inode, int index,
return -EOPNOTSUPP;
}
static inline ssize_t erofs_listxattr(struct dentry *dentry,
char *buffer, size_t buffer_size)
{
return -EOPNOTSUPP;
}
#define erofs_listxattr (NULL)
#define erofs_xattr_handlers (NULL)
#endif /* !CONFIG_EROFS_FS_XATTR */
#ifdef CONFIG_EROFS_FS_POSIX_ACL

744
fs/erofs/zdata.c
View file

File diff suppressed because it is too large Load diff

25
fs/erofs/zdata.h
View file

@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_FS_ZDATA_H
@ -10,6 +10,7 @@
#include "internal.h"
#include "zpvec.h"
#define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE)
#define Z_EROFS_NR_INLINE_PAGEVECS 3
/*
@ -59,16 +60,17 @@ struct z_erofs_pcluster {
/* A: point to next chained pcluster or TAILs */
z_erofs_next_pcluster_t next;
/* A: compressed pages (including multi-usage pages) */
struct page *compressed_pages[Z_EROFS_CLUSTER_MAX_PAGES];
/* A: lower limit of decompressed length and if full length or not */
unsigned int length;
/* I: physical cluster size in pages */
unsigned short pclusterpages;
/* I: compression algorithm format */
unsigned char algorithmformat;
/* I: bit shift of physical cluster size */
unsigned char clusterbits;
/* A: compressed pages (can be cached or inplaced pages) */
struct page *compressed_pages[];
};
#define z_erofs_primarycollection(pcluster) (&(pcluster)->primary_collection)
@ -82,9 +84,8 @@ struct z_erofs_pcluster {
#define Z_EROFS_PCLUSTER_NIL (NULL)
#define Z_EROFS_WORKGROUP_SIZE sizeof(struct z_erofs_pcluster)
struct z_erofs_unzip_io {
struct z_erofs_decompressqueue {
struct super_block *sb;
atomic_t pending_bios;
z_erofs_next_pcluster_t head;
@ -94,11 +95,6 @@ struct z_erofs_unzip_io {
} u;
};
struct z_erofs_unzip_io_sb {
struct z_erofs_unzip_io io;
struct super_block *sb;
};
#define MNGD_MAPPING(sbi) ((sbi)->managed_cache->i_mapping)
static inline bool erofs_page_is_managed(const struct erofs_sb_info *sbi,
struct page *page)
@ -177,6 +173,7 @@ static inline void z_erofs_onlinepage_endio(struct page *page)
v = atomic_dec_return(u.o);
if (!(v & Z_EROFS_ONLINEPAGE_COUNT_MASK)) {
set_page_private(page, 0);
ClearPagePrivate(page);
if (!PageError(page))
SetPageUptodate(page);

213
fs/erofs/zmap.c
View file

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2018-2019 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#include "internal.h"
@ -11,22 +11,21 @@
int z_erofs_fill_inode(struct inode *inode)
{
struct erofs_inode *const vi = EROFS_I(inode);
struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
if (!erofs_sb_has_big_pcluster(sbi) &&
vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
vi->z_advise = 0;
vi->z_algorithmtype[0] = 0;
vi->z_algorithmtype[1] = 0;
vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits;
vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits;
set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
}
inode->i_mapping->a_ops = &z_erofs_vle_normalaccess_aops;
inode->i_mapping->a_ops = &z_erofs_aops;
return 0;
}
static int fill_inode_lazy(struct inode *inode)
static int z_erofs_fill_inode_lazy(struct inode *inode)
{
struct erofs_inode *const vi = EROFS_I(inode);
struct super_block *const sb = inode->i_sb;
@ -52,7 +51,8 @@ static int fill_inode_lazy(struct inode *inode)
if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
goto out_unlock;
DBG_BUGON(vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
vi->xattr_isize, 8);
@ -77,18 +77,22 @@ static int fill_inode_lazy(struct inode *inode)
}
vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7);
vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits +
((h->h_clusterbits >> 3) & 3);
if (vi->z_physical_clusterbits[0] != LOG_BLOCK_SIZE) {
erofs_err(sb, "unsupported physical clusterbits %u for nid %llu, please upgrade kernel",
vi->z_physical_clusterbits[0], vi->nid);
err = -EOPNOTSUPP;
if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu",
vi->nid);
err = -EFSCORRUPTED;
goto unmap_done;
}
if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION &&
!(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^
!(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",
vi->nid);
err = -EFSCORRUPTED;
goto unmap_done;
}
vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits +
((h->h_clusterbits >> 5) & 7);
/* paired with smp_mb() at the beginning of the function */
smp_mb();
set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
@ -111,7 +115,7 @@ struct z_erofs_maprecorder {
u8 type;
u16 clusterofs;
u16 delta[2];
erofs_blk_t pblk;
erofs_blk_t pblk, compressedlcs;
};
static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
@ -146,8 +150,8 @@ static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
return 0;
}
static int vle_legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
unsigned long lcn)
static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
unsigned long lcn)
{
struct inode *const inode = m->inode;
struct erofs_inode *const vi = EROFS_I(inode);
@ -174,6 +178,15 @@ static int vle_legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
m->clusterofs = 1 << vi->z_logical_clusterbits;
m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) {
if (!(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) {
DBG_BUGON(1);
return -EFSCORRUPTED;
}
m->compressedlcs = m->delta[0] &
~Z_EROFS_VLE_DI_D0_CBLKCNT;
m->delta[0] = 1;
}
m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
break;
case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
@ -214,6 +227,7 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
unsigned int vcnt, base, lo, encodebits, nblk;
int i;
u8 *in, type;
bool big_pcluster;
if (1 << amortizedshift == 4 && lclusterbits <= 14)
vcnt = 2;
@ -222,6 +236,7 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
else
return -EOPNOTSUPP;
big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
base = round_down(eofs, vcnt << amortizedshift);
in = m->kaddr + base;
@ -233,7 +248,15 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
m->type = type;
if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
m->clusterofs = 1 << lclusterbits;
if (i + 1 != vcnt) {
if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
if (!big_pcluster) {
DBG_BUGON(1);
return -EFSCORRUPTED;
}
m->compressedlcs = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
m->delta[0] = 1;
return 0;
} else if (i + 1 != (int)vcnt) {
m->delta[0] = lo;
return 0;
}
@ -246,22 +269,48 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m,
in, encodebits * (i - 1), &type);
if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
lo = 0;
else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT)
lo = 1;
m->delta[0] = lo + 1;
return 0;
}
m->clusterofs = lo;
m->delta[0] = 0;
/* figout out blkaddr (pblk) for HEAD lclusters */
nblk = 1;
while (i > 0) {
--i;
lo = decode_compactedbits(lclusterbits, lomask,
in, encodebits * i, &type);
if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
i -= lo;
if (!big_pcluster) {
nblk = 1;
while (i > 0) {
--i;
lo = decode_compactedbits(lclusterbits, lomask,
in, encodebits * i, &type);
if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
i -= lo;
if (i >= 0)
if (i >= 0)
++nblk;
}
} else {
nblk = 0;
while (i > 0) {
--i;
lo = decode_compactedbits(lclusterbits, lomask,
in, encodebits * i, &type);
if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
--i;
nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
continue;
}
/* bigpcluster shouldn't have plain d0 == 1 */
if (lo <= 1) {
DBG_BUGON(1);
return -EFSCORRUPTED;
}
i -= lo - 2;
continue;
}
++nblk;
}
}
in += (vcnt << amortizedshift) - sizeof(__le32);
m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
@ -319,13 +368,13 @@ out:
return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos));
}
static int vle_load_cluster_from_disk(struct z_erofs_maprecorder *m,
unsigned int lcn)
static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
unsigned int lcn)
{
const unsigned int datamode = EROFS_I(m->inode)->datalayout;
if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
return vle_legacy_load_cluster_from_disk(m, lcn);
return legacy_load_cluster_from_disk(m, lcn);
if (datamode == EROFS_INODE_FLAT_COMPRESSION)
return compacted_load_cluster_from_disk(m, lcn);
@ -333,8 +382,8 @@ static int vle_load_cluster_from_disk(struct z_erofs_maprecorder *m,
return -EINVAL;
}
static int vle_extent_lookback(struct z_erofs_maprecorder *m,
unsigned int lookback_distance)
static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
unsigned int lookback_distance)
{
struct erofs_inode *const vi = EROFS_I(m->inode);
struct erofs_map_blocks *const map = m->map;
@ -351,7 +400,7 @@ static int vle_extent_lookback(struct z_erofs_maprecorder *m,
/* load extent head logical cluster if needed */
lcn -= lookback_distance;
err = vle_load_cluster_from_disk(m, lcn);
err = z_erofs_load_cluster_from_disk(m, lcn);
if (err)
return err;
@ -364,10 +413,10 @@ static int vle_extent_lookback(struct z_erofs_maprecorder *m,
DBG_BUGON(1);
return -EFSCORRUPTED;
}
return vle_extent_lookback(m, m->delta[0]);
return z_erofs_extent_lookback(m, m->delta[0]);
case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
map->m_flags &= ~EROFS_MAP_ZIPPED;
/* fallthrough */
fallthrough;
case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
map->m_la = (lcn << lclusterbits) | m->clusterofs;
break;
@ -381,6 +430,75 @@ static int vle_extent_lookback(struct z_erofs_maprecorder *m,
return 0;
}
static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
unsigned int initial_lcn)
{
struct erofs_inode *const vi = EROFS_I(m->inode);
struct erofs_map_blocks *const map = m->map;
const unsigned int lclusterbits = vi->z_logical_clusterbits;
unsigned long lcn;
int err;
DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD);
if (!(map->m_flags & EROFS_MAP_ZIPPED) ||
!(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) {
map->m_plen = 1 << lclusterbits;
return 0;
}
lcn = m->lcn + 1;
if (m->compressedlcs)
goto out;
err = z_erofs_load_cluster_from_disk(m, lcn);
if (err)
return err;
/*
* If the 1st NONHEAD lcluster has already been handled initially w/o
* valid compressedlcs, which means at least it mustn't be CBLKCNT, or
* an internal implemenatation error is detected.
*
* The following code can also handle it properly anyway, but let's
* BUG_ON in the debugging mode only for developers to notice that.
*/
DBG_BUGON(lcn == initial_lcn &&
m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD);
switch (m->type) {
case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
/*
* if the 1st NONHEAD lcluster is actually PLAIN or HEAD type
* rather than CBLKCNT, it's a 1 lcluster-sized pcluster.
*/
m->compressedlcs = 1;
break;
case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
if (m->delta[0] != 1)
goto err_bonus_cblkcnt;
if (m->compressedlcs)
break;
fallthrough;
default:
erofs_err(m->inode->i_sb,
"cannot found CBLKCNT @ lcn %lu of nid %llu",
lcn, vi->nid);
DBG_BUGON(1);
return -EFSCORRUPTED;
}
out:
map->m_plen = m->compressedlcs << lclusterbits;
return 0;
err_bonus_cblkcnt:
erofs_err(m->inode->i_sb,
"bogus CBLKCNT @ lcn %lu of nid %llu",
lcn, vi->nid);
DBG_BUGON(1);
return -EFSCORRUPTED;
}
int z_erofs_map_blocks_iter(struct inode *inode,
struct erofs_map_blocks *map,
int flags)
@ -392,6 +510,7 @@ int z_erofs_map_blocks_iter(struct inode *inode,
};
int err = 0;
unsigned int lclusterbits, endoff;
unsigned long initial_lcn;
unsigned long long ofs, end;
trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
@ -404,16 +523,16 @@ int z_erofs_map_blocks_iter(struct inode *inode,
goto out;
}
err = fill_inode_lazy(inode);
err = z_erofs_fill_inode_lazy(inode);
if (err)
goto out;
lclusterbits = vi->z_logical_clusterbits;
ofs = map->m_la;
m.lcn = ofs >> lclusterbits;
initial_lcn = ofs >> lclusterbits;
endoff = ofs & ((1 << lclusterbits) - 1);
err = vle_load_cluster_from_disk(&m, m.lcn);
err = z_erofs_load_cluster_from_disk(&m, initial_lcn);
if (err)
goto unmap_out;
@ -424,7 +543,7 @@ int z_erofs_map_blocks_iter(struct inode *inode,
case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
if (endoff >= m.clusterofs)
map->m_flags &= ~EROFS_MAP_ZIPPED;
/* fallthrough */
fallthrough;
case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
if (endoff >= m.clusterofs) {
map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
@ -441,10 +560,10 @@ int z_erofs_map_blocks_iter(struct inode *inode,
end = (m.lcn << lclusterbits) | m.clusterofs;
map->m_flags |= EROFS_MAP_FULL_MAPPED;
m.delta[0] = 1;
/* fallthrough */
fallthrough;
case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
/* get the correspoinding first chunk */
err = vle_extent_lookback(&m, m.delta[0]);
/* get the corresponding first chunk */
err = z_erofs_extent_lookback(&m, m.delta[0]);
if (err)
goto unmap_out;
break;
@ -457,10 +576,12 @@ int z_erofs_map_blocks_iter(struct inode *inode,
}
map->m_llen = end - map->m_la;
map->m_plen = 1 << lclusterbits;
map->m_pa = blknr_to_addr(m.pblk);
map->m_flags |= EROFS_MAP_MAPPED;
err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
if (err)
goto out;
unmap_out:
if (m.kaddr)
kunmap_atomic(m.kaddr);

2
fs/erofs/zpvec.h
View file

@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2018 HUAWEI, Inc.
* http://www.huawei.com/
* https://www.huawei.com/
* Created by Gao Xiang <gaoxiang25@huawei.com>
*/
#ifndef __EROFS_FS_ZPVEC_H