A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-09 13:49:24 -04:00
Code Issues Projects Releases Packages Wiki Activity

introduce moto mm and moto swap driver for RAM boost 2.0

Browse source 
Change-Id: I9c3f82c29337d2Montana56ba5ec5f3fd8c0ffbcebee
Signed-off-by: huangzq2 <huangzq2@motorola.com>
Reviewed-on: https://gerrit.mot.com/2745763
SME-Granted: SME Approvals Granted
SLTApproved: Slta Waiver
Tested-by: Jira Key
Reviewed-by: Xiangpo Zhao <zhaoxp3@motorola.com>
Submit-Approved: Jira Key
This commit is contained in:
huangzq2 2023-09-20 14:11:04 +08:00 • committed by Zhangqing Huang
commit add7f46e35
25 changed files with 15646 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

12
drivers/moto_mm/Android.mk Normal file
View file

@ -0,0 +1,12 @@
DLKM_DIR := motorola/kernel/modules
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := moto_mm.ko
LOCAL_MODULE_TAGS := optional
LOCAL_MODULE_PATH := $(KERNEL_MODULES_OUT)
KBUILD_OPTIONS_GKI += GKI_OBJ_MODULE_DIR=gki
include $(DLKM_DIR)/AndroidKernelModule.mk

6
drivers/moto_mm/Kbuild Normal file
View file

@ -0,0 +1,6 @@
# add -Wall to try to catch everything we can.
EXTRA_CFLAGS += -Wall
EXTRA_CFLAGS += -I$(ANDROID_BUILD_TOP)/motorola/kernel/modules/include
obj-m += moto_mm.o

1
drivers/moto_mm/Kconfig Normal file
View file

@ -0,0 +1 @@
# SPDX-License-Identifier: GPL-2.0

14
drivers/moto_mm/Makefile Executable file
View file

@ -0,0 +1,14 @@
all: modules
modules:
$(MAKE) -C $(KERNEL_SRC) M=$(M) modules $(KBUILD_OPTIONS)
modules_install:
$(MAKE) INSTALL_MOD_STRIP=1 -C $(KERNEL_SRC) M=$(M) modules_install
%:
$(MAKE) -C $(KERNEL_SRC) M=$(M) $@ $(KBUILD_OPTIONS)
clean:
rm -f *.o *.ko *.mod.c *.mod.o *~ .*.cmd Module.symvers
rm -rf .tmp_versions

89
drivers/moto_mm/moto_mm.c Executable file
View file

@ -0,0 +1,89 @@
/*
* Copyright (C) 2023 Motorola Mobility LLC
*
* This software is licensed under the terms of the GNU General Public
* License version 2, as published by the Free Software Foundation, and
* may be copied, distributed, and modified under those terms.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*/
#define pr_fmt(fmt) "moto_mm: " fmt
#include <linux/module.h>
#include <linux/types.h>
#include <trace/hooks/vmscan.h>
#include <linux/swap.h>
static void tune_inactive_ratio_hook(void *data, unsigned long *inactive_ratio, int file)
{
if (file)
*inactive_ratio = min(2UL, *inactive_ratio);
else
*inactive_ratio = 1;
return;
}
#define REGISTER_HOOK(name) do {\
rc = register_trace_android_vh_##name(name##_hook, NULL);\
if (rc) {\
pr_err("register hook %s failed", #name);\
goto err_out_##name;\
}\
} while (0)
#define UNREGISTER_HOOK(name) do {\
unregister_trace_android_vh_##name(name##_hook, NULL);\
} while (0)
#define ERROR_OUT(name) err_out_##name
static int register_all_hooks(void)
{
int rc;
/* tune_inactive_ratio_hook */
REGISTER_HOOK(tune_inactive_ratio);
return 0;
UNREGISTER_HOOK(tune_inactive_ratio);
ERROR_OUT(tune_inactive_ratio):
return rc;
}
static void unregister_all_hook(void)
{
UNREGISTER_HOOK(tune_inactive_ratio);
}
static int __init moto_mm_init(void)
{
int ret = 0;
ret = register_all_hooks();
if (ret != 0) {
return ret;
}
pr_info("moto_mm_init succeed!\n");
return 0;
}
static void __exit moto_mm_exit(void)
{
unregister_all_hook();
pr_info("moto_mm_exit succeed!\n");
return;
}
module_init(moto_mm_init);
module_exit(moto_mm_exit);
MODULE_DESCRIPTION("Motorola mm optimizations driver");
MODULE_LICENSE("GPL v2");

17
drivers/moto_swap/Android.mk Normal file
View file

@ -0,0 +1,17 @@
DLKM_DIR := motorola/kernel/modules
LOCAL_PATH := $(call my-dir)
KERNEL_CFLAGS += CONFIG_HYBRIDSWAP_ZRAM=y
#KERNEL_CFLAGS += CONFIG_HYBRIDSWAP_ZRAM_WRITEBACK=y
#KERNEL_CFLAGS += CONFIG_HYBRIDSWAP_ASYNC_COMPRESS=y
KERNEL_CFLAGS += CONFIG_HYBRIDSWAP=y
KERNEL_CFLAGS += CONFIG_HYBRIDSWAP_SWAPD=y
KERNEL_CFLAGS += CONFIG_HYBRIDSWAP_CORE=y
include $(CLEAR_VARS)
LOCAL_MODULE := moto_swap.ko
LOCAL_MODULE_TAGS := optional
LOCAL_MODULE_PATH := $(KERNEL_MODULES_OUT)
KBUILD_OPTIONS_GKI += GKI_OBJ_MODULE_DIR=gki
include $(DLKM_DIR)/AndroidKernelModule.mk

45
drivers/moto_swap/Kbuild Normal file
View file

@ -0,0 +1,45 @@
HAVE_KERNEL_5_4 = $(shell test -d $(ANDROID_BUILD_TOP)/kernel/msm-5.4 && echo 1)
ifeq ($(HAVE_KERNEL_5_4),1)
ZRAM_SRC = zram-5.4
EXTRA_CFLAGS += -DCONFIG_ZRAM_5_4
else
ZRAM_SRC = zram-5.10
endif
# add -Wall to try to catch everything we can.
EXTRA_CFLAGS += -Wall
EXTRA_CFLAGS += -I$(ANDROID_BUILD_TOP)/motorola/kernel/modules/include
ifneq ($(filter m y,$(CONFIG_HYBRIDSWAP_ZRAM)),)
EXTRA_CFLAGS += -DCONFIG_HYBRIDSWAP_ZRAM
endif
ifneq ($(filter m y,$(CONFIG_HYBRIDSWAP_ZRAM_WRITEBACK)),)
EXTRA_CFLAGS += -DCONFIG_HYBRIDSWAP_ZRAM_WRITEBACK
endif
ifneq ($(filter m y,$(CONFIG_HYBRIDSWAP)),)
EXTRA_CFLAGS += -DCONFIG_HYBRIDSWAP
endif
ifneq ($(filter m y,$(CONFIG_HYBRIDSWAP_SWAPD)),)
EXTRA_CFLAGS += -DCONFIG_HYBRIDSWAP_SWAPD
endif
ifneq ($(filter m y,$(CONFIG_HYBRIDSWAP_CORE)),)
EXTRA_CFLAGS += -DCONFIG_HYBRIDSWAP_CORE
endif
moto_swap-objs += $(ZRAM_SRC)/zram_drv.o
moto_swap-objs += hybridswap/hybridswap_main.o
moto_swap-objs += hybridswap/hybridswap_eswap.o
ifneq ($(filter m y,$(CONFIG_HYBRIDSWAP_ASYNC_COMPRESS)),)
moto_swap-objs += hybridswap/hybridswap_akcompress.o
endif
moto_swap-objs += hybridswap/hybridswap_swapd.o
moto_swap-objs += $(ZRAM_SRC)/zcomp.o
obj-m += moto_swap.o

78
drivers/moto_swap/Kconfig Normal file
View file

@ -0,0 +1,78 @@
# SPDX-License-Identifier: GPL-2.0
config HYBRIDSWAP_ZRAM
tristate "Compressed RAM block device support"
depends on BLOCK && SYSFS && ZSMALLOC && CRYPTO && !ZRAM
select CRYPTO_LZO
help
Creates virtual block devices called /dev/zramX (X = 0, 1, ...).
Pages written to these disks are compressed and stored in memory
itself. These disks allow very fast I/O and compression provides
good amounts of memory savings.
It has several use cases, for example: /tmp storage, use as swap
disks and maybe many more.
See Documentation/admin-guide/blockdev/zram.rst for more information.
config HYBRIDSWAP_ZRAM_WRITEBACK
bool "Write back incompressible or idle page to backing device"
depends on HYBRIDSWAP_ZRAM
help
With incompressible page, there is no memory saving to keep it
in memory. Instead, write it out to backing device.
For this feature, admin should set up backing device via
/sys/block/zramX/backing_dev.
With /sys/block/zramX/{idle,writeback}, application could ask
idle page's writeback to the backing device to save in memory.
See Documentation/admin-guide/blockdev/zram.rst for more information.
config HYBRIDSWAP_ZRAM_MEMORY_TRACKING
bool "Track zRam block status"
depends on HYBRIDSWAP_ZRAM && DEBUG_FS
help
With this feature, admin can track the state of allocated blocks
of zRAM. Admin could see the information via
/sys/kernel/debug/zram/zramX/block_state.
See Documentation/admin-guide/blockdev/zram.rst for more information.
config HYBRIDSWAP
bool "Enable Hybridswap"
depends on MEMCG && HYBRIDSWAP_ZRAM && !HYBRIDSWAP_ZRAM_WRITEBACK
default y
help
Hybridswap is a intelligent memory management solution.
config HYBRIDSWAP_SWAPD
bool "Enable hybridswap swapd thread to reclaim anon pages in background"
default y
depends on HYBRIDSWAP
help
swapd is a kernel thread that reclaim anonymous pages in the
background. When the use of swap pages reaches the watermark
and the refault of anonymous pages is high, the content of
zram will exchanged to eswap by a certain percentage.
# Selected when system need hybridswap container
config HYBRIDSWAP_CORE
bool "Hybridswap container device support"
depends on HYBRIDSWAP_ZRAM && HYBRIDSWAP
default y
help
Say Y here if you want to use the hybridswap
as the backend device in ZRAM.
If unsure, say N here.
This module can't be compiled as a module,
the module is as one part of the ZRAM driver.
config HYBRIDSWAP_ASYNC_COMPRESS
bool "hypbridswap support asynchronous compress anon pages"
depends on HYBRIDSWAP_ZRAM && HYBRIDSWAP
default n
help
Say Y here if you want to create asynchronous thread
for compress anon pages.
If unsure, say N here.
This feature will reduce the kswapd cpu load.

14
drivers/moto_swap/Makefile Normal file
View file

@ -0,0 +1,14 @@
all: modules
modules:
$(MAKE) -C $(KERNEL_SRC) M=$(M) modules $(KBUILD_OPTIONS)
modules_install:
$(MAKE) INSTALL_MOD_STRIP=1 -C $(KERNEL_SRC) M=$(M) modules_install
%:
$(MAKE) -C $(KERNEL_SRC) M=$(M) $@ $(KBUILD_OPTIONS)
clean:
rm -f *.o *.ko *.mod.c *.mod.o *~ .*.cmd Module.symvers
rm -rf .tmp_versions

98
drivers/moto_swap/hybridswap/hybridswap.h Normal file
View file

@ -0,0 +1,98 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020-2022 Oplus. All rights reserved.
*/
#ifndef HYBRIDSWAP_H
#define HYBRIDSWAP_H
extern int __init hybridswap_pre_init(void);
extern ssize_t hybridswap_vmstat_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern ssize_t hybridswap_loglevel_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len);
extern ssize_t hybridswap_loglevel_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern ssize_t hybridswap_enable_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len);
extern ssize_t hybridswap_enable_show(struct device *dev,
struct device_attribute *attr, char *buf);
#ifdef CONFIG_HYBRIDSWAP_CORE
extern void hybridswap_record(struct zram *zram, u32 index, struct mem_cgroup *memcg);
extern void hybridswap_untrack(struct zram *zram, u32 index);
extern int hybridswap_page_fault(struct zram *zram, u32 index);
extern bool hybridswap_delete(struct zram *zram, u32 index);
extern ssize_t hybridswap_report_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern ssize_t hybridswap_stat_snap_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern ssize_t hybridswap_meminfo_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern ssize_t hybridswap_core_enable_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len);
extern ssize_t hybridswap_core_enable_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern ssize_t hybridswap_loop_device_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len);
extern ssize_t hybridswap_loop_device_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern ssize_t hybridswap_dev_life_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len);
extern ssize_t hybridswap_dev_life_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern ssize_t hybridswap_quota_day_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len);
extern ssize_t hybridswap_quota_day_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern ssize_t hybridswap_zram_increase_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len);
extern ssize_t hybridswap_zram_increase_show(struct device *dev,
struct device_attribute *attr, char *buf);
#endif
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
/* 63---48,47--32,31-0 : cgroup id, thread_index, index*/
#define ZRAM_INDEX_SHIFT 32
#define CACHE_INDEX_SHIFT 32
#define CACHE_INDEX_MASK ((1llu << CACHE_INDEX_SHIFT) - 1)
#define ZRAM_INDEX_MASK ((1llu << ZRAM_INDEX_SHIFT) - 1)
#define cache_index_val(index) (((unsigned long)index & CACHE_INDEX_MASK) << ZRAM_INDEX_SHIFT)
#define zram_index_val(id) ((unsigned long)id & ZRAM_INDEX_MASK)
#define mk_page_val(cache_index, index) (cache_index_val(cache_index) | zram_index_val(index))
#define fetch_cache_id(page) ((page->private >> 32) & CACHE_INDEX_MASK)
#define fetch_zram_index(page) (page->private & ZRAM_INDEX_MASK)
#define zram_set_page(zram, index, page) (zram->table[index].page = page)
#define zram_fetch_page(zram, index) (zram->table[index].page)
extern void del_page_from_cache(struct page *page);
extern int add_anon_page2cache(struct zram * zram, u32 index,
struct page *page);
extern ssize_t hybridswap_akcompress_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len);
extern ssize_t hybridswap_akcompress_show(struct device *dev,
struct device_attribute *attr, char *buf);
extern void put_free_page(struct page *page);
extern void put_anon_pages(struct page *page);
extern int akcompress_cache_page_fault(struct zram *zram,
struct page *page, u32 index);
extern void destroy_akcompressd_task(struct zram *zram);
#endif
#ifdef CONFIG_HYBRIDSWAP_SWAPD
extern ssize_t hybridswap_swapd_pause_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len);
extern ssize_t hybridswap_swapd_pause_show(struct device *dev,
struct device_attribute *attr, char *buf);
#endif
static inline bool current_is_mswapd(void)
{
#ifdef CONFIG_HYBRIDSWAP_SWAPD
return (strncmp(current->comm, "mswapd:", sizeof("mswapd:") - 1) == 0);
#else
return false;
#endif
}
#endif /* HYBRIDSWAP_H */

580
drivers/moto_swap/hybridswap/hybridswap_akcompress.c Normal file
View file

@ -0,0 +1,580 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020-2022 Oplus. All rights reserved.
*/
#define pr_fmt(fmt) "moto_swap: " fmt
#include <linux/types.h>
#include <linux/spinlock_types.h>
#include <linux/atomic.h>
#include <linux/idr.h>
#include <linux/freezer.h>
#ifdef CONFIG_ZRAM_5_4
#include "../zram-5.4/zram_drv.h"
#include "../zram-5.4/zram_drv_internal.h"
#else
#include "../zram-5.10/zram_drv.h"
#include "../zram-5.10/zram_drv_internal.h"
#endif
#include "hybridswap_internal.h"
#include "hybridswap.h"
struct compress_info_s {
struct list_head free_page_head;
spinlock_t free_lock;
unsigned int free_cnt;
unsigned int max_cnt;
} compress_info;
#define MAX_AKCOMPRESSD_THREADS 4
#define DEFAULT_CACHE_SIZE_MB 64
#define DEFAULT_COMPRESS_BATCH_MB 1
#define DEFAULT_CACHE_COUNT ((DEFAULT_CACHE_SIZE_MB << 20) >> PAGE_SHIFT)
#define WAKEUP_AKCOMPRESSD_WATERMARK ((DEFAULT_COMPRESS_BATCH_MB << 20) >> PAGE_SHIFT)
static wait_queue_head_t akcompressd_wait;
static struct task_struct *akc_task[MAX_AKCOMPRESSD_THREADS];
static atomic64_t akc_cnt[MAX_AKCOMPRESSD_THREADS];
static int akcompressd_threads = 0;
static atomic64_t cached_cnt;
static struct zram *zram_info;
static DEFINE_MUTEX(akcompress_init_lock);
struct idr cached_idr = IDR_INIT(cached_idr);
DEFINE_SPINLOCK(cached_idr_lock);
static void wake_all_akcompressd(void);
void clear_page_memcg(struct cgroup_cache_page *cache)
{
struct list_head *pos;
struct page *page;
spin_lock(&cache->lock);
if (list_empty(&cache->head))
goto out;
list_for_each(pos, &cache->head) {
page = list_entry(pos, struct page, lru);
if (!page->mem_cgroup)
BUG();
page->mem_cgroup = NULL;
}
out:
cache->dead = 1;
spin_unlock(&cache->lock);
}
static inline struct page * fetch_free_page(void)
{
struct page *page = NULL;
spin_lock(&compress_info.free_lock);
if (compress_info.free_cnt > 0) {
if (list_empty(&compress_info.free_page_head))
BUG();
page = lru_to_page(&compress_info.free_page_head);
list_del(&page->lru);
compress_info.free_cnt--;
}
spin_unlock(&compress_info.free_lock);
return page;
}
void put_free_page(struct page *page)
{
set_page_private(page, 0);
spin_lock(&compress_info.free_lock);
list_add_tail(&page->lru, &compress_info.free_page_head);
compress_info.free_cnt++;
spin_unlock(&compress_info.free_lock);
}
static inline struct cgroup_cache_page *find_and_fetch_memcg_cache(int cache_id)
{
struct cgroup_cache_page *cache;
spin_lock(&cached_idr_lock);
cache = (struct cgroup_cache_page *)idr_find(&cached_idr, cache_id);
if (unlikely(!cache)) {
spin_unlock(&cached_idr_lock);
pr_err("cache_id %d cache not find.\n", cache_id);
return NULL;
}
fetch_memcg_cache(container_of(cache, memcg_hybs_t, cache));
spin_unlock(&cached_idr_lock);
return cache;
}
void del_page_from_cache(struct page *page)
{
int cache_id;
struct cgroup_cache_page *cache;
if (!page)
return;
cache_id = fetch_cache_id(page);
if (unlikely(cache_id < 0 || cache_id > MEM_CGROUP_ID_MAX)) {
hybp(HYB_ERR, "page %p cache_id %d index %u is invalid.\n",
page, cache_id, fetch_zram_index(page));
return;
}
cache = find_and_fetch_memcg_cache(cache_id);
if (!cache)
return;
spin_lock(&cache->lock);
list_del(&page->lru);
cache->cnt--;
spin_unlock(&cache->lock);
put_memcg_cache(container_of(cache, memcg_hybs_t, cache));
atomic64_dec(&cached_cnt);
}
void del_page_from_cache_with_cache(struct page *page,
struct cgroup_cache_page *cache)
{
spin_lock(&cache->lock);
list_del(&page->lru);
cache->cnt--;
spin_unlock(&cache->lock);
atomic64_dec(&cached_cnt);
}
void put_anon_pages(struct page *page)
{
memcg_hybs_t *hybs = MEMCGRP_ITEM_DATA(page->mem_cgroup);
spin_lock(&hybs->cache.lock);
list_add(&page->lru, &hybs->cache.head);
hybs->cache.cnt++;
spin_unlock(&hybs->cache.lock);
}
static inline bool can_stop_working(struct cgroup_cache_page *cache, int index)
{
spin_lock(&cache->lock);
if (unlikely(!list_empty(&cache->head))) {
spin_unlock(&cache->lock);
return false;
}
spin_unlock(&cache->lock);
return 1;
}
static int check_cache_state(struct cgroup_cache_page *cache)
{
if (cache->cnt == 0 || cache->compressing == 1)
return 0;
spin_lock(&cache->lock);
if (cache->cnt == 0 || cache->compressing) {
spin_unlock(&cache->lock);
return 0;
}
cache->compressing = 1;
spin_unlock(&cache->lock);
fetch_memcg_cache(container_of(cache, memcg_hybs_t, cache));
return 1;
}
struct cgroup_cache_page *fetch_one_cache(void)
{
struct cgroup_cache_page *cache = NULL;
int id;
spin_lock(&cached_idr_lock);
idr_for_each_entry(&cached_idr, cache, id) {
if (check_cache_state(cache))
break;
}
spin_unlock(&cached_idr_lock);
return cache;
}
void mark_compressing_stop(struct cgroup_cache_page *cache)
{
spin_lock(&cache->lock);
if (cache->dead)
hybp(HYB_WARN, "stop compressing, may be cgroup is delelted\n");
cache->compressing = 0;
spin_unlock(&cache->lock);
put_memcg_cache(container_of(cache, memcg_hybs_t, cache));
}
static inline struct page *fetch_anon_page(struct zram *zram,
struct cgroup_cache_page *cache)
{
struct page *page, *prev_page;
int index;
if (compress_info.free_cnt == 0)
return NULL;
prev_page = NULL;
try_again:
page = NULL;
spin_lock(&cache->lock);
if (!list_empty(&cache->head)) {
page = lru_to_page(&cache->head);
index = fetch_zram_index(page);
}
spin_unlock(&cache->lock);
if (page) {
if (prev_page && (page == prev_page)) {
hybp(HYB_ERR, "zram %p index %d page %p\n",
zram, index, page);
BUG();
}
zram_slot_lock(zram, index);
if (!zram_test_flag(zram, index, ZRAM_CACHED)) {
zram_slot_unlock(zram, index);
prev_page = page;
goto try_again;
}
prev_page = NULL;
zram_clear_flag(zram, index, ZRAM_CACHED);
del_page_from_cache_with_cache(page, cache);
zram_set_flag(zram, index, ZRAM_CACHED_COMPRESS);
zram_slot_unlock(zram, index);
}
return page;
}
int add_anon_page2cache(struct zram * zram, u32 index, struct page *page)
{
struct page *dst_page;
void *src, *dst;
struct mem_cgroup *memcg;
struct cgroup_cache_page *cache;
memcg_hybs_t *hybs;
if (akcompressd_threads == 0)
return 0;
memcg = page->mem_cgroup;
if (!memcg || !MEMCGRP_ITEM_DATA(memcg))
return 0;
hybs = MEMCGRP_ITEM_DATA(memcg);
cache = &hybs->cache;
if (find_and_fetch_memcg_cache(cache->id) != cache)
return 0;
spin_lock(&cache->lock);
if (cache->dead == 1) {
spin_unlock(&cache->lock);
return 0;
}
spin_unlock(&cache->lock);
dst_page = fetch_free_page();
if (!dst_page)
return 0;
src = kmap_atomic(page);
dst = kmap_atomic(dst_page);
memcpy(dst, src, PAGE_SIZE);
kunmap_atomic(src);
kunmap_atomic(dst);
dst_page->mem_cgroup = memcg;
set_page_private(dst_page, mk_page_val(cache->id, index));
update_zram_index(zram, index, (unsigned long)dst_page);
atomic64_inc(&cached_cnt);
wake_all_akcompressd();
hybp(HYB_DEBUG, "add_anon_page2cache index %u page %p passed\n",
index, dst_page);
return 1;
}
static inline void akcompressd_try_to_sleep(wait_queue_head_t *waitq)
{
DEFINE_WAIT(wait);
prepare_to_wait(waitq, &wait, TASK_INTERRUPTIBLE);
freezable_schedule();
finish_wait(waitq, &wait);
}
static int akcompressd_func(void *data)
{
struct page *page;
int ret, thread_index;
struct list_head compress_fail_list;
struct cgroup_cache_page *cache = NULL;
thread_index = (int)data;
if (thread_index < 0 || thread_index >= MAX_AKCOMPRESSD_THREADS) {
hybp(HYB_ERR, "akcompress task index %d is invalid.\n", thread_index);
return -EINVAL;
}
set_freezable();
while (!kthread_should_stop()) {
akcompressd_try_to_sleep(&akcompressd_wait);
count_swapd_event(AKCOMPRESSD_WAKEUP);
cache = fetch_one_cache();
if (!cache)
continue;
finish_last_jobs:
INIT_LIST_HEAD(&compress_fail_list);
page = fetch_anon_page(zram_info, cache);
while (page) {
ret = async_compress_page(zram_info, page);
put_memcg_cache(container_of(cache, memcg_hybs_t, cache));
if (ret)
list_add(&page->lru, &compress_fail_list);
else {
atomic64_inc(&akc_cnt[thread_index]);
page->mem_cgroup = NULL;
put_free_page(page);
}
page = fetch_anon_page(zram_info, cache);
}
if (!list_empty(&compress_fail_list))
hybp(HYB_ERR, "have some compress failed pages.\n");
if (kthread_should_stop()) {
if (!can_stop_working(cache, thread_index))
goto finish_last_jobs;
}
mark_compressing_stop(cache);
}
return 0;
}
static int update_akcompressd_threads(int thread_count, struct zram *zram)
{
int drop, increase;
int last_index, start_index, hid;
static DEFINE_MUTEX(update_lock);
if (thread_count < 0 || thread_count > MAX_AKCOMPRESSD_THREADS) {
hybp(HYB_ERR, "thread_count %d is invalid\n", thread_count);
return -EINVAL;
}
mutex_lock(&update_lock);
if (!zram_info || zram_info != zram)
zram_info = zram;
if (thread_count == akcompressd_threads) {
mutex_unlock(&update_lock);
return thread_count;
}
last_index = akcompressd_threads - 1;
if (thread_count < akcompressd_threads) {
drop = akcompressd_threads - thread_count;
for (hid = last_index; hid > (last_index - drop); hid--) {
if (akc_task[hid]) {
kthread_stop(akc_task[hid]);
akc_task[hid] = NULL;
}
}
} else {
increase = thread_count - akcompressd_threads;
start_index = last_index + 1;
for (hid = start_index; hid < (start_index + increase); hid++) {
if (unlikely(akc_task[hid]))
BUG();
akc_task[hid]= kthread_run(akcompressd_func,
(void*)(unsigned long)hid, "akcompressd:%d", hid);
if (IS_ERR(akc_task[hid])) {
pr_err("Failed to start akcompressd%d\n", hid);
akc_task[hid] = NULL;
break;
}
}
}
hybp(HYB_INFO, "akcompressd_threads count changed, old:%d new:%d\n",
akcompressd_threads, thread_count);
akcompressd_threads = thread_count;
mutex_unlock(&update_lock);
return thread_count;
}
static void wake_all_akcompressd(void)
{
if (atomic64_read(&cached_cnt) < WAKEUP_AKCOMPRESSD_WATERMARK)
return;
if (!waitqueue_active(&akcompressd_wait))
return;
wake_up_interruptible(&akcompressd_wait);
}
int create_akcompressd_task(struct zram *zram)
{
return update_akcompressd_threads(1, zram) != 1;
}
void destroy_akcompressd_task(struct zram *zram)
{
(void)update_akcompressd_threads(0, zram);
}
ssize_t hybridswap_akcompress_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len)
{
int ret;
unsigned long val;
struct zram *zram = dev_to_zram(dev);
ret = kstrtoul(buf, 0, &val);
if (unlikely(ret)) {
hybp(HYB_ERR, "val is error!\n");
return -EINVAL;
}
ret = update_akcompressd_threads(val, zram);
if (ret < 0) {
hybp(HYB_ERR, "create task failed, val %d\n", val);
return ret;
}
return len;
}
ssize_t hybridswap_akcompress_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
int len = 0, id, i;
struct cgroup_cache_page *cache = NULL;
unsigned long cnt = atomic64_read(&cached_cnt);
memcg_hybs_t *hybs;
len += sprintf(buf + len, "akcompressd_threads: %d\n", akcompressd_threads);
len += sprintf(buf + len, "cached page cnt: %lu\n", cnt);
len += sprintf(buf + len, "free page cnt: %u\n", compress_info.free_cnt);
for (i = 0; i < MAX_AKCOMPRESSD_THREADS; i++)
len += sprintf(buf + len, "%-d %-d\n", i, atomic64_read(&akc_cnt[i]));
if (cnt == 0)
return len;
spin_lock(&cached_idr_lock);
idr_for_each_entry(&cached_idr, cache, id) {
hybs = container_of(cache, memcg_hybs_t, cache);
if (cache->cnt == 0)
continue;
len += scnprintf(buf + len, PAGE_SIZE - len, "%s %d\n",
hybs->name, cache->cnt);
if (len >= PAGE_SIZE)
break;
}
spin_unlock(&cached_idr_lock);
return len;
}
void __init akcompressd_pre_init(void)
{
int i;
struct page *page;
mutex_lock(&akcompress_init_lock);
INIT_LIST_HEAD(&compress_info.free_page_head);
spin_lock_init(&compress_info.free_lock);
compress_info.free_cnt = 0;
init_waitqueue_head(&akcompressd_wait);
atomic64_set(&cached_cnt, 0);
for (i = 0; i < MAX_AKCOMPRESSD_THREADS; i++)
atomic64_set(&akc_cnt[i], 0);
for (i = 0; i < DEFAULT_CACHE_COUNT; i ++) {
page = alloc_page(GFP_KERNEL);
if (page) {
list_add_tail(&page->lru, &compress_info.free_page_head);
} else
break;
}
compress_info.free_cnt = i;
mutex_unlock(&akcompress_init_lock);
}
void __exit akcompressd_pre_deinit(void)
{
int i;
struct page *page, *tmp;
mutex_lock(&akcompress_init_lock);
if (list_empty(&compress_info.free_page_head))
goto out;
list_for_each_entry_safe(page, tmp, &compress_info.free_page_head , lru) {
list_del(&page->lru);
free_page(page);
}
out:
compress_info.free_cnt = 0;
mutex_unlock(&akcompress_init_lock);
}
int akcompress_cache_page_fault(struct zram *zram,
struct page *page, u32 index)
{
void *src, *dst;
if (zram_test_flag(zram, index, ZRAM_CACHED)) {
struct page *src_page = (struct page *)zram_fetch_page(zram, index);
src = kmap_atomic(src_page);
dst = kmap_atomic(page);
memcpy(dst, src, PAGE_SIZE);
kunmap_atomic(src);
kunmap_atomic(dst);
zram_slot_unlock(zram, index);
hybp(HYB_DEBUG, "read_anon_page_from_cache index %u page %p passed, ZRAM_CACHED\n",
index, src_page);
return 1;
}
if (zram_test_flag(zram, index, ZRAM_CACHED_COMPRESS)) {
struct page *src_page = (struct page *)zram_fetch_page(zram, index);
src = kmap_atomic(src_page);
dst = kmap_atomic(page);
memcpy(dst, src, PAGE_SIZE);
kunmap_atomic(src);
kunmap_atomic(dst);
zram_slot_unlock(zram, index);
hybp(HYB_DEBUG, "read_anon_page_from_cache index %u page %p passed, ZRAM_CACHED_COMPRESS\n",
index, src_page);
return 1;
}
return 0;
}

5503
drivers/moto_swap/hybridswap/hybridswap_eswap.c Normal file
View file

File diff suppressed because it is too large Load diff

560
drivers/moto_swap/hybridswap/hybridswap_internal.h Normal file
View file

@ -0,0 +1,560 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020-2022 Oplus. All rights reserved.
*/
#ifndef HYBRIDSWAP_INTERNAL_H
#define HYBRIDSWAP_INTERNAL_H
#include <linux/sched.h>
#include <linux/zsmalloc.h>
#include <linux/timer.h>
#include <linux/device.h>
#include <linux/memcontrol.h>
#define ESWAP_SHIFT 15
#define ESWAP_SIZE (1UL << ESWAP_SHIFT)
#define ESWAP_PG_CNT (ESWAP_SIZE >> PAGE_SHIFT)
#define ESWAP_SECTOR_SIZE (ESWAP_PG_CNT << 3)
#define ESWAP_MAX_OBJ_CNT (30 * ESWAP_PG_CNT)
#define ESWAP_MASK (~(ESWAP_SIZE - 1))
#define ESWAP_ALIGN_UP(size) ((size + ESWAP_SIZE - 1) & ESWAP_MASK)
#define MAX_FAIL_RECORD_NUM 4
#define MAX_APP_GRADE 600
#define HYBRIDSWAP_QUOTA_DAY 0x280000000 /* 10G bytes */
#define HYBRIDSWAP_CHECK_GAP 86400 /* 24 hour */
#define MEM_CGROUP_NAME_MAX_LEN 32
#define MAX_RATIO 100
#define MIN_RATIO 0
enum {
HYB_ERR = 0,
HYB_WARN,
HYB_INFO,
HYB_DEBUG,
HYB_MAX
};
void hybridswap_loglevel_set(int level);
int hybridswap_loglevel(void);
#define DUMP_STACK_ON_ERR 0
#define pt(l, f, ...) pr_err("[%s][%s]:"f, #l, __func__, ##__VA_ARGS__)
static inline void pr_none(void) {}
#define hybp(l, f, ...) do {\
(l <= hybridswap_loglevel()) ? pt(l, f, ##__VA_ARGS__) : pr_none();\
if (DUMP_STACK_ON_ERR && l == HYB_ERR) dump_stack();\
} while (0)
enum hybridswap_class {
HYB_RECLAIM_IN = 0,
HYB_FAULT_OUT,
HYB_BATCH_OUT,
HYB_PRE_OUT,
HYB_CLASS_BUTT
};
enum hybridswap_key_point {
HYB_START = 0,
HYB_INIT,
HYB_IOENTRY_ALLOC,
HYB_FIND_ESWAP,
HYB_IO_ESWAP,
HYB_SEGMENT_ALLOC,
HYB_BIO_ALLOC,
HYB_SUBMIT_BIO,
HYB_END_IO,
HYB_SCHED_WORK,
HYB_END_WORK,
HYB_CALL_BACK,
HYB_WAKE_UP,
HYB_ZRAM_LOCK,
HYB_DONE,
HYB_KYE_POINT_BUTT
};
enum hybridswap_mcg_member {
MCG_ZRAM_STORED_SZ = 0,
MCG_ZRAM_STORED_PG_SZ,
MCG_DISK_STORED_SZ,
MCG_DISK_STORED_PG_SZ,
MCG_ANON_FAULT_CNT,
MCG_DISK_FAULT_CNT,
MCG_ESWAPOUT_CNT,
MCG_ESWAPOUT_SZ,
MCG_ESWAPIN_CNT,
MCG_ESWAPIN_SZ,
MCG_DISK_SPACE,
MCG_DISK_SPACE_PEAK,
};
enum hybridswap_fail_point {
HYB_FAULT_OUT_INIT_FAIL = 0,
HYB_FAULT_OUT_ENTRY_ALLOC_FAIL,
HYB_FAULT_OUT_IO_ENTRY_PARA_FAIL,
HYB_FAULT_OUT_SEGMENT_ALLOC_FAIL,
HYB_FAULT_OUT_BIO_ALLOC_FAIL,
HYB_FAULT_OUT_BIO_ADD_FAIL,
HYB_FAULT_OUT_IO_FAIL,
HYBRIDSWAP_FAIL_POINT_BUTT
};
struct hybridswap_fail_record {
unsigned char task_comm[TASK_COMM_LEN];
enum hybridswap_fail_point point;
ktime_t time;
u32 index;
int eswapid;
};
struct hybridswap_fail_record_info {
int num;
spinlock_t lock;
struct hybridswap_fail_record record[MAX_FAIL_RECORD_NUM];
};
struct hybridswap_key_point_info {
unsigned int record_cnt;
unsigned int end_cnt;
ktime_t first_time;
ktime_t last_time;
s64 proc_total_time;
s64 proc_max_time;
unsigned long long last_ravg_sum;
unsigned long long proc_ravg_sum;
spinlock_t time_lock;
};
struct hybridswap_key_point_record {
struct timer_list lat_monitor;
unsigned long warn_level;
int page_cnt;
int segment_cnt;
int nice;
bool timeout_flag;
unsigned char task_comm[TASK_COMM_LEN];
struct task_struct *task;
enum hybridswap_class class;
struct hybridswap_key_point_info key_point[HYB_KYE_POINT_BUTT];
};
struct hybridswapiowrkstat {
atomic64_t total_lat;
atomic64_t max_lat;
atomic64_t timeout_cnt;
};
struct hybridswap_fault_timeout_cnt{
atomic64_t timeout_100ms_cnt;
atomic64_t timeout_500ms_cnt;
};
struct hybstatus {
atomic64_t reclaimin_cnt;
atomic64_t reclaimin_bytes;
atomic64_t reclaimin_real_load;
atomic64_t reclaimin_bytes_daily;
atomic64_t reclaimin_pages;
atomic64_t reclaimin_infight;
atomic64_t batchout_cnt;
atomic64_t batchout_bytes;
atomic64_t batchout_real_load;
atomic64_t batchout_pages;
atomic64_t batchout_inflight;
atomic64_t fault_cnt;
atomic64_t hybridswap_fault_cnt;
atomic64_t reout_pages;
atomic64_t reout_bytes;
atomic64_t zram_stored_pages;
atomic64_t zram_stored_size;
atomic64_t stored_pages;
atomic64_t stored_size;
atomic64_t notify_free;
atomic64_t frag_cnt;
atomic64_t mcg_cnt;
atomic64_t eswap_cnt;
atomic64_t miss_free;
atomic64_t memcgid_clear;
atomic64_t skip_track_cnt;
atomic64_t used_swap_pages;
atomic64_t null_memcg_skip_track_cnt;
atomic64_t stored_wm_scale;
atomic64_t dropped_eswap_size;
atomic64_t io_fail_cnt[HYB_CLASS_BUTT];
atomic64_t alloc_fail_cnt[HYB_CLASS_BUTT];
struct hybridswapiowrkstat lat[HYB_CLASS_BUTT];
struct hybridswap_fault_timeout_cnt fault_stat[2]; /* 0:bg 1:fg */
struct hybridswap_fail_record_info record;
};
struct hybridswap_page_pool {
struct list_head page_pool_list;
spinlock_t page_pool_lock;
};
struct io_eswapent {
int eswapid;
struct zram *zram;
struct mem_cgroup *mcg;
struct page *pages[ESWAP_PG_CNT];
u32 index[ESWAP_MAX_OBJ_CNT];
int cnt;
int real_load;
struct hybridswap_page_pool *pool;
};
struct hybridswap_buffer {
struct zram *zram;
struct hybridswap_page_pool *pool;
struct page **dest_pages;
};
struct hybridswap_entry {
int eswapid;
sector_t addr;
struct page **dest_pages;
int pages_sz;
struct list_head list;
void *private;
void *manager_private;
};
struct hybridswap_io_req;
struct hybridswap_io {
struct block_device *bdev;
enum hybridswap_class class;
void (*done_callback)(struct hybridswap_entry *, int, struct hybridswap_io_req *);
void (*complete_notify)(void *);
void *private;
struct hybridswap_key_point_record *record;
};
struct hybridswap_io_req {
struct hybridswap_io io_para;
struct kref refcount;
struct mutex refmutex;
struct wait_queue_head io_wait;
atomic_t eswap_doing;
struct completion io_end_flag;
struct hyb_sgm *segment;
bool limit_doing_flag;
bool wait_io_finish_flag;
int page_cnt;
int segment_cnt;
int nice;
atomic64_t real_load;
};
/* Change hybridswap_event_item, you should change swapd_text togather*/
enum hybridswap_event_item {
#ifdef CONFIG_HYBRIDSWAP_SWAPD
SWAPD_WAKEUP,
SWAPD_REFAULT,
SWAPD_MEMCG_RATIO_SKIP,
SWAPD_MEMCG_REFAULT_SKIP,
SWAPD_SHRINK_ANON,
SWAPD_SWAPOUT,
SWAPD_SKIP_SWAPOUT,
SWAPD_EMPTY_ROUND,
SWAPD_OVER_MIN_BUFFER_SKIP_TIMES,
SWAPD_EMPTY_ROUND_SKIP_TIMES,
SWAPD_SNAPSHOT_TIMES,
SWAPD_SKIP_SHRINK_OF_WINDOW,
SWAPD_MANUAL_PAUSE,
#ifdef CONFIG_OPLUS_JANK
SWAPD_CPU_BUSY_SKIP_TIMES,
SWAPD_CPU_BUSY_BREAK_TIMES,
#endif
#endif
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
AKCOMPRESSD_WAKEUP,
#endif
NR_EVENT_ITEMS
};
struct swapd_event_state {
unsigned long event[NR_EVENT_ITEMS];
};
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
struct cgroup_cache_page {
spinlock_t lock;
struct list_head head;
unsigned int cnt;
int id;
char compressing;
char dead;
};
#endif
typedef struct mem_cgroup_hybridswap {
#ifdef CONFIG_HYBRIDSWAP
atomic64_t ufs2zram_scale;
atomic_t zram2ufs_scale;
atomic64_t app_grade;
atomic64_t app_uid;
struct list_head grade_node;
char name[MEM_CGROUP_NAME_MAX_LEN];
struct zram *zram;
struct mem_cgroup *memcg;
refcount_t usage;
#endif
#ifdef CONFIG_HYBRIDSWAP_SWAPD
atomic_t mem2zram_scale;
atomic_t pagefault_level;
unsigned long long reclaimed_pagefault;
long long can_reclaimed;
#endif
#ifdef CONFIG_HYBRIDSWAP_CORE
unsigned long swap_sorted_list;
unsigned long eswap_lru;
struct list_head link_list;
spinlock_t zram_init_lock;
long long can_eswaped;
atomic64_t zram_stored_size;
atomic64_t zram_page_size;
unsigned long zram_watermark;
atomic_t hybridswap_extcnt;
atomic_t hybridswap_peakextcnt;
atomic64_t hybridswap_stored_pages;
atomic64_t hybridswap_stored_size;
atomic64_t hybridswap_eswap_notify_free;
atomic64_t hybridswap_outcnt;
atomic64_t hybridswap_incnt;
atomic64_t hybridswap_allfaultcnt;
atomic64_t hybridswap_faultcnt;
atomic64_t hybridswap_outextcnt;
atomic64_t hybridswap_inextcnt;
struct mutex swap_lock;
bool in_swapin;
bool force_swapout;
#endif
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
struct cgroup_cache_page cache;
#endif
}memcg_hybs_t;
#define MEMCGRP_ITEM_DATA(memcg) ((memcg_hybs_t *)(memcg)->android_oem_data1)
#define MEMCGRP_ITEM(memcg, item) (MEMCGRP_ITEM_DATA(memcg)->item)
extern void __put_memcg_cache(memcg_hybs_t *hybs);
static inline memcg_hybs_t *fetch_memcg_cache(memcg_hybs_t *hybs)
{
refcount_inc(&hybs->usage);
return hybs;
}
static inline void put_memcg_cache(memcg_hybs_t *hybs)
{
if (refcount_dec_and_test(&hybs->usage))
__put_memcg_cache(hybs);
}
DECLARE_PER_CPU(struct swapd_event_state, swapd_event_states);
extern struct mutex reclaim_para_lock;
static inline void __count_swapd_event(enum hybridswap_event_item item)
{
raw_cpu_inc(swapd_event_states.event[item]);
}
static inline void count_swapd_event(enum hybridswap_event_item item)
{
this_cpu_inc(swapd_event_states.event[item]);
}
static inline void __count_swapd_events(enum hybridswap_event_item item, long delta)
{
raw_cpu_add(swapd_event_states.event[item], delta);
}
static inline void count_swapd_events(enum hybridswap_event_item item, long delta)
{
this_cpu_add(swapd_event_states.event[item], delta);
}
void *hybridswap_malloc(size_t size, bool fast, bool nofail);
void hybridswap_free(const void *mem);
unsigned long hybridswap_zsmalloc(struct zs_pool *zs_pool,
size_t size, struct hybridswap_page_pool *pool);
struct page *hybridswap_alloc_page(
struct hybridswap_page_pool *pool, gfp_t gfp,
bool fast, bool nofail);
void hybridswap_page_recycle(struct page *page,
struct hybridswap_page_pool *pool);
struct hybstatus *hybridswap_fetch_stat_obj(void);
int hybridswap_manager_init(struct zram *zram);
void hybridswap_manager_memcg_init(struct zram *zram,
struct mem_cgroup *memcg);
void hybridswap_manager_memcg_deinit(struct mem_cgroup *mcg);
void hybridswap_swap_sorted_list_add(struct zram *zram, u32 index,
struct mem_cgroup *memcg);
void hybridswap_swap_sorted_list_del(struct zram *zram, u32 index);
unsigned long hybridswap_eswap_create(struct mem_cgroup *memcg,
int *eswapid,
struct hybridswap_buffer *dest_buf,
void **private);
void hybridswap_eswap_register(void *private, struct hybridswap_io_req *req);
void hybridswap_eswap_objs_del(struct zram *zram, u32 index);
int hybridswap_find_eswap_by_index(
unsigned long eswpentry, struct hybridswap_buffer *buf, void **private);
int hybridswap_find_eswap_by_memcg(
struct mem_cgroup *mcg,
struct hybridswap_buffer *dest_buf, void **private);
void hybridswap_eswap_destroy(void *private, enum hybridswap_class class);
void hybridswap_eswap_exception(enum hybridswap_class class,
void *private);
void hybridswap_manager_deinit(struct zram *zram);
struct mem_cgroup *hybridswap_zram_fetch_mcg(struct zram *zram, u32 index);
int hyb_io_work_begin(void);
void *hybridswap_plug_start(struct hybridswap_io *io_para);
int hybridswap_read_eswap(void *iohandle,
struct hybridswap_entry *ioentry);
int hybridswap_write_eswap(void *iohandle,
struct hybridswap_entry *ioentry);
int hybridswap_plug_finish(void *iohandle);
void hybperf_start(
struct hybridswap_key_point_record *record,
ktime_t stsrt, unsigned long long start_ravg_sum,
enum hybridswap_class class);
void hybperf_end(struct hybridswap_key_point_record *record);
void hybperfiowrkstart(
struct hybridswap_key_point_record *record,
enum hybridswap_key_point type);
void hybperfiowrkend(
struct hybridswap_key_point_record *record,
enum hybridswap_key_point type);
void hybperfiowrkpoint(
struct hybridswap_key_point_record *record,
enum hybridswap_key_point type);
void hybperf_async_perf(
struct hybridswap_key_point_record *record,
enum hybridswap_key_point type, ktime_t start,
unsigned long long start_ravg_sum);
void hybperf_io_stat(
struct hybridswap_key_point_record *record, int page_cnt,
int segment_cnt);
static inline unsigned long long hybridswap_fetch_ravg_sum(void)
{
return 0;
}
void hybridswap_fail_record(enum hybridswap_fail_point point,
u32 index, int eswapid, unsigned char *task_comm);
bool hybridswap_reach_life_protect(void);
struct workqueue_struct *hybridswap_fetch_reclaim_workqueue(void);
extern struct mem_cgroup *fetch_next_memcg(struct mem_cgroup *prev);
extern void fetch_next_memcg_break(struct mem_cgroup *prev);
extern memcg_hybs_t *hybridswap_cache_alloc(struct mem_cgroup *memcg, bool atomic);
extern void memcg_app_grade_resort(void);
extern unsigned long memcg_anon_pages(struct mem_cgroup *memcg);
#ifdef CONFIG_HYBRIDSWAP_CORE
extern bool hybridswap_core_enabled(void);
extern bool hybridswap_out_to_eswap_enable(void);
extern void hybridswap_mem_cgroup_deinit(struct mem_cgroup *memcg);
extern unsigned long hybridswap_out_to_eswap(unsigned long size);
extern int hybridswap_batches(struct mem_cgroup *mcg,
unsigned long size, bool preload);
extern unsigned long zram_zsmalloc(struct zs_pool *zs_pool,
size_t size, gfp_t gfp);
extern struct task_struct *fetch_task_from_proc(struct inode *inode);
extern unsigned long hybridswap_fetch_zram_used_pages(void);
extern unsigned long long hybridswap_fetch_zram_pagefault(void);
extern bool hybridswap_reclaim_work_running(void);
extern void hybridswap_force_reclaim(struct mem_cgroup *mcg);
extern bool hybridswap_stored_wm_ok(void);
extern void mem_cgroup_id_remove_hook(void *data, struct mem_cgroup *memcg);
extern int mem_cgroup_stored_wm_scale_write(
struct cgroup_subsys_state *css, struct cftype *cft, s64 val);
extern s64 mem_cgroup_stored_wm_scale_read(
struct cgroup_subsys_state *css, struct cftype *cft);
extern bool hybridswap_delete(struct zram *zram, u32 index);
extern int hybridswap_stored_info(unsigned long *total, unsigned long *used);
extern unsigned long long hybridswap_read_mcg_stats(
struct mem_cgroup *mcg, enum hybridswap_mcg_member mcg_member);
extern int hybridswap_core_enable(void);
extern void hybridswap_core_disable(void);
extern int hybridswap_psi_show(struct seq_file *m, void *v);
#else
static inline unsigned long long hybridswap_read_mcg_stats(
struct mem_cgroup *mcg, enum hybridswap_mcg_member mcg_member)
{
return 0;
}
unsigned long long hybridswap_fetch_zram_pagefault(void)
{
return 0;
}
static inline unsigned long long hybridswap_fetch_zram_pagefault(void)
{
return 0;
}
static inline bool hybridswap_reclaim_work_running(void)
{
return false;
}
static inline bool hybridswap_core_enabled(void) { return false; }
static inline bool hybridswap_out_to_eswap_enable(void) { return false; }
#endif
#ifdef CONFIG_HYBRIDSWAP_SWAPD
extern atomic_long_t page_fault_pause;
extern atomic_long_t page_fault_pause_cnt;
extern struct cftype mem_cgroup_swapd_legacy_files[];
extern bool zram_watermark_ok(void);
extern void wake_all_swapd(void);
extern void alloc_pages_slowpath_hook(void *data, gfp_t gfp_mask,
unsigned int order, unsigned long delta);
extern void rmqueue_hook(void *data, struct zone *preferred_zone,
struct zone *zone, unsigned int order, gfp_t gfp_flags,
unsigned int alloc_flags, int migratetype);
extern void __init swapd_pre_init(void);
extern void swapd_pre_deinit(void);
extern void update_swapd_mcg_setup(struct mem_cgroup *memcg);
extern bool free_zram_is_ok(void);
extern bool free_swap_is_low(void);
extern unsigned long fetch_nr_zram_total(void);
extern int swapd_init(struct zram *zram);
extern void swapd_exit(void);
extern bool hybridswap_swapd_enabled(void);
#else
static inline bool hybridswap_swapd_enabled(void) { return false; }
#endif
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
extern spinlock_t cached_idr_lock;
extern struct idr cached_idr;
extern void __init akcompressd_pre_init(void);
extern void __exit akcompressd_pre_deinit(void);
extern int create_akcompressd_task(struct zram *zram);
extern void clear_page_memcg(struct cgroup_cache_page *cache);
#endif
#endif /* end of HYBRIDSWAP_INTERNAL_H */

1062
drivers/moto_swap/hybridswap/hybridswap_main.c Normal file
View file

File diff suppressed because it is too large Load diff

1963
drivers/moto_swap/hybridswap/hybridswap_swapd.c Normal file
View file

File diff suppressed because it is too large Load diff

232
drivers/moto_swap/zram-5.10/zcomp.c Normal file
View file

@ -0,0 +1,232 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2014 Sergey Senozhatsky.
*/
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <linux/cpu.h>
#include <linux/crypto.h>
#include "zcomp.h"
static const char * const backends[] = {
"lzo",
"lzo-rle",
#if IS_ENABLED(CONFIG_CRYPTO_LZ4)
"lz4",
#endif
#if IS_ENABLED(CONFIG_CRYPTO_LZ4HC)
"lz4hc",
#endif
#if IS_ENABLED(CONFIG_CRYPTO_842)
"842",
#endif
#if IS_ENABLED(CONFIG_CRYPTO_ZSTD)
"zstd",
#endif
};
static void zcomp_strm_free(struct zcomp_strm *zstrm)
{
if (!IS_ERR_OR_NULL(zstrm->tfm))
crypto_free_comp(zstrm->tfm);
free_pages((unsigned long)zstrm->buffer, 1);
zstrm->tfm = NULL;
zstrm->buffer = NULL;
}
/*
* Initialize zcomp_strm structure with ->tfm initialized by backend, and
* ->buffer. Return a negative value on error.
*/
static int zcomp_strm_init(struct zcomp_strm *zstrm, struct zcomp *comp)
{
zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0);
/*
* allocate 2 pages. 1 for compressed data, plus 1 extra for the
* case when compressed size is larger than the original one
*/
zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) {
zcomp_strm_free(zstrm);
return -ENOMEM;
}
return 0;
}
bool zcomp_available_algorithm(const char *comp)
{
int i;
i = sysfs_match_string(backends, comp);
if (i >= 0)
return true;
/*
* Crypto does not ignore a trailing new line symbol,
* so make sure you don't supply a string containing
* one.
* This also means that we permit zcomp initialisation
* with any compressing algorithm known to crypto api.
*/
return crypto_has_comp(comp, 0, 0) == 1;
}
/* show available compressors */
ssize_t zcomp_available_show(const char *comp, char *buf)
{
bool known_algorithm = false;
ssize_t sz = 0;
int i;
for (i = 0; i < ARRAY_SIZE(backends); i++) {
if (!strcmp(comp, backends[i])) {
known_algorithm = true;
sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
"[%s] ", backends[i]);
} else {
sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
"%s ", backends[i]);
}
}
/*
* Out-of-tree module known to crypto api or a missing
* entry in `backends'.
*/
if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1)
sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
"[%s] ", comp);
sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n");
return sz;
}
struct zcomp_strm *zcomp_stream_get(struct zcomp *comp)
{
local_lock(&comp->stream->lock);
return this_cpu_ptr(comp->stream);
}
void zcomp_stream_put(struct zcomp *comp)
{
local_unlock(&comp->stream->lock);
}
int zcomp_compress(struct zcomp_strm *zstrm,
const void *src, unsigned int *dst_len)
{
/*
* Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized
* because sometimes we can endup having a bigger compressed data
* due to various reasons: for example compression algorithms tend
* to add some padding to the compressed buffer. Speaking of padding,
* comp algorithm `842' pads the compressed length to multiple of 8
* and returns -ENOSP when the dst memory is not big enough, which
* is not something that ZRAM wants to see. We can handle the
* `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we
* receive -ERRNO from the compressing backend we can't help it
* anymore. To make `842' happy we need to tell the exact size of
* the dst buffer, zram_drv will take care of the fact that
* compressed buffer is too big.
*/
*dst_len = PAGE_SIZE * 2;
return crypto_comp_compress(zstrm->tfm,
src, PAGE_SIZE,
zstrm->buffer, dst_len);
}
int zcomp_decompress(struct zcomp_strm *zstrm,
const void *src, unsigned int src_len, void *dst)
{
unsigned int dst_len = PAGE_SIZE;
return crypto_comp_decompress(zstrm->tfm,
src, src_len,
dst, &dst_len);
}
int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)
{
struct zcomp *comp = hlist_entry(node, struct zcomp, node);
struct zcomp_strm *zstrm;
int ret;
zstrm = per_cpu_ptr(comp->stream, cpu);
local_lock_init(&zstrm->lock);
ret = zcomp_strm_init(zstrm, comp);
if (ret)
pr_err("Can't allocate a compression stream\n");
return ret;
}
int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node)
{
struct zcomp *comp = hlist_entry(node, struct zcomp, node);
struct zcomp_strm *zstrm;
zstrm = per_cpu_ptr(comp->stream, cpu);
zcomp_strm_free(zstrm);
return 0;
}
static int zcomp_init(struct zcomp *comp)
{
int ret;
comp->stream = alloc_percpu(struct zcomp_strm);
if (!comp->stream)
return -ENOMEM;
ret = cpuhp_state_add_instance(CPUHP_ZCOMP_PREPARE, &comp->node);
if (ret < 0)
goto cleanup;
return 0;
cleanup:
free_percpu(comp->stream);
return ret;
}
void zcomp_destroy(struct zcomp *comp)
{
cpuhp_state_remove_instance(CPUHP_ZCOMP_PREPARE, &comp->node);
free_percpu(comp->stream);
kfree(comp);
}
/*
* search available compressors for requested algorithm.
* allocate new zcomp and initialize it. return compressing
* backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL)
* if requested algorithm is not supported, ERR_PTR(-ENOMEM) in
* case of allocation error, or any other error potentially
* returned by zcomp_init().
*/
struct zcomp *zcomp_create(const char *compress)
{
struct zcomp *comp;
int error;
if (!zcomp_available_algorithm(compress))
return ERR_PTR(-EINVAL);
comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL);
if (!comp)
return ERR_PTR(-ENOMEM);
comp->name = compress;
error = zcomp_init(comp);
if (error) {
kfree(comp);
return ERR_PTR(error);
}
return comp;
}

43
drivers/moto_swap/zram-5.10/zcomp.h Normal file
View file

@ -0,0 +1,43 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Copyright (C) 2014 Sergey Senozhatsky.
*/
#ifndef _ZCOMP_H_
#define _ZCOMP_H_
#include <linux/local_lock.h>
struct zcomp_strm {
/* The members ->buffer and ->tfm are protected by ->lock. */
local_lock_t lock;
/* compression/decompression buffer */
void *buffer;
struct crypto_comp *tfm;
};
/* dynamic per-device compression frontend */
struct zcomp {
struct zcomp_strm __percpu *stream;
const char *name;
struct hlist_node node;
};
int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node);
int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node);
ssize_t zcomp_available_show(const char *comp, char *buf);
bool zcomp_available_algorithm(const char *comp);
struct zcomp *zcomp_create(const char *comp);
void zcomp_destroy(struct zcomp *comp);
struct zcomp_strm *zcomp_stream_get(struct zcomp *comp);
void zcomp_stream_put(struct zcomp *comp);
int zcomp_compress(struct zcomp_strm *zstrm,
const void *src, unsigned int *dst_len);
int zcomp_decompress(struct zcomp_strm *zstrm,
const void *src, unsigned int src_len, void *dst);
bool zcomp_set_max_streams(struct zcomp *comp, int num_strm);
#endif /* _ZCOMP_H_ */

2339
drivers/moto_swap/zram-5.10/zram_drv.c Normal file
View file

File diff suppressed because it is too large Load diff

150
drivers/moto_swap/zram-5.10/zram_drv.h Normal file
View file

@ -0,0 +1,150 @@
/*
* Compressed RAM block device
*
* Copyright (C) 2008, 2009, 2010 Nitin Gupta
* 2012, 2013 Minchan Kim
*
* This code is released using a dual license strategy: BSD/GPL
* You can choose the licence that better fits your requirements.
*
* Released under the terms of 3-clause BSD License
* Released under the terms of GNU General Public License Version 2.0
*
*/
#ifndef _ZRAM_DRV_H_
#define _ZRAM_DRV_H_
#include <linux/rwsem.h>
#include <linux/zsmalloc.h>
#include <linux/crypto.h>
#include "zcomp.h"
#define SECTORS_PER_PAGE_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
#define SECTORS_PER_PAGE (1 << SECTORS_PER_PAGE_SHIFT)
#define ZRAM_LOGICAL_BLOCK_SHIFT 12
#define ZRAM_LOGICAL_BLOCK_SIZE (1 << ZRAM_LOGICAL_BLOCK_SHIFT)
#define ZRAM_SECTOR_PER_LOGICAL_BLOCK \
(1 << (ZRAM_LOGICAL_BLOCK_SHIFT - SECTOR_SHIFT))
/*
* The lower ZRAM_FLAG_SHIFT bits of table.flags is for
* object size (excluding header), the higher bits is for
* zram_pageflags.
*
* zram is mainly used for memory efficiency so we want to keep memory
* footprint small so we can squeeze size and flags into a field.
* The lower ZRAM_FLAG_SHIFT bits is for object size (excluding header),
* the higher bits is for zram_pageflags.
*/
#define ZRAM_FLAG_SHIFT 24
/* Flags for zram pages (table[page_no].flags) */
enum zram_pageflags {
/* zram slot is locked */
ZRAM_LOCK = ZRAM_FLAG_SHIFT,
ZRAM_SAME, /* Page consists the same element */
ZRAM_WB, /* page is stored on backing_device */
ZRAM_UNDER_WB, /* page is under writeback */
ZRAM_HUGE, /* Incompressible page */
ZRAM_IDLE, /* not accessed page since last idle marking */
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
ZRAM_CACHED, /* page is cached in async compress cache buffer */
ZRAM_CACHED_COMPRESS, /* page is under async compress */
#endif
#ifdef CONFIG_HYBRIDSWAP_CORE
ZRAM_BATCHING_OUT,
ZRAM_FROM_HYBRIDSWAP,
ZRAM_MCGID_CLEAR,
ZRAM_IN_BD, /* zram stored in back device */
#endif
__NR_ZRAM_PAGEFLAGS,
};
/*-- Data structures */
/* Allocated for each disk page */
struct zram_table_entry {
union {
unsigned long handle;
unsigned long element;
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
unsigned long page;
#endif
};
unsigned long flags;
#ifdef CONFIG_HYBRIDSWAP_ZRAM_MEMORY_TRACKING
ktime_t ac_time;
#endif
};
struct zram_stats {
atomic64_t compr_data_size; /* compressed size of pages stored */
atomic64_t num_reads; /* failed + successful */
atomic64_t num_writes; /* --do-- */
atomic64_t failed_reads; /* can happen when memory is too low */
atomic64_t failed_writes; /* can happen when memory is too low */
atomic64_t invalid_io; /* non-page-aligned I/O requests */
atomic64_t notify_free; /* no. of swap slot free notifications */
atomic64_t same_pages; /* no. of same element filled pages */
atomic64_t huge_pages; /* no. of huge pages */
atomic64_t pages_stored; /* no. of pages currently stored */
atomic_long_t max_used_pages; /* no. of maximum pages stored */
atomic64_t writestall; /* no. of write slow paths */
atomic64_t miss_free; /* no. of missed free */
#ifdef CONFIG_HYBRIDSWAP_ZRAM_WRITEBACK
atomic64_t bd_count; /* no. of pages in backing device */
atomic64_t bd_reads; /* no. of reads from backing device */
atomic64_t bd_writes; /* no. of writes from backing device */
#endif
};
struct zram {
struct zram_table_entry *table;
struct zs_pool *mem_pool;
struct zcomp *comp;
struct gendisk *disk;
/* Prevent concurrent execution of device init */
struct rw_semaphore init_lock;
/*
* the number of pages zram can consume for storing compressed data
*/
unsigned long limit_pages;
struct zram_stats stats;
/*
* This is the limit on amount of *uncompressed* worth of data
* we can store in a disk.
*/
u64 disksize; /* bytes */
char compressor[CRYPTO_MAX_ALG_NAME];
/*
* zram is claimed so open request will be failed
*/
bool claim; /* Protected by bdev->bd_mutex */
struct file *backing_dev;
#ifdef CONFIG_HYBRIDSWAP_ZRAM_WRITEBACK
spinlock_t wb_limit_lock;
bool wb_limit_enable;
u64 bd_wb_limit;
struct block_device *bdev;
unsigned int old_block_size;
unsigned long *bitmap;
unsigned long nr_pages;
#endif
#ifdef CONFIG_HYBRIDSWAP_ZRAM_MEMORY_TRACKING
struct dentry *debugfs_dir;
#endif
#if (defined CONFIG_HYBRIDSWAP_ZRAM_WRITEBACK) || (defined CONFIG_HYBRIDSWAP_CORE)
struct block_device *bdev;
unsigned int old_block_size;
unsigned long nr_pages;
unsigned long increase_nr_pages;
#endif
#ifdef CONFIG_HYBRIDSWAP_CORE
struct hyb_info *infos;
#endif
};
#endif

39
drivers/moto_swap/zram-5.10/zram_drv_internal.h Normal file
View file

@ -0,0 +1,39 @@
#ifndef _ZRAM_DRV_INTERNAL_H_
#define _ZRAM_DRV_INTERNAL_H_
#ifdef BIT
#undef BIT
#define BIT(nr) (1lu << (nr))
#endif
#define zram_slot_lock(zram, index) (bit_spin_lock(ZRAM_LOCK, &zram->table[index].flags))
#define zram_slot_unlock(zram, index) (bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags))
#define init_done(zram) (zram->disksize)
#define dev_to_zram(dev) ((struct zram *)dev_to_disk(dev)->private_data)
#define zram_get_handle(zram, index) (zram->table[index].handle)
#define zram_set_handle(zram, index, handle_val) (zram->table[index].handle = handle_val)
#define zram_test_flag(zram, index, flag) (zram->table[index].flags & BIT(flag))
#define zram_set_flag(zram, index, flag) (zram->table[index].flags |= BIT(flag))
#define zram_clear_flag(zram, index, flag) (zram->table[index].flags &= ~BIT(flag))
#define zram_set_element(zram, index, element) (zram->table[index].element = element)
#define zram_get_obj_size(zram, index) (zram->table[index].flags & (BIT(ZRAM_FLAG_SHIFT) - 1))
#define zram_set_obj_size(zram, index, size) do {\
unsigned long flags = zram->table[index].flags >> ZRAM_FLAG_SHIFT; \
zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size; \
} while(0)
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
extern int async_compress_page(struct zram *zram, struct page* page);
extern void update_zram_index(struct zram *zram, u32 index, unsigned long page);
#endif
#endif

239
drivers/moto_swap/zram-5.4/zcomp.c Normal file
View file

@ -0,0 +1,239 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2014 Sergey Senozhatsky.
*/
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <linux/cpu.h>
#include <linux/crypto.h>
#include "zcomp.h"
static const char * const backends[] = {
"lzo",
"lzo-rle",
#if IS_ENABLED(CONFIG_CRYPTO_LZ4)
"lz4",
#endif
#if IS_ENABLED(CONFIG_CRYPTO_LZ4HC)
"lz4hc",
#endif
#if IS_ENABLED(CONFIG_CRYPTO_842)
"842",
#endif
#if IS_ENABLED(CONFIG_CRYPTO_ZSTD)
"zstd",
#endif
NULL
};
static void zcomp_strm_free(struct zcomp_strm *zstrm)
{
if (!IS_ERR_OR_NULL(zstrm->tfm))
crypto_free_comp(zstrm->tfm);
free_pages((unsigned long)zstrm->buffer, 1);
kfree(zstrm);
}
/*
* allocate new zcomp_strm structure with ->tfm initialized by
* backend, return NULL on error
*/
static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp)
{
struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL);
if (!zstrm)
return NULL;
zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0);
/*
* allocate 2 pages. 1 for compressed data, plus 1 extra for the
* case when compressed size is larger than the original one
*/
zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) {
zcomp_strm_free(zstrm);
zstrm = NULL;
}
return zstrm;
}
bool zcomp_available_algorithm(const char *comp)
{
int i;
i = __sysfs_match_string(backends, -1, comp);
if (i >= 0)
return true;
/*
* Crypto does not ignore a trailing new line symbol,
* so make sure you don't supply a string containing
* one.
* This also means that we permit zcomp initialisation
* with any compressing algorithm known to crypto api.
*/
return crypto_has_comp(comp, 0, 0) == 1;
}
/* show available compressors */
ssize_t zcomp_available_show(const char *comp, char *buf)
{
bool known_algorithm = false;
ssize_t sz = 0;
int i = 0;
for (; backends[i]; i++) {
if (!strcmp(comp, backends[i])) {
known_algorithm = true;
sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
"[%s] ", backends[i]);
} else {
sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
"%s ", backends[i]);
}
}
/*
* Out-of-tree module known to crypto api or a missing
* entry in `backends'.
*/
if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1)
sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
"[%s] ", comp);
sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n");
return sz;
}
struct zcomp_strm *zcomp_stream_get(struct zcomp *comp)
{
return *get_cpu_ptr(comp->stream);
}
void zcomp_stream_put(struct zcomp *comp)
{
put_cpu_ptr(comp->stream);
}
int zcomp_compress(struct zcomp_strm *zstrm,
const void *src, unsigned int *dst_len)
{
/*
* Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized
* because sometimes we can endup having a bigger compressed data
* due to various reasons: for example compression algorithms tend
* to add some padding to the compressed buffer. Speaking of padding,
* comp algorithm `842' pads the compressed length to multiple of 8
* and returns -ENOSP when the dst memory is not big enough, which
* is not something that ZRAM wants to see. We can handle the
* `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we
* receive -ERRNO from the compressing backend we can't help it
* anymore. To make `842' happy we need to tell the exact size of
* the dst buffer, zram_drv will take care of the fact that
* compressed buffer is too big.
*/
*dst_len = PAGE_SIZE * 2;
return crypto_comp_compress(zstrm->tfm,
src, PAGE_SIZE,
zstrm->buffer, dst_len);
}
int zcomp_decompress(struct zcomp_strm *zstrm,
const void *src, unsigned int src_len, void *dst)
{
unsigned int dst_len = PAGE_SIZE;
return crypto_comp_decompress(zstrm->tfm,
src, src_len,
dst, &dst_len);
}
int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)
{
struct zcomp *comp = hlist_entry(node, struct zcomp, node);
struct zcomp_strm *zstrm;
if (WARN_ON(*per_cpu_ptr(comp->stream, cpu)))
return 0;
zstrm = zcomp_strm_alloc(comp);
if (IS_ERR_OR_NULL(zstrm)) {
pr_err("Can't allocate a compression stream\n");
return -ENOMEM;
}
*per_cpu_ptr(comp->stream, cpu) = zstrm;
return 0;
}
int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node)
{
struct zcomp *comp = hlist_entry(node, struct zcomp, node);
struct zcomp_strm *zstrm;
zstrm = *per_cpu_ptr(comp->stream, cpu);
if (!IS_ERR_OR_NULL(zstrm))
zcomp_strm_free(zstrm);
*per_cpu_ptr(comp->stream, cpu) = NULL;
return 0;
}
static int zcomp_init(struct zcomp *comp)
{
int ret;
comp->stream = alloc_percpu(struct zcomp_strm *);
if (!comp->stream)
return -ENOMEM;
ret = cpuhp_state_add_instance(CPUHP_ZCOMP_PREPARE, &comp->node);
if (ret < 0)
goto cleanup;
return 0;
cleanup:
free_percpu(comp->stream);
return ret;
}
void zcomp_destroy(struct zcomp *comp)
{
cpuhp_state_remove_instance(CPUHP_ZCOMP_PREPARE, &comp->node);
free_percpu(comp->stream);
kfree(comp);
}
/*
* search available compressors for requested algorithm.
* allocate new zcomp and initialize it. return compressing
* backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL)
* if requested algorithm is not supported, ERR_PTR(-ENOMEM) in
* case of allocation error, or any other error potentially
* returned by zcomp_init().
*/
struct zcomp *zcomp_create(const char *compress)
{
struct zcomp *comp;
int error;
if (!zcomp_available_algorithm(compress))
return ERR_PTR(-EINVAL);
comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL);
if (!comp)
return ERR_PTR(-ENOMEM);
comp->name = compress;
error = zcomp_init(comp);
if (error) {
kfree(comp);
return ERR_PTR(error);
}
return comp;
}

40
drivers/moto_swap/zram-5.4/zcomp.h Normal file
View file

@ -0,0 +1,40 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Copyright (C) 2014 Sergey Senozhatsky.
*/
#ifndef _ZCOMP_H_
#define _ZCOMP_H_
struct zcomp_strm {
/* compression/decompression buffer */
void *buffer;
struct crypto_comp *tfm;
};
/* dynamic per-device compression frontend */
struct zcomp {
struct zcomp_strm * __percpu *stream;
const char *name;
struct hlist_node node;
};
int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node);
int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node);
ssize_t zcomp_available_show(const char *comp, char *buf);
bool zcomp_available_algorithm(const char *comp);
struct zcomp *zcomp_create(const char *comp);
void zcomp_destroy(struct zcomp *comp);
struct zcomp_strm *zcomp_stream_get(struct zcomp *comp);
void zcomp_stream_put(struct zcomp *comp);
int zcomp_compress(struct zcomp_strm *zstrm,
const void *src, unsigned int *dst_len);
int zcomp_decompress(struct zcomp_strm *zstrm,
const void *src, unsigned int src_len, void *dst);
bool zcomp_set_max_streams(struct zcomp *comp, int num_strm);
#endif /* _ZCOMP_H_ */

2333
drivers/moto_swap/zram-5.4/zram_drv.c Normal file
View file

File diff suppressed because it is too large Load diff

150
drivers/moto_swap/zram-5.4/zram_drv.h Normal file
View file

@ -0,0 +1,150 @@
/*
* Compressed RAM block device
*
* Copyright (C) 2008, 2009, 2010 Nitin Gupta
* 2012, 2013 Minchan Kim
*
* This code is released using a dual license strategy: BSD/GPL
* You can choose the licence that better fits your requirements.
*
* Released under the terms of 3-clause BSD License
* Released under the terms of GNU General Public License Version 2.0
*
*/
#ifndef _ZRAM_DRV_H_
#define _ZRAM_DRV_H_
#include <linux/rwsem.h>
#include <linux/zsmalloc.h>
#include <linux/crypto.h>
#include "zcomp.h"
#define SECTORS_PER_PAGE_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
#define SECTORS_PER_PAGE (1 << SECTORS_PER_PAGE_SHIFT)
#define ZRAM_LOGICAL_BLOCK_SHIFT 12
#define ZRAM_LOGICAL_BLOCK_SIZE (1 << ZRAM_LOGICAL_BLOCK_SHIFT)
#define ZRAM_SECTOR_PER_LOGICAL_BLOCK \
(1 << (ZRAM_LOGICAL_BLOCK_SHIFT - SECTOR_SHIFT))
/*
* The lower ZRAM_FLAG_SHIFT bits of table.flags is for
* object size (excluding header), the higher bits is for
* zram_pageflags.
*
* zram is mainly used for memory efficiency so we want to keep memory
* footprint small so we can squeeze size and flags into a field.
* The lower ZRAM_FLAG_SHIFT bits is for object size (excluding header),
* the higher bits is for zram_pageflags.
*/
#define ZRAM_FLAG_SHIFT 24
/* Flags for zram pages (table[page_no].flags) */
enum zram_pageflags {
/* zram slot is locked */
ZRAM_LOCK = ZRAM_FLAG_SHIFT,
ZRAM_SAME, /* Page consists the same element */
ZRAM_WB, /* page is stored on backing_device */
ZRAM_UNDER_WB, /* page is under writeback */
ZRAM_HUGE, /* Incompressible page */
ZRAM_IDLE, /* not accessed page since last idle marking */
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
ZRAM_CACHED, /* page is cached in async compress cache buffer */
ZRAM_CACHED_COMPRESS, /* page is under async compress */
#endif
#ifdef CONFIG_HYBRIDSWAP_CORE
ZRAM_BATCHING_OUT,
ZRAM_FROM_HYBRIDSWAP,
ZRAM_MCGID_CLEAR,
ZRAM_IN_BD, /* zram stored in back device */
#endif
__NR_ZRAM_PAGEFLAGS,
};
/*-- Data structures */
/* Allocated for each disk page */
struct zram_table_entry {
union {
unsigned long handle;
unsigned long element;
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
unsigned long page;
#endif
};
unsigned long flags;
#ifdef CONFIG_HYBRIDSWAP_ZRAM_MEMORY_TRACKING
ktime_t ac_time;
#endif
};
struct zram_stats {
atomic64_t compr_data_size; /* compressed size of pages stored */
atomic64_t num_reads; /* failed + successful */
atomic64_t num_writes; /* --do-- */
atomic64_t failed_reads; /* can happen when memory is too low */
atomic64_t failed_writes; /* can happen when memory is too low */
atomic64_t invalid_io; /* non-page-aligned I/O requests */
atomic64_t notify_free; /* no. of swap slot free notifications */
atomic64_t same_pages; /* no. of same element filled pages */
atomic64_t huge_pages; /* no. of huge pages */
atomic64_t pages_stored; /* no. of pages currently stored */
atomic_long_t max_used_pages; /* no. of maximum pages stored */
atomic64_t writestall; /* no. of write slow paths */
atomic64_t miss_free; /* no. of missed free */
#ifdef CONFIG_HYBRIDSWAP_ZRAM_WRITEBACK
atomic64_t bd_count; /* no. of pages in backing device */
atomic64_t bd_reads; /* no. of reads from backing device */
atomic64_t bd_writes; /* no. of writes from backing device */
#endif
};
struct zram {
struct zram_table_entry *table;
struct zs_pool *mem_pool;
struct zcomp *comp;
struct gendisk *disk;
/* Prevent concurrent execution of device init */
struct rw_semaphore init_lock;
/*
* the number of pages zram can consume for storing compressed data
*/
unsigned long limit_pages;
struct zram_stats stats;
/*
* This is the limit on amount of *uncompressed* worth of data
* we can store in a disk.
*/
u64 disksize; /* bytes */
char compressor[CRYPTO_MAX_ALG_NAME];
/*
* zram is claimed so open request will be failed
*/
bool claim; /* Protected by bdev->bd_mutex */
struct file *backing_dev;
#ifdef CONFIG_HYBRIDSWAP_ZRAM_WRITEBACK
spinlock_t wb_limit_lock;
bool wb_limit_enable;
u64 bd_wb_limit;
struct block_device *bdev;
unsigned int old_block_size;
unsigned long *bitmap;
unsigned long nr_pages;
#endif
#ifdef CONFIG_HYBRIDSWAP_ZRAM_MEMORY_TRACKING
struct dentry *debugfs_dir;
#endif
#if (defined CONFIG_HYBRIDSWAP_ZRAM_WRITEBACK) || (defined CONFIG_HYBRIDSWAP_CORE)
struct block_device *bdev;
unsigned int old_block_size;
unsigned long nr_pages;
unsigned long increase_nr_pages;
#endif
#ifdef CONFIG_HYBRIDSWAP_CORE
struct hyb_info *infos;
#endif
};
#endif

39
drivers/moto_swap/zram-5.4/zram_drv_internal.h Normal file
View file

@ -0,0 +1,39 @@
#ifndef _ZRAM_DRV_INTERNAL_H_
#define _ZRAM_DRV_INTERNAL_H_
#ifdef BIT
#undef BIT
#define BIT(nr) (1lu << (nr))
#endif
#define zram_slot_lock(zram, index) (bit_spin_lock(ZRAM_LOCK, &zram->table[index].flags))
#define zram_slot_unlock(zram, index) (bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags))
#define init_done(zram) (zram->disksize)
#define dev_to_zram(dev) ((struct zram *)dev_to_disk(dev)->private_data)
#define zram_get_handle(zram, index) (zram->table[index].handle)
#define zram_set_handle(zram, index, handle_val) (zram->table[index].handle = handle_val)
#define zram_test_flag(zram, index, flag) (zram->table[index].flags & BIT(flag))
#define zram_set_flag(zram, index, flag) (zram->table[index].flags |= BIT(flag))
#define zram_clear_flag(zram, index, flag) (zram->table[index].flags &= ~BIT(flag))
#define zram_set_element(zram, index, element) (zram->table[index].element = element)
#define zram_get_obj_size(zram, index) (zram->table[index].flags & (BIT(ZRAM_FLAG_SHIFT) - 1))
#define zram_set_obj_size(zram, index, size) do {\
unsigned long flags = zram->table[index].flags >> ZRAM_FLAG_SHIFT; \
zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size; \
} while(0)
#ifdef CONFIG_HYBRIDSWAP_ASYNC_COMPRESS
extern int async_compress_page(struct zram *zram, struct page* page);
extern void update_zram_index(struct zram *zram, u32 index, unsigned long page);
#endif
#endif