From 1a8c6e9580d5208d9be5b56daca239ab7d14cbf1 Mon Sep 17 00:00:00 2001 From: wrh <812507909@qq.com> Date: Thu, 11 Sep 2025 16:10:35 +0800 Subject: [PATCH 1/3] =?UTF-8?q?delete=20:=20KAESnappy=20=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=B8=85=E7=90=86=EF=BC=8Clz4=E9=81=97=E7=95=99frame=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E4=BE=9D=E8=B5=96=E5=87=BD=E6=95=B0=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- KAESnappy/include/kaelz4.h | 2 - KAESnappy/src/kaelz4_adapter.c | 11 +- KAESnappy/src/kaelz4_adapter.h | 4 +- KAESnappy/src/utils/kaelz4_common.c | 6 +- KAESnappy/src/v1/kaelz4_comp.c | 161 +------ KAESnappy/src/v1/kaelz4_comp.h | 4 +- KAESnappy/src/v1/lz4frame.h | 692 ---------------------------- 7 files changed, 9 insertions(+), 871 deletions(-) delete mode 100644 KAESnappy/src/v1/lz4frame.h diff --git a/KAESnappy/include/kaelz4.h b/KAESnappy/include/kaelz4.h index 4815c36..5252393 100644 --- a/KAESnappy/include/kaelz4.h +++ b/KAESnappy/include/kaelz4.h @@ -101,8 +101,6 @@ typedef int (*sw_compress_fn)(const char* src, char* dst, int srcSize, int dstCa typedef size_t (*sw_compress_frame_fn)(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const void* preferences_ptr); int KAELZ4_compress_async(const void *src, void *dst, lz4_async_callback callback, struct kaelz4_result *result); -int KAELZ4F_compressFrame_async(const void *src, void *dst, lz4_async_callback callback, - struct kaelz4_result *result, const void *preferences_ptr); void KAELZ4_teardown_async_compress(void); int KAELZ4_async_compress_init(sw_compress_fn sw_compress, sw_compress_frame_fn sw_compress_frame); #endif \ No newline at end of file diff --git a/KAESnappy/src/kaelz4_adapter.c b/KAESnappy/src/kaelz4_adapter.c index 0477312..9ccff74 100644 --- a/KAESnappy/src/kaelz4_adapter.c +++ b/KAESnappy/src/kaelz4_adapter.c @@ -465,7 +465,7 @@ static unsigned int kaelz4_get_queue_id(void) } static int kaelz4_async_do_comp(const void *src, void *dst, lz4_async_callback callback, struct kaelz4_result *result, - enum kae_lz4_async_data_format data_format, const LZ4F_preferences_t* preferences_ptr) + enum kae_lz4_async_data_format data_format, const int* preferences_ptr) { // 初始化队列容量 if (unlikely(g_task_queues.init == 0)) { @@ -503,12 +503,3 @@ int KAELZ4_compress_async(const void *src, void *dst, lz4_async_callback callbac return kaelz4_async_do_comp(src, dst, callback, result, KAELZ4_ASYNC_BLOCK, NULL); } -int KAELZ4F_compressFrame_async(const void* src, void* dst, lz4_async_callback callback, - struct kaelz4_result *result, const void *preferences_ptr) -{ - if (unlikely(src == NULL || dst == NULL || callback == NULL || result == NULL || result->src_size == 0)) { - return KAE_LZ4_INVAL_PARA; - } - - return kaelz4_async_do_comp(src, dst, callback, result, KAELZ4_ASYNC_FRAME, preferences_ptr); -} diff --git a/KAESnappy/src/kaelz4_adapter.h b/KAESnappy/src/kaelz4_adapter.h index 1018e23..073de5c 100644 --- a/KAESnappy/src/kaelz4_adapter.h +++ b/KAESnappy/src/kaelz4_adapter.h @@ -32,7 +32,7 @@ typedef struct { lz4_async_callback callback; struct kaelz4_result *result; enum kae_lz4_async_data_format data_format; - LZ4F_preferences_t preferences; + int preferences; } lz4_async_task_t; typedef struct { @@ -62,7 +62,7 @@ void kaelz4_setstatus_v1(LZ4_CCtx* zc, unsigned int status); int kaelz4_compress_v1(LZ4_CCtx* zc, const void* src, size_t srcSize); void kaelz4_compress_async(const void *src, void *dst, lz4_async_callback callback, struct kaelz4_result *result, - enum kae_lz4_async_data_format data_format, const LZ4F_preferences_t *ptr); + enum kae_lz4_async_data_format data_format, const int *ptr); int kaelz4_async_compress_polling(int budget); int kaelz4_init_v2(LZ4_CCtx* zc); diff --git a/KAESnappy/src/utils/kaelz4_common.c b/KAESnappy/src/utils/kaelz4_common.c index e4608d1..a247fc4 100644 --- a/KAESnappy/src/utils/kaelz4_common.c +++ b/KAESnappy/src/utils/kaelz4_common.c @@ -55,7 +55,7 @@ static ARCH_TYPE KaeLz4Detect(void) return CPU_HISILICOM_V2; } else if ((vendor == 0x48) && (partId == 0xD03)) { return CPU_HISILICOM_V3; - } else if (partId == 0xD22 || partId == 0xD06) { + } else if (partId == 0xD22) { return CPU_HISILICOM_V4; } return CPU_UNKNOW; @@ -67,8 +67,8 @@ HIDDEN_API void CONSTRUCTOR KaeLz4Init(void) return; } - if (KaeLz4Detect() == CPU_HISILICOM_V1 || KaeLz4Detect() == CPU_UNKNOW) { - fprintf(stderr, "KAElz4 only support in V2+, please check CPU ID.\n"); + if (KaeLz4Detect() != CPU_HISILICOM_V2) { + fprintf(stderr, "KAElz4 only support in V2, please check CPU ID.\n"); abort(); } g_kaelz4Initialized = 1; diff --git a/KAESnappy/src/v1/kaelz4_comp.c b/KAESnappy/src/v1/kaelz4_comp.c index 330ad54..5e8f7ae 100644 --- a/KAESnappy/src/v1/kaelz4_comp.c +++ b/KAESnappy/src/v1/kaelz4_comp.c @@ -711,163 +711,6 @@ static void kaelz4_do_compress_polling(struct kaelz4_async_req *req) return; } -static void KAELZ4F_writeLE32 (void* dst, uint32_t value32) -{ - BYTE* const dstPtr = (BYTE*)dst; - dstPtr[0] = (BYTE)value32; - dstPtr[1] = (BYTE)(value32 >> 8); - dstPtr[2] = (BYTE)(value32 >> 16); - dstPtr[3] = (BYTE)(value32 >> 24); -} - -static void KAELZ4F_writeLE64 (void* dst, uint64_t value64) -{ - BYTE* const dstPtr = (BYTE*)dst; - dstPtr[0] = (BYTE)value64; - dstPtr[1] = (BYTE)(value64 >> 8); - dstPtr[2] = (BYTE)(value64 >> 16); - dstPtr[3] = (BYTE)(value64 >> 24); - dstPtr[4] = (BYTE)(value64 >> 32); - dstPtr[5] = (BYTE)(value64 >> 40); - dstPtr[6] = (BYTE)(value64 >> 48); - dstPtr[7] = (BYTE)(value64 >> 56); -} - -static int KAELZ4HeaderGen(unsigned char *dstPtr, LZ4F_frameInfo_t *frameinfo_ptr) -{ - uint32_t src_len = frameinfo_ptr->contentSize; - uint8_t flag = 0; - if (frameinfo_ptr->blockChecksumFlag == LZ4F_blockChecksumEnabled) { - flag |= KAELZ4_BLOCK_CHECKSUM_FLAG; - } - if (frameinfo_ptr->contentChecksumFlag == LZ4F_contentChecksumEnabled) { - flag |= KAELZ4_CONTENT_CHECKSUM_FLAG; - } - if (src_len) { - flag |= KAELZ4_CONTENT_SIZE_FLAG; - } - int dst_len = 0; - // MAGIC NUMBER - KAELZ4F_writeLE32(dstPtr, KAELZ4_MAGIC_NUMBER); - dstPtr += 4; - dst_len += 4; - unsigned char *headerStart = dstPtr; - // FLG - *dstPtr++ = (BYTE)(((KAELZ4_VERSION & 0x03) << 6) - | ((KAELZ4_BLOCK_INDEPENDENCE_FLAG & 0x01) << 5) - | (KAELZ4_DICTIONARY_ID_FLAG & 0x01) - | flag); - dst_len++; - // BD - *dstPtr++ = (BYTE)((KAELZ4_MAX_BLK_SIZE & 0x07) << 4); - dst_len++; - // CONTENT SIZE - if (src_len) { - KAELZ4F_writeLE64(dstPtr, src_len); - dstPtr += 8; - dst_len += 8; - } - // HEADER CRC - *dstPtr = (unsigned char)((XXH32(headerStart, (size_t)(dstPtr - headerStart), 0) >> 8) & 0xFF); - dstPtr++; - dst_len++; - return dst_len; -} - -static int KAELZ4FooterGen(unsigned char* dstPtr, unsigned char *srcPtr, uint32_t src_len, uint8_t checksumEnabled) -{ - int dst_len = 4; - // ENDMARK - KAELZ4F_writeLE32(dstPtr, 0); - dstPtr += 4; - // CHECKSUM(可选) - if (checksumEnabled) { - uint32_t xxh = XXH32(srcPtr, src_len, 0); - KAELZ4F_writeLE32(dstPtr, xxh); - dstPtr += 4; - dst_len += 4; - } - return dst_len; -} - -static int KAELZ4BlockHeaderGen(unsigned char *dstPtr, uint32_t compressed_len, - uint8_t stored_block_flag) -{ - int dst_len = 4; - if (stored_block_flag) { // 场景1. 压缩异常或负压 - KAELZ4F_writeLE32(dstPtr, compressed_len | KAELZ4_STOREDBLOCK_FLAG); - } else { - KAELZ4F_writeLE32(dstPtr, compressed_len); - } - dstPtr += 4; - return dst_len; -} - -static int KAELZ4BlockFooterGen(unsigned char *dstPtr, uint32_t compressed_len) -{ - int dst_len = 0; - uint32_t xxh = XXH32(dstPtr-compressed_len, compressed_len, 0); - KAELZ4F_writeLE32(dstPtr, xxh); - dstPtr += 4; - dst_len += 4; - return dst_len; -} - -static int kaelz4_async_frame_padding(struct kaelz4_async_req *req, const void *source, void *dst_tmp) -{ - int ret = 0; - int padding_len = 0; - void *dst_after_frameheader = dst_tmp; - LZ4F_frameInfo_t frameinfo_ptr = req->compress_ctx->preferences.frameInfo; - // 如果是第一个block块，添加frame头部 - if (req->idx == 0) { - int len1 = KAELZ4HeaderGen(dst_tmp, &frameinfo_ptr); - padding_len += len1; - dst_after_frameheader += len1; // 记录此时的位置 - dst_tmp += len1; // 真实dst空间直接进行偏移 - } - - dst_tmp += 4; // 直接往后偏移4个字节(KAELZ4BlockHeaderGen 的返回值), 预留block头的空间 - // 写入真实 block 数据 - ret = kaelz4_triples_rebuild(req, source, dst_tmp); - if (ret < 0) { - return ret; - } - uint8_t stored_block_flag = 0; - if (ret > req->src_size) { // 负压 - LZ4_memcpy(dst_tmp, source, req->src_size); - ret = req->src_size; - stored_block_flag = 1; - } - - int bloc_checksum_enabled = 0; - if (frameinfo_ptr.blockChecksumFlag == LZ4F_blockChecksumEnabled) { - bloc_checksum_enabled = 1; - } - // 使用 block 数据块的真实长度ret，在真实block数据之前写入4字节的block头 - int len2 = KAELZ4BlockHeaderGen(dst_after_frameheader, ret, stored_block_flag); - padding_len += len2; // 最终总数据量增加 - dst_tmp += ret; - - if (bloc_checksum_enabled == 1) { - // block 数据尾。如果配置有block checksum，那么就填充它，否则不填充。一般是4字节 - int len3 = KAELZ4BlockFooterGen(dst_tmp, ret); - padding_len += len3; - dst_tmp += len3; - } - - // 如果是最后一个block块，添加frame尾部 - if (req->last == 1) { - int contentChecksum = frameinfo_ptr.contentChecksumFlag; - int len4 = KAELZ4FooterGen(dst_tmp, (unsigned char *)req->compress_ctx->src, req->compress_ctx->srcSize, - contentChecksum); - padding_len += len4; - } - - ret += padding_len; // 计算本次一共生成的数据总量 - return ret; -} - int kaelz4_async_is_thread_do_comp_full(void) { return g_async_ctrl.cur_num_in_comp < MAX_NUM_IN_COMP ? 0 : 1; @@ -876,7 +719,6 @@ int kaelz4_async_is_thread_do_comp_full(void) void kaelz4_async_init(volatile int *stop, sw_compress_fn sw_compress, sw_compress_frame_fn sw_compress_frame) { g_async_ctrl.stop_flag = stop; - g_async_ctrl.sw_compress = sw_compress; g_async_ctrl.sw_compress_frame = sw_compress_frame; } @@ -1171,12 +1013,11 @@ void kaelz4_async_deinit(void) const kaelz4_post_process_handle_t g_post_process_handle[KAELZ4_ASYNC_BUTT] = { [KAELZ4_ASYNC_SMALL_BLOCK] = kaelz4_triples_rebuild, [KAELZ4_ASYNC_BLOCK] = kaelz4_triples_rebuild_64Kblock, - [KAELZ4_ASYNC_FRAME] = kaelz4_async_frame_padding, }; void kaelz4_compress_async(const void *src, void *dst, lz4_async_callback callback, struct kaelz4_result *result, - enum kae_lz4_async_data_format data_format, const LZ4F_preferences_t *ptr) + enum kae_lz4_async_data_format data_format, const int *ptr) { struct kaelz4_compress_ctx *compress_ctx = (struct kaelz4_compress_ctx *)kae_malloc(sizeof(struct kaelz4_compress_ctx)); if (unlikely(compress_ctx == NULL)) { diff --git a/KAESnappy/src/v1/kaelz4_comp.h b/KAESnappy/src/v1/kaelz4_comp.h index 6b3bcad..19b1736 100644 --- a/KAESnappy/src/v1/kaelz4_comp.h +++ b/KAESnappy/src/v1/kaelz4_comp.h @@ -13,7 +13,7 @@ #include #include #include -#include + #define TOKEN_NUM_CONTROL 0 // 用于控制生成压缩块中三元组数目，来保证解压速度（对齐实际match length需+3） #define ML_BITS 4 @@ -60,7 +60,7 @@ struct kaelz4_compress_ctx { lz4_async_callback callback; struct kaelz4_result *result; enum kae_lz4_async_data_format data_format; - LZ4F_preferences_t preferences; + int preferences; kaelz4_post_process_handle_t kaelz4_post_process_handle; struct kaelz4_async_req *req_list; struct kaelz4_compress_ctx *next; diff --git a/KAESnappy/src/v1/lz4frame.h b/KAESnappy/src/v1/lz4frame.h deleted file mode 100644 index 1bdf6c4..0000000 --- a/KAESnappy/src/v1/lz4frame.h +++ /dev/null @@ -1,692 +0,0 @@ -/* - LZ4F - LZ4-Frame library - Header File - Copyright (C) 2011-2020, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/* LZ4F is a stand-alone API able to create and decode LZ4 frames - * conformant with specification v1.6.1 in doc/lz4_Frame_format.md . - * Generated frames are compatible with `lz4` CLI. - * - * LZ4F also offers streaming capabilities. - * - * lz4.h is not required when using lz4frame.h, - * except to extract common constants such as LZ4_VERSION_NUMBER. - * */ - -#ifndef LZ4F_H_09782039843 -#define LZ4F_H_09782039843 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* --- Dependency --- */ -#include /* size_t */ - - -/** - * Introduction - * - * lz4frame.h implements LZ4 frame specification: see doc/lz4_Frame_format.md . - * LZ4 Frames are compatible with `lz4` CLI, - * and designed to be interoperable with any system. -**/ - -/*-*************************************************************** - * Compiler specifics - *****************************************************************/ -/* LZ4_DLL_EXPORT : - * Enable exporting of functions when building a Windows DLL - * LZ4FLIB_VISIBILITY : - * Control library symbols visibility. - */ -#ifndef LZ4FLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define LZ4FLIB_VISIBILITY __attribute__ ((visibility ("default"))) -# else -# define LZ4FLIB_VISIBILITY -# endif -#endif -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4FLIB_API __declspec(dllexport) LZ4FLIB_VISIBILITY -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4FLIB_API __declspec(dllimport) LZ4FLIB_VISIBILITY -#else -# define LZ4FLIB_API LZ4FLIB_VISIBILITY -#endif - -#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS -# define LZ4F_DEPRECATE(x) x -#else -# if defined(_MSC_VER) -# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */ -# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6)) -# define LZ4F_DEPRECATE(x) x __attribute__((deprecated)) -# else -# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */ -# endif -#endif - - -/*-************************************ - * Error management - **************************************/ -typedef size_t LZ4F_errorCode_t; - -LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells when a function result is an error code */ -LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; for debugging */ - - -/*-************************************ - * Frame compression types - ************************************* */ -/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */ -#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS -# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x -#else -# define LZ4F_OBSOLETE_ENUM(x) -#endif - -/* The larger the block size, the (slightly) better the compression ratio, - * though there are diminishing returns. - * Larger blocks also increase memory usage on both compression and decompression sides. - */ -typedef enum { - LZ4F_default=0, - LZ4F_max64KB=4, - LZ4F_max256KB=5, - LZ4F_max1MB=6, - LZ4F_max4MB=7 - LZ4F_OBSOLETE_ENUM(max64KB) - LZ4F_OBSOLETE_ENUM(max256KB) - LZ4F_OBSOLETE_ENUM(max1MB) - LZ4F_OBSOLETE_ENUM(max4MB) -} LZ4F_blockSizeID_t; - -/* Linked blocks sharply reduce inefficiencies when using small blocks, - * they compress better. - * However, some LZ4 decoders are only compatible with independent blocks */ -typedef enum { - LZ4F_blockLinked=0, - LZ4F_blockIndependent - LZ4F_OBSOLETE_ENUM(blockLinked) - LZ4F_OBSOLETE_ENUM(blockIndependent) -} LZ4F_blockMode_t; - -typedef enum { - LZ4F_noContentChecksum=0, - LZ4F_contentChecksumEnabled - LZ4F_OBSOLETE_ENUM(noContentChecksum) - LZ4F_OBSOLETE_ENUM(contentChecksumEnabled) -} LZ4F_contentChecksum_t; - -typedef enum { - LZ4F_noBlockChecksum=0, - LZ4F_blockChecksumEnabled -} LZ4F_blockChecksum_t; - -typedef enum { - LZ4F_frame=0, - LZ4F_skippableFrame - LZ4F_OBSOLETE_ENUM(skippableFrame) -} LZ4F_frameType_t; - -#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS -typedef LZ4F_blockSizeID_t blockSizeID_t; -typedef LZ4F_blockMode_t blockMode_t; -typedef LZ4F_frameType_t frameType_t; -typedef LZ4F_contentChecksum_t contentChecksum_t; -#endif - -/*! LZ4F_frameInfo_t : - * makes it possible to set or read frame parameters. - * Structure must be first init to 0, using memset() or LZ4F_INIT_FRAMEINFO, - * setting all parameters to default. - * It's then possible to update selectively some parameters */ -typedef struct { - LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default */ - LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */ - LZ4F_contentChecksum_t contentChecksumFlag; /* 1: frame terminated with 32-bit checksum of decompressed data; 0: disabled (default) */ - LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */ - unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */ - unsigned dictID; /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */ - LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */ -} LZ4F_frameInfo_t; - -#define LZ4F_INIT_FRAMEINFO { LZ4F_default, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0ULL, 0U, LZ4F_noBlockChecksum } /* v1.8.3+ */ - -/*! LZ4F_preferences_t : - * makes it possible to supply advanced compression instructions to streaming interface. - * Structure must be first init to 0, using memset() or LZ4F_INIT_PREFERENCES, - * setting all parameters to default. - * All reserved fields must be set to zero. */ -typedef struct { - LZ4F_frameInfo_t frameInfo; - int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */ - unsigned autoFlush; /* 1: always flush; reduces usage of internal buffers */ - unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4HC_CLEVEL_OPT_MIN) */ /* v1.8.2+ */ - unsigned reserved[3]; /* must be zero for forward compatibility */ -} LZ4F_preferences_t; - -#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0u, 0u, { 0u, 0u, 0u } } /* v1.8.3+ */ - - -/*-********************************* -* Simple compression function -***********************************/ - -LZ4FLIB_API int LZ4F_compressionLevel_max(void); /* v1.8.0+ */ - -/*! LZ4F_compressFrameBound() : - * Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences. - * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences. - * Note : this result is only usable with LZ4F_compressFrame(). - * It may also be relevant to LZ4F_compressUpdate() _only if_ no flush() operation is ever performed. - */ -LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); - -/*! LZ4F_compressFrame() : - * Compress an entire srcBuffer into a valid LZ4 frame. - * dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). - * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. - * @return : number of bytes written into dstBuffer. - * or an error code if it fails (can be tested using LZ4F_isError()) - */ -LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, - const void* srcBuffer, size_t srcSize, - const LZ4F_preferences_t* preferencesPtr); - - -/*-*********************************** -* Advanced compression functions -*************************************/ -typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */ -typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with older APIs, prefer using LZ4F_cctx */ - -typedef struct { - unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */ - unsigned reserved[3]; -} LZ4F_compressOptions_t; - -/*--- Resource Management ---*/ - -#define LZ4F_VERSION 100 /* This number can be used to check for an incompatible API breaking change */ -LZ4FLIB_API unsigned LZ4F_getVersion(void); - -/*! LZ4F_createCompressionContext() : - * The first thing to do is to create a compressionContext object, - * which will keep track of operation state during streaming compression. - * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version, - * and a pointer to LZ4F_cctx*, to write the resulting pointer into. - * @version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. - * The function provides a pointer to a fully allocated LZ4F_cctx object. - * @cctxPtr MUST be != NULL. - * If @return != zero, context creation failed. - * A created compression context can be employed multiple times for consecutive streaming operations. - * Once all streaming compression jobs are completed, - * the state object can be released using LZ4F_freeCompressionContext(). - * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored. - * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing). -**/ -LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); -LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); - - -/*---- Compression ----*/ - -#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected parameters */ -#define LZ4F_HEADER_SIZE_MAX 19 - -/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */ -#define LZ4F_BLOCK_HEADER_SIZE 4 - -/* Size in bytes of a block checksum footer in little-endian format. */ -#define LZ4F_BLOCK_CHECKSUM_SIZE 4 - -/* Size in bytes of the content checksum. */ -#define LZ4F_CONTENT_CHECKSUM_SIZE 4 - -/*! LZ4F_compressBegin() : - * will write the frame header into dstBuffer. - * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. - * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. - * @return : number of bytes written into dstBuffer for the header - * or an error code (which can be tested using LZ4F_isError()) - */ -LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_preferences_t* prefsPtr); - -/*! LZ4F_compressBound() : - * Provides minimum dstCapacity required to guarantee success of - * LZ4F_compressUpdate(), given a srcSize and preferences, for a worst case scenario. - * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() instead. - * Note that the result is only valid for a single invocation of LZ4F_compressUpdate(). - * When invoking LZ4F_compressUpdate() multiple times, - * if the output buffer is gradually filled up instead of emptied and re-used from its start, - * one must check if there is enough remaining capacity before each invocation, using LZ4F_compressBound(). - * @return is always the same for a srcSize and prefsPtr. - * prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. - * tech details : - * @return if automatic flushing is not enabled, includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. - * It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd(). - * @return doesn't include frame header, as it was already generated by LZ4F_compressBegin(). - */ -LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); - -/*! LZ4F_compressUpdate() : - * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. - * Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations. - * This value is provided by LZ4F_compressBound(). - * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). - * After an error, the state is left in a UB state, and must be re-initialized or freed. - * If previously an uncompressed block was written, buffered data is flushed - * before appending compressed data is continued. - * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. - * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). - * or an error code if it fails (which can be tested using LZ4F_isError()) - */ -LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const void* srcBuffer, size_t srcSize, - const LZ4F_compressOptions_t* cOptPtr); - -/*! LZ4F_flush() : - * When data must be generated and sent immediately, without waiting for a block to be completely filled, - * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. - * `dstCapacity` must be large enough to ensure the operation will be successful. - * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. - * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx) - * or an error code if it fails (which can be tested using LZ4F_isError()) - * Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). - */ -LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_compressOptions_t* cOptPtr); - -/*! LZ4F_compressEnd() : - * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). - * It will flush whatever data remained within `cctx` (like LZ4_flush()) - * and properly finalize the frame, with an endMark and a checksum. - * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. - * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark), - * or an error code if it fails (which can be tested using LZ4F_isError()) - * Note : LZ4F_compressEnd() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). - * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. - */ -LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_compressOptions_t* cOptPtr); - - -/*-********************************* -* Decompression functions -***********************************/ -typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */ -typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */ - -typedef struct { - unsigned stableDst; /* pledges that last 64KB decompressed data will remain available unmodified between invocations. - * This optimization skips storage operations in tmp buffers. */ - unsigned skipChecksums; /* disable checksum calculation and verification, even when one is present in frame, to save CPU time. - * Setting this option to 1 once disables all checksums for the rest of the frame. */ - unsigned reserved1; /* must be set to zero for forward compatibility */ - unsigned reserved0; /* idem */ -} LZ4F_decompressOptions_t; - - -/* Resource management */ - -/*! LZ4F_createDecompressionContext() : - * Create an LZ4F_dctx object, to track all decompression operations. - * @version provided MUST be LZ4F_VERSION. - * @dctxPtr MUST be valid. - * The function fills @dctxPtr with the value of a pointer to an allocated and initialized LZ4F_dctx object. - * The @return is an errorCode, which can be tested using LZ4F_isError(). - * dctx memory can be released using LZ4F_freeDecompressionContext(); - * Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released. - * That is, it should be == 0 if decompression has been completed fully and correctly. - */ -LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); -LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); - - -/*-*********************************** -* Streaming decompression functions -*************************************/ - -#define LZ4F_MAGICNUMBER 0x184D2204U -#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U -#define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5 - -/*! LZ4F_headerSize() : v1.9.0+ - * Provide the header size of a frame starting at `src`. - * `srcSize` must be >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH, - * which is enough to decode the header length. - * @return : size of frame header - * or an error code, which can be tested using LZ4F_isError() - * note : Frame header size is variable, but is guaranteed to be - * >= LZ4F_HEADER_SIZE_MIN bytes, and <= LZ4F_HEADER_SIZE_MAX bytes. - */ -LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize); - -/*! LZ4F_getFrameInfo() : - * This function extracts frame parameters (max blockSize, dictID, etc.). - * Its usage is optional: user can also invoke LZ4F_decompress() directly. - * - * Extracted information will fill an existing LZ4F_frameInfo_t structure. - * This can be useful for allocation and dictionary identification purposes. - * - * LZ4F_getFrameInfo() can work in the following situations : - * - * 1) At the beginning of a new frame, before any invocation of LZ4F_decompress(). - * It will decode header from `srcBuffer`, - * consuming the header and starting the decoding process. - * - * Input size must be large enough to contain the full frame header. - * Frame header size can be known beforehand by LZ4F_headerSize(). - * Frame header size is variable, but is guaranteed to be >= LZ4F_HEADER_SIZE_MIN bytes, - * and not more than <= LZ4F_HEADER_SIZE_MAX bytes. - * Hence, blindly providing LZ4F_HEADER_SIZE_MAX bytes or more will always work. - * It's allowed to provide more input data than the header size, - * LZ4F_getFrameInfo() will only consume the header. - * - * If input size is not large enough, - * aka if it's smaller than header size, - * function will fail and return an error code. - * - * 2) After decoding has been started, - * it's possible to invoke LZ4F_getFrameInfo() anytime - * to extract already decoded frame parameters stored within dctx. - * - * Note that, if decoding has barely started, - * and not yet read enough information to decode the header, - * LZ4F_getFrameInfo() will fail. - * - * The number of bytes consumed from srcBuffer will be updated in *srcSizePtr (necessarily <= original value). - * LZ4F_getFrameInfo() only consumes bytes when decoding has not yet started, - * and when decoding the header has been successful. - * Decompression must then resume from (srcBuffer + *srcSizePtr). - * - * @return : a hint about how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError(). - * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. - * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. - */ -LZ4FLIB_API size_t -LZ4F_getFrameInfo(LZ4F_dctx* dctx, - LZ4F_frameInfo_t* frameInfoPtr, - const void* srcBuffer, size_t* srcSizePtr); - -/*! LZ4F_decompress() : - * Call this function repetitively to regenerate data compressed in `srcBuffer`. - * - * The function requires a valid dctx state. - * It will read up to *srcSizePtr bytes from srcBuffer, - * and decompress data into dstBuffer, of capacity *dstSizePtr. - * - * The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value). - * The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value). - * - * The function does not necessarily read all input bytes, so always check value in *srcSizePtr. - * Unconsumed source data must be presented again in subsequent invocations. - * - * `dstBuffer` can freely change between each consecutive function invocation. - * `dstBuffer` content will be overwritten. - * - * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. - * Schematically, it's the size of the current (or remaining) compressed block + header of next block. - * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. - * This is just a hint though, it's always possible to provide any srcSize. - * - * When a frame is fully decoded, @return will be 0 (no more data expected). - * When provided with more bytes than necessary to decode a frame, - * LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0. - * - * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). - * After a decompression error, the `dctx` context is not resumable. - * Use LZ4F_resetDecompressionContext() to return to clean state. - * - * After a frame is fully decoded, dctx can be used again to decompress another frame. - */ -LZ4FLIB_API size_t -LZ4F_decompress(LZ4F_dctx* dctx, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const LZ4F_decompressOptions_t* dOptPtr); - - -/*! LZ4F_resetDecompressionContext() : added in v1.8.0 - * In case of an error, the context is left in "undefined" state. - * In which case, it's necessary to reset it, before re-using it. - * This method can also be used to abruptly stop any unfinished decompression, - * and start a new one using same context resources. */ -LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */ - - - -#if defined (__cplusplus) -} -#endif - -#endif /* LZ4F_H_09782039843 */ - -#if defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) -#define LZ4F_H_STATIC_09782039843 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* These declarations are not stable and may change in the future. - * They are therefore only safe to depend on - * when the caller is statically linked against the library. - * To access their declarations, define LZ4F_STATIC_LINKING_ONLY. - * - * By default, these symbols aren't published into shared/dynamic libraries. - * You can override this behavior and force them to be published - * by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. - * Use at your own risk. - */ -#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS -# define LZ4FLIB_STATIC_API LZ4FLIB_API -#else -# define LZ4FLIB_STATIC_API -#endif - - -/* --- Error List --- */ -#define LZ4F_LIST_ERRORS(ITEM) \ - ITEM(OK_NoError) \ - ITEM(ERROR_GENERIC) \ - ITEM(ERROR_maxBlockSize_invalid) \ - ITEM(ERROR_blockMode_invalid) \ - ITEM(ERROR_contentChecksumFlag_invalid) \ - ITEM(ERROR_compressionLevel_invalid) \ - ITEM(ERROR_headerVersion_wrong) \ - ITEM(ERROR_blockChecksum_invalid) \ - ITEM(ERROR_reservedFlag_set) \ - ITEM(ERROR_allocation_failed) \ - ITEM(ERROR_srcSize_tooLarge) \ - ITEM(ERROR_dstMaxSize_tooSmall) \ - ITEM(ERROR_frameHeader_incomplete) \ - ITEM(ERROR_frameType_unknown) \ - ITEM(ERROR_frameSize_wrong) \ - ITEM(ERROR_srcPtr_wrong) \ - ITEM(ERROR_decompressionFailed) \ - ITEM(ERROR_headerChecksum_invalid) \ - ITEM(ERROR_contentChecksum_invalid) \ - ITEM(ERROR_frameDecoding_alreadyStarted) \ - ITEM(ERROR_compressionState_uninitialized) \ - ITEM(ERROR_parameter_null) \ - ITEM(ERROR_maxCode) - -#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, - -/* enum list is exposed, to handle specific errors */ -typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) - _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes; - -LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); - - -/*! LZ4F_getBlockSize() : - * Return, in scalar format (size_t), - * the maximum block size associated with blockSizeID. -**/ -LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID); - -/*! LZ4F_uncompressedUpdate() : - * LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary. - * Important rule: dstCapacity MUST be large enough to store the entire source buffer as - * no compression is done for this operation - * If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode). - * After an error, the state is left in a UB state, and must be re-initialized or freed. - * If previously a compressed block was written, buffered data is flushed - * before appending uncompressed data is continued. - * This is only supported when LZ4F_blockIndependent is used - * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. - * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). - * or an error code if it fails (which can be tested using LZ4F_isError()) - */ -LZ4FLIB_STATIC_API size_t -LZ4F_uncompressedUpdate(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const void* srcBuffer, size_t srcSize, - const LZ4F_compressOptions_t* cOptPtr); - -/********************************** - * Bulk processing dictionary API - *********************************/ - -/* A Dictionary is useful for the compression of small messages (KB range). - * It dramatically improves compression efficiency. - * - * LZ4 can ingest any input as dictionary, though only the last 64 KB are useful. - * Best results are generally achieved by using Zstandard's Dictionary Builder - * to generate a high-quality dictionary from a set of samples. - * - * Loading a dictionary has a cost, since it involves construction of tables. - * The Bulk processing dictionary API makes it possible to share this cost - * over an arbitrary number of compression jobs, even concurrently, - * markedly improving compression latency for these cases. - * - * The same dictionary will have to be used on the decompression side - * for decoding to be successful. - * To help identify the correct dictionary at decoding stage, - * the frame header allows optional embedding of a dictID field. - */ -typedef struct LZ4F_CDict_s LZ4F_CDict; - -/*! LZ4_createCDict() : - * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it just once. - * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. - * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ -LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); -LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict); - - -/*! LZ4_compressFrame_usingCDict() : - * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary. - * cctx must point to a context created by LZ4F_createCompressionContext(). - * If cdict==NULL, compress without a dictionary. - * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). - * If this condition is not respected, function will fail (@return an errorCode). - * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, - * but it's not recommended, as it's the only way to provide dictID in the frame header. - * @return : number of bytes written into dstBuffer. - * or an error code if it fails (can be tested using LZ4F_isError()) */ -LZ4FLIB_STATIC_API size_t -LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* preferencesPtr); - - -/*! LZ4F_compressBegin_usingCDict() : - * Inits streaming dictionary compression, and writes the frame header into dstBuffer. - * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. - * `prefsPtr` is optional : you may provide NULL as argument, - * however, it's the only way to provide dictID in the frame header. - * @return : number of bytes written into dstBuffer for the header, - * or an error code (which can be tested using LZ4F_isError()) */ -LZ4FLIB_STATIC_API size_t -LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, - void* dstBuffer, size_t dstCapacity, - const LZ4F_CDict* cdict, - const LZ4F_preferences_t* prefsPtr); - - -/*! LZ4F_decompress_usingDict() : - * Same as LZ4F_decompress(), using a predefined dictionary. - * Dictionary is used "in place", without any preprocessing. -** It must remain accessible throughout the entire frame decoding. */ -LZ4FLIB_STATIC_API size_t -LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, - void* dstBuffer, size_t* dstSizePtr, - const void* srcBuffer, size_t* srcSizePtr, - const void* dict, size_t dictSize, - const LZ4F_decompressOptions_t* decompressOptionsPtr); - - -/*! Custom memory allocation : - * These prototypes make it possible to pass custom allocation/free functions. - * LZ4F_customMem is provided at state creation time, using LZ4F_create*_advanced() listed below. - * All allocation/free operations will be completed using these custom variants instead of regular ones. - */ -typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size); -typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size); -typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address); -typedef struct { - LZ4F_AllocFunction customAlloc; - LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */ - LZ4F_FreeFunction customFree; - void* opaqueState; -} LZ4F_CustomMem; -static -#ifdef __GNUC__ -__attribute__((__unused__)) -#endif -LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ - -LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); -LZ4FLIB_STATIC_API LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); -LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict_advanced(LZ4F_CustomMem customMem, const void* dictBuffer, size_t dictSize); - - -#if defined (__cplusplus) -} -#endif - -#endif /* defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) */ -- Gitee From edea388009287f53a7f501217614f81281fead50 Mon Sep 17 00:00:00 2001 From: wrh <812507909@qq.com> Date: Thu, 11 Sep 2025 16:29:57 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix:=E5=A4=8D=E7=94=A8=E8=87=AAKAELZ4?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=94=B9=E5=90=8D=E4=B8=BAKAESnappy,?= =?UTF-8?q?=E6=97=A0=E5=85=B3=E6=B5=8B=E8=AF=95=E6=96=87=E4=BB=B6=E5=88=A0?= =?UTF-8?q?=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- KAESnappy/Makefile | 8 +- KAESnappy/include/{kaelz4.h => kaesnappy.h} | 4 +- .../{kaelz4_adapter.c => kaesnappy_adapter.c} | 12 +- .../{kaelz4_adapter.h => kaesnappy_adapter.h} | 9 +- .../{kaelz4_common.c => kaesnappy_common.c} | 4 +- .../{kaelz4_common.h => kaesnappy_common.h} | 6 +- .../utils/{kaelz4_log.c => kaesnappy_log.c} | 2 +- .../utils/{kaelz4_log.h => kaesnappy_log.h} | 4 +- .../{kaelz4_utils.h => kaesnappy_utils.h} | 4 +- .../v1/{kaelz4_comp.c => kaesnappy_comp.c} | 8 +- .../v1/{kaelz4_comp.h => kaesnappy_comp.h} | 8 +- .../src/v1/{kaelz4_ctx.c => kaesnappy_ctx.c} | 8 +- .../src/v1/{kaelz4_ctx.h => kaesnappy_ctx.h} | 4 +- .../v1/{kaelz4_init.c => kaesnappy_init.c} | 8 +- .../v1/{kaelz4_init.h => kaesnappy_init.h} | 8 +- KAESnappy/src/v1/wd_queue_memory.c | 4 +- KAESnappy/src/v1/wd_queue_memory.h | 6 +- ...kaelz4_compress.c => kaesnappy_compress.c} | 6 +- .../{kaelz4_config.c => kaesnappy_config.c} | 6 +- .../{kaelz4_config.h => kaesnappy_config.h} | 4 +- KAESnappy/test/cli_test/cli_lz4_test.sh | 67 - KAESnappy/test/gtest/Makefile | 91 -- KAESnappy/test/gtest/README.md | 12 - KAESnappy/test/gtest/build.sh | 50 - .../test/gtest/gtest-download/download.sh | 32 - .../test/gtest/gtest-download/download.xml | 12 - .../googletest-release-1.11.0.zip | 0 KAESnappy/test/gtest/run.sh | 7 - KAESnappy/test/gtest/src/base_case.cpp | 89 -- KAESnappy/test/gtest/src/test2.cpp | 49 - KAESnappy/test/kzip/README.md | 27 - KAESnappy/test/kzip/alg/KAELz4/lz4.c | 44 - KAESnappy/test/kzip/alg/KAELz4/lz4Frame.c | 66 - .../test/kzip/alg/KAELz4Async/lz4AsyncBlock.c | 42 - .../test/kzip/alg/KAELz4Async/lz4AsyncFrame.c | 63 - KAESnappy/test/kzip/alg/manage.c | 51 - KAESnappy/test/kzip/alg/manage.h | 54 - KAESnappy/test/kzip/build.sh | 19 - KAESnappy/test/kzip/compress_ctx.h | 86 -- KAESnappy/test/kzip/datagen.c | 189 --- KAESnappy/test/kzip/datagen.h | 40 - KAESnappy/test/kzip/delayRecord.c | 61 - KAESnappy/test/kzip/delayRecord.h | 15 - KAESnappy/test/kzip/kaelz4.cnf | 2 - KAESnappy/test/kzip/main.c | 1220 ----------------- KAESnappy/test/kzip/platform.h | 155 --- KAESnappy/test/kzip/runAffinityFunc.sh | 77 -- KAESnappy/test/kzip/runError.sh | 113 -- KAESnappy/test/kzip/runFunc.sh | 54 - KAESnappy/test/kzip/runLogfunc.sh | 56 - KAESnappy/test/kzip/runPerf.sh | 101 -- .../test/kzip/scene_test_functions/entry.c | 32 - .../test/kzip/scene_test_functions/entry.h | 7 - .../scene_test_functions/test_async_frame.c | 201 --- KAESnappy/test/kzip/util.h | 697 ---------- KAESnappy/test/perftest/Readme.md | 26 - KAESnappy/test/perftest/kaelz4_perf.c | 362 ----- KAESnappy/test/perftest/makefile | 16 - 58 files changed, 61 insertions(+), 4347 deletions(-) rename KAESnappy/include/{kaelz4.h => kaesnappy.h} (98%) rename KAESnappy/src/{kaelz4_adapter.c => kaesnappy_adapter.c} (98%) rename KAESnappy/src/{kaelz4_adapter.h => kaesnappy_adapter.h} (93%) rename KAESnappy/src/utils/{kaelz4_common.c => kaesnappy_common.c} (96%) rename KAESnappy/src/utils/{kaelz4_common.h => kaesnappy_common.h} (83%) rename KAESnappy/src/utils/{kaelz4_log.c => kaesnappy_log.c} (99%) rename KAESnappy/src/utils/{kaelz4_log.h => kaesnappy_log.h} (98%) rename KAESnappy/src/utils/{kaelz4_utils.h => kaesnappy_utils.h} (97%) rename KAESnappy/src/v1/{kaelz4_comp.c => kaesnappy_comp.c} (99%) rename KAESnappy/src/v1/{kaelz4_comp.h => kaesnappy_comp.h} (97%) rename KAESnappy/src/v1/{kaelz4_ctx.c => kaesnappy_ctx.c} (99%) rename KAESnappy/src/v1/{kaelz4_ctx.h => kaesnappy_ctx.h} (98%) rename KAESnappy/src/v1/{kaelz4_init.c => kaesnappy_init.c} (89%) rename KAESnappy/src/v1/{kaelz4_init.h => kaesnappy_init.h} (69%) rename KAESnappy/src/v2/{kaelz4_compress.c => kaesnappy_compress.c} (96%) rename KAESnappy/src/v2/{kaelz4_config.c => kaesnappy_config.c} (98%) rename KAESnappy/src/v2/{kaelz4_config.h => kaesnappy_config.h} (97%) delete mode 100644 KAESnappy/test/cli_test/cli_lz4_test.sh delete mode 100644 KAESnappy/test/gtest/Makefile delete mode 100644 KAESnappy/test/gtest/README.md delete mode 100644 KAESnappy/test/gtest/build.sh delete mode 100644 KAESnappy/test/gtest/gtest-download/download.sh delete mode 100644 KAESnappy/test/gtest/gtest-download/download.xml delete mode 100644 KAESnappy/test/gtest/gtest-download/googletest-release-1.11.0.zip delete mode 100644 KAESnappy/test/gtest/run.sh delete mode 100644 KAESnappy/test/gtest/src/base_case.cpp delete mode 100644 KAESnappy/test/gtest/src/test2.cpp delete mode 100644 KAESnappy/test/kzip/README.md delete mode 100644 KAESnappy/test/kzip/alg/KAELz4/lz4.c delete mode 100644 KAESnappy/test/kzip/alg/KAELz4/lz4Frame.c delete mode 100644 KAESnappy/test/kzip/alg/KAELz4Async/lz4AsyncBlock.c delete mode 100644 KAESnappy/test/kzip/alg/KAELz4Async/lz4AsyncFrame.c delete mode 100644 KAESnappy/test/kzip/alg/manage.c delete mode 100644 KAESnappy/test/kzip/alg/manage.h delete mode 100644 KAESnappy/test/kzip/build.sh delete mode 100644 KAESnappy/test/kzip/compress_ctx.h delete mode 100644 KAESnappy/test/kzip/datagen.c delete mode 100644 KAESnappy/test/kzip/datagen.h delete mode 100644 KAESnappy/test/kzip/delayRecord.c delete mode 100644 KAESnappy/test/kzip/delayRecord.h delete mode 100644 KAESnappy/test/kzip/kaelz4.cnf delete mode 100644 KAESnappy/test/kzip/main.c delete mode 100644 KAESnappy/test/kzip/platform.h delete mode 100644 KAESnappy/test/kzip/runAffinityFunc.sh delete mode 100644 KAESnappy/test/kzip/runError.sh delete mode 100644 KAESnappy/test/kzip/runFunc.sh delete mode 100644 KAESnappy/test/kzip/runLogfunc.sh delete mode 100644 KAESnappy/test/kzip/runPerf.sh delete mode 100644 KAESnappy/test/kzip/scene_test_functions/entry.c delete mode 100644 KAESnappy/test/kzip/scene_test_functions/entry.h delete mode 100644 KAESnappy/test/kzip/scene_test_functions/test_async_frame.c delete mode 100644 KAESnappy/test/kzip/util.h delete mode 100644 KAESnappy/test/perftest/Readme.md delete mode 100644 KAESnappy/test/perftest/kaelz4_perf.c delete mode 100644 KAESnappy/test/perftest/makefile diff --git a/KAESnappy/Makefile b/KAESnappy/Makefile index da314c5..f498c5c 100644 --- a/KAESnappy/Makefile +++ b/KAESnappy/Makefile @@ -108,14 +108,14 @@ install : install -m 755 $(TARGET) $(ENGINE_INSTALL_PATH)/lib $(LN) $(ENGINE_INSTALL_PATH)/lib/$(TARGET) $(ENGINE_INSTALL_PATH)/lib/$(SOFTLINK) $(LN) $(ENGINE_INSTALL_PATH)/lib/$(TARGET) $(ENGINE_INSTALL_PATH)/lib/$(SOFTLINK).0 - install -m 755 $(WORK_PATH)/include/kaelz4.h $(ENGINE_INSTALL_PATH)/include - install -m 755 $(WORK_PATH)/src/utils/kaelz4_log.h $(ENGINE_INSTALL_PATH)/include + install -m 755 $(WORK_PATH)/include/kaesnappy.h $(ENGINE_INSTALL_PATH)/include + install -m 755 $(WORK_PATH)/src/utils/kaesnappy_log.h $(ENGINE_INSTALL_PATH)/include uninstall : $(RM) $(ENGINE_INSTALL_PATH)/lib/$(SOFTLINK) $(RM) $(ENGINE_INSTALL_PATH)/lib/$(SOFTLINK).0 $(RM) $(ENGINE_INSTALL_PATH)/lib/$(TARGET) $(RM) $(ENGINE_INSTALL_PATH)/lib/libkaelz4.a - $(RM) $(ENGINE_INSTALL_PATH)/include/kaelz4.h - $(RM) $(ENGINE_INSTALL_PATH)/include/kaelz4_log.h + $(RM) $(ENGINE_INSTALL_PATH)/include/kaesnappy.h + $(RM) $(ENGINE_INSTALL_PATH)/include/kaesnappy_log.h $(RM) $(ENGINE_INSTALL_PATH)/include/lz4.h $(RM) $(ENGINE_INSTALL_PATH)/include/lz4frame.h \ No newline at end of file diff --git a/KAESnappy/include/kaelz4.h b/KAESnappy/include/kaesnappy.h similarity index 98% rename from KAESnappy/include/kaelz4.h rename to KAESnappy/include/kaesnappy.h index 5252393..fe51282 100644 --- a/KAESnappy/include/kaelz4.h +++ b/KAESnappy/include/kaesnappy.h @@ -5,8 +5,8 @@ * Create: 2024-7-6 */ -#ifndef KAELZ4_H -#define KAELZ4_H +#ifndef KAESNAPPY_H +#define KAESNAPPY_H #include #include diff --git a/KAESnappy/src/kaelz4_adapter.c b/KAESnappy/src/kaesnappy_adapter.c similarity index 98% rename from KAESnappy/src/kaelz4_adapter.c rename to KAESnappy/src/kaesnappy_adapter.c index 9ccff74..056a994 100644 --- a/KAESnappy/src/kaelz4_adapter.c +++ b/KAESnappy/src/kaesnappy_adapter.c @@ -1,6 +1,6 @@ /* * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * @Description: kaelz4 adapter for sva(v2) and nosva(v1) + * @Description: kaesnappy adapter for sva(v2) and nosva(v1) * @Author: LiuYongYang * @Date: 2024-02-22 * @LastEditTime: 2024-02-26 @@ -8,11 +8,11 @@ #include #include -#include "kaelz4_common.h" -#include "kaelz4.h" -#include "kaelz4_utils.h" -#include "kaelz4_adapter.h" -#include "kaelz4_log.h" +#include "kaesnappy_common.h" +#include "kaesnappy.h" +#include "kaesnappy_utils.h" +#include "kaesnappy_adapter.h" +#include "kaesnappy_log.h" #include "uadk/wd.h" lz4_task_queues g_task_queues = {0}; diff --git a/KAESnappy/src/kaelz4_adapter.h b/KAESnappy/src/kaesnappy_adapter.h similarity index 93% rename from KAESnappy/src/kaelz4_adapter.h rename to KAESnappy/src/kaesnappy_adapter.h index 073de5c..61617fb 100644 --- a/KAESnappy/src/kaelz4_adapter.h +++ b/KAESnappy/src/kaesnappy_adapter.h @@ -1,15 +1,14 @@ /* * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * @Description: kaelz4 adapter for sva(v2) and nosva(v1) header file + * @Description: kaesnappy adapter for sva(v2) and nosva(v1) header file * @Author: LiuYongYang * @Date: 2024-02-22 * @LastEditTime: 2024-02-22 */ -#ifndef KAELZ4_ADAPTER -#define KAELZ4_ADAPTER -#include -#include "kaelz4_common.h" +#ifndef KAESNAPPY_ADAPTER +#define KAESNAPPY_ADAPTER +#include "kaesnappy_common.h" enum { HW_NONE, diff --git a/KAESnappy/src/utils/kaelz4_common.c b/KAESnappy/src/utils/kaesnappy_common.c similarity index 96% rename from KAESnappy/src/utils/kaelz4_common.c rename to KAESnappy/src/utils/kaesnappy_common.c index a247fc4..bae5172 100644 --- a/KAESnappy/src/utils/kaelz4_common.c +++ b/KAESnappy/src/utils/kaesnappy_common.c @@ -1,13 +1,13 @@ /* * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * @Description: lz4 common func + * @Description: snappy common func * @Author: LiuYongYang * @Date: 2024-02-22 * @LastEditTime: 2024-03-28 */ #include #include -#include "kaelz4_common.h" +#include "kaesnappy_common.h" #define HIDDEN_API __attribute__((visibility("hidden"))) #define CONSTRUCTOR __attribute__((constructor)) diff --git a/KAESnappy/src/utils/kaelz4_common.h b/KAESnappy/src/utils/kaesnappy_common.h similarity index 83% rename from KAESnappy/src/utils/kaelz4_common.h rename to KAESnappy/src/utils/kaesnappy_common.h index 4d82830..1b00e25 100644 --- a/KAESnappy/src/utils/kaelz4_common.h +++ b/KAESnappy/src/utils/kaesnappy_common.h @@ -5,8 +5,8 @@ * Create: 2024-7-6 */ -#ifndef KAELZ4_COMMON_H -#define KAELZ4_COMMON_H +#ifndef KAESNAPPY_COMMON_H +#define KAESNAPPY_COMMON_H #define CONF_KAELZ4 @@ -20,6 +20,6 @@ enum kae_lz4_async_data_format { KAELZ4_ASYNC_BUTT, }; -#include "kaelz4.h" +#include "kaesnappy.h" #endif diff --git a/KAESnappy/src/utils/kaelz4_log.c b/KAESnappy/src/utils/kaesnappy_log.c similarity index 99% rename from KAESnappy/src/utils/kaelz4_log.c rename to KAESnappy/src/utils/kaesnappy_log.c index b6d2e79..50ec6a9 100644 --- a/KAESnappy/src/utils/kaelz4_log.c +++ b/KAESnappy/src/utils/kaesnappy_log.c @@ -11,7 +11,7 @@ #include #include #include -#include "kaelz4_log.h" +#include "kaesnappy_log.h" #define KAE_CONFIG_FILE_NAME "/kaelz4.cnf" #define MAX_LEVEL_LEN 10 diff --git a/KAESnappy/src/utils/kaelz4_log.h b/KAESnappy/src/utils/kaesnappy_log.h similarity index 98% rename from KAESnappy/src/utils/kaelz4_log.h rename to KAESnappy/src/utils/kaesnappy_log.h index 56dba7b..c665aec 100644 --- a/KAESnappy/src/utils/kaelz4_log.h +++ b/KAESnappy/src/utils/kaesnappy_log.h @@ -5,8 +5,8 @@ * Create: 2023-5-30 */ -#ifndef KAELZ4_LOG_H -#define KAELZ4_LOG_H +#ifndef KAESNAPPY_LOG_H +#define KAESNAPPY_LOG_H #include #include #include diff --git a/KAESnappy/src/utils/kaelz4_utils.h b/KAESnappy/src/utils/kaesnappy_utils.h similarity index 97% rename from KAESnappy/src/utils/kaelz4_utils.h rename to KAESnappy/src/utils/kaesnappy_utils.h index 6c7d578..434b6f6 100644 --- a/KAESnappy/src/utils/kaelz4_utils.h +++ b/KAESnappy/src/utils/kaesnappy_utils.h @@ -1,13 +1,13 @@ /* * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * @Description: kaelz4 init and utils + * @Description: kaesnappy init and utils * @Author: LiuYongYang * @Date: 2024-02-23 * @LastEditTime: 2024-02-23 */ /***************************************************************************** - * @file kaelz4_utils.h + * @file kaesnappy_utils.h * * This file provides the utils funtion; * diff --git a/KAESnappy/src/v1/kaelz4_comp.c b/KAESnappy/src/v1/kaesnappy_comp.c similarity index 99% rename from KAESnappy/src/v1/kaelz4_comp.c rename to KAESnappy/src/v1/kaesnappy_comp.c index 5e8f7ae..98175b3 100644 --- a/KAESnappy/src/v1/kaelz4_comp.c +++ b/KAESnappy/src/v1/kaesnappy_comp.c @@ -1,14 +1,14 @@ /* * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * @Description: kaelz4 nosva compress + * @Description: kaesnappy nosva compress * @Author: LiuYongYang * @Date: 2024-02-26 * @LastEditTime: 2024-03-28 */ #include -#include "kaelz4_ctx.h" -#include "kaelz4_comp.h" -#include "kaelz4_log.h" +#include "kaesnappy_ctx.h" +#include "kaesnappy_comp.h" +#include "kaesnappy_log.h" #include __thread struct kaelz4_async_ctrl g_async_ctrl = {0}; diff --git a/KAESnappy/src/v1/kaelz4_comp.h b/KAESnappy/src/v1/kaesnappy_comp.h similarity index 97% rename from KAESnappy/src/v1/kaelz4_comp.h rename to KAESnappy/src/v1/kaesnappy_comp.h index 19b1736..4836f53 100644 --- a/KAESnappy/src/v1/kaelz4_comp.h +++ b/KAESnappy/src/v1/kaesnappy_comp.h @@ -1,15 +1,15 @@ /* * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * @Description: kaelz4 nosva compress header file + * @Description: kaesnappy nosva compress header file * @Author: LiuYongYang * @Date: 2024-02-26 * @LastEditTime: 2024-03-28 */ -#ifndef KAELZ4_COMP_H -#define KAELZ4_COMP_H +#ifndef KAESNAPPY_COMP_H +#define KAESNAPPY_COMP_H -#include "kaelz4_common.h" +#include "kaesnappy_common.h" #include #include #include diff --git a/KAESnappy/src/v1/kaelz4_ctx.c b/KAESnappy/src/v1/kaesnappy_ctx.c similarity index 99% rename from KAESnappy/src/v1/kaelz4_ctx.c rename to KAESnappy/src/v1/kaesnappy_ctx.c index ba7b35b..cb79fcb 100644 --- a/KAESnappy/src/v1/kaelz4_ctx.c +++ b/KAESnappy/src/v1/kaesnappy_ctx.c @@ -1,13 +1,13 @@ /* * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * @Description: kaelz4 ctx func + * @Description: kaesnappy ctx func * @Author: LiuYongYang * @Date: 2024-02-23 * @LastEditTime: 2024-03-28 */ -#include "kaelz4_ctx.h" -#include "kaelz4_utils.h" -#include "kaelz4_log.h" +#include "kaesnappy_ctx.h" +#include "kaesnappy_utils.h" +#include "kaesnappy_log.h" static KAE_QUEUE_POOL_HEAD_S* g_kaelz4_deflate_qp = NULL; static KAE_QUEUE_POOL_HEAD_S* g_kaelz4_inflate_qp = NULL; diff --git a/KAESnappy/src/v1/kaelz4_ctx.h b/KAESnappy/src/v1/kaesnappy_ctx.h similarity index 98% rename from KAESnappy/src/v1/kaelz4_ctx.h rename to KAESnappy/src/v1/kaesnappy_ctx.h index 7308ef0..e7e57ca 100644 --- a/KAESnappy/src/v1/kaelz4_ctx.h +++ b/KAESnappy/src/v1/kaesnappy_ctx.h @@ -5,8 +5,8 @@ * *****************************************************************************/ -#ifndef KAELZ4_CTX_H -#define KAELZ4_CTX_H +#ifndef KAESNAPPY_CTX_H +#define KAESNAPPY_CTX_H #include #include "wd_queue_memory.h" #include "uadk/v1/wd_comp.h" diff --git a/KAESnappy/src/v1/kaelz4_init.c b/KAESnappy/src/v1/kaesnappy_init.c similarity index 89% rename from KAESnappy/src/v1/kaelz4_init.c rename to KAESnappy/src/v1/kaesnappy_init.c index ad60946..2147636 100644 --- a/KAESnappy/src/v1/kaelz4_init.c +++ b/KAESnappy/src/v1/kaesnappy_init.c @@ -1,13 +1,13 @@ /* * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * @Description: kaelz4 nosva init + * @Description: kaesnappy nosva init * @Author: LiuYongYang * @Date: 2024-02-26 * @LastEditTime: 2024-02-26 */ -#include "kaelz4_ctx.h" -#include "kaelz4_init.h" -#include "kaelz4_log.h" +#include "kaesnappy_ctx.h" +#include "kaesnappy_init.h" +#include "kaesnappy_log.h" int kaelz4_init_v1(LZ4_CCtx* zc) { diff --git a/KAESnappy/src/v1/kaelz4_init.h b/KAESnappy/src/v1/kaesnappy_init.h similarity index 69% rename from KAESnappy/src/v1/kaelz4_init.h rename to KAESnappy/src/v1/kaesnappy_init.h index 4793c5a..7df8c73 100644 --- a/KAESnappy/src/v1/kaelz4_init.h +++ b/KAESnappy/src/v1/kaesnappy_init.h @@ -1,15 +1,15 @@ /* * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * @Description: kaelz4 nosva init head file + * @Description: kaesnappy nosva init head file * @Author: LiuYongYang * @Date: 2024-02-26 * @LastEditTime: 2024-02-26 */ -#ifndef KAELZ4_INIT_H -#define KAELZ4_INIT_H +#ifndef KAESNAPPY_INIT_H +#define KAESNAPPY_INIT_H -#include "kaelz4_common.h" +#include "kaesnappy_common.h" int kaelz4_init_v1(LZ4_CCtx* zc); void kaelz4_reset_v1(LZ4_CCtx* zc); diff --git a/KAESnappy/src/v1/wd_queue_memory.c b/KAESnappy/src/v1/wd_queue_memory.c index 91ffd1c..32dce6d 100644 --- a/KAESnappy/src/v1/wd_queue_memory.c +++ b/KAESnappy/src/v1/wd_queue_memory.c @@ -7,10 +7,10 @@ #include #include "wd_queue_memory.h" -#include "kaelz4_log.h" +#include "kaesnappy_log.h" #include "uadk/v1/wd_bmm.h" #include "uadk/v1/wd_comp.h" -#include "kaelz4_ctx.h" +#include "kaesnappy_ctx.h" void kaelz4_wd_free_queue(struct wd_queue* queue); struct wd_queue* kaelz4_wd_new_queue(int comp_alg_type, int comp_optype); diff --git a/KAESnappy/src/v1/wd_queue_memory.h b/KAESnappy/src/v1/wd_queue_memory.h index 739d14f..215a05f 100644 --- a/KAESnappy/src/v1/wd_queue_memory.h +++ b/KAESnappy/src/v1/wd_queue_memory.h @@ -20,12 +20,12 @@ * *****************************************************************************/ -#ifndef __KAELZ4_QUEUE_MEMORY_H -#define __KAELZ4_QUEUE_MEMORY_H +#ifndef __KAESNAPPY_QUEUE_MEMORY_H +#define __KAESNAPPY_QUEUE_MEMORY_H #include #include "uadk/v1/wd.h" -#include "kaelz4_utils.h" +#include "kaesnappy_utils.h" #define KAE_QUEUE_POOL_MAX_SIZE (512) #define CHECK_QUEUE_TIME_SECONDS (60) // seconds diff --git a/KAESnappy/src/v2/kaelz4_compress.c b/KAESnappy/src/v2/kaesnappy_compress.c similarity index 96% rename from KAESnappy/src/v2/kaelz4_compress.c rename to KAESnappy/src/v2/kaesnappy_compress.c index ac56215..ff14ce5 100644 --- a/KAESnappy/src/v2/kaelz4_compress.c +++ b/KAESnappy/src/v2/kaesnappy_compress.c @@ -15,9 +15,9 @@ #include "uadk/wd_comp.h" #include "uadk/uacce.h" -#include "kaelz4_common.h" -#include "kaelz4_config.h" -#include "kaelz4_log.h" +#include "kaesnappy_common.h" +#include "kaesnappy_config.h" +#include "kaesnappy_log.h" void kaelz4_setstatus_v2(LZ4_CCtx* zc, unsigned int status) { diff --git a/KAESnappy/src/v2/kaelz4_config.c b/KAESnappy/src/v2/kaesnappy_config.c similarity index 98% rename from KAESnappy/src/v2/kaelz4_config.c rename to KAESnappy/src/v2/kaesnappy_config.c index a80ec1c..5d52d47 100644 --- a/KAESnappy/src/v2/kaelz4_config.c +++ b/KAESnappy/src/v2/kaesnappy_config.c @@ -16,9 +16,9 @@ #include "uadk/wd_sched.h" #include "uadk/uacce.h" -#include "kaelz4_common.h" -#include "kaelz4_config.h" -#include "kaelz4_log.h" +#include "kaesnappy_common.h" +#include "kaesnappy_config.h" +#include "kaesnappy_log.h" #define CTX_SET_SIZE 4 #define CTX_SET_NUM 1 diff --git a/KAESnappy/src/v2/kaelz4_config.h b/KAESnappy/src/v2/kaesnappy_config.h similarity index 97% rename from KAESnappy/src/v2/kaelz4_config.h rename to KAESnappy/src/v2/kaesnappy_config.h index 294233b..99996e8 100644 --- a/KAESnappy/src/v2/kaelz4_config.h +++ b/KAESnappy/src/v2/kaesnappy_config.h @@ -5,8 +5,8 @@ * Create: 2021-7-19 */ -#ifndef KAELZ4_CTX_H -#define KAELZ4_CTX_H +#ifndef KAESNAPPY_CTX_H +#define KAESNAPPY_CTX_H #include #include diff --git a/KAESnappy/test/cli_test/cli_lz4_test.sh b/KAESnappy/test/cli_test/cli_lz4_test.sh deleted file mode 100644 index 26d2b63..0000000 --- a/KAESnappy/test/cli_test/cli_lz4_test.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# lz4可执行文件路径 -lz4_executable="/usr/bin/lz4" -kaelz4_executable="/usr/local/kaelz4/bin/lz4" -EXE="$lz4_executable $kaelz4_executable" - -# 压缩级别 -compression_level="1" - -SRC_PATH=$(pwd) - -# 测试文件名称 -BASE_TESTDATA_ADD="${SRC_PATH}/../../../scripts/compressTestDataset/" -test_file_name="ooffice osdb samba webster xml x-ray" -blocksize="4 5 6 7" - -# 测试函数 -function run_test() { - local filename="$1" - local exe="$2" - local compression_level="$3" - local blocksize="$4" - local RESDATA - local COMPRESS_SPEED - local COMPRESS_RATE - local UNCOMPRESS_SPEED - local compressrate - - echo "KAE_LZ4_LEVEL=$compression_level $exe -b$compression_level -B$blocksize $BASE_TESTDATA_ADD$filename" - $exe -b$compression_level -B$blocksize $BASE_TESTDATA_ADD$filename - - #if [ $exe == $lz4_executable ]; then - # COMPRESS_SPEED=$(echo $RESDATA | awk '{print $(59)}') - # UNCOMPRESS_SPEED=$(echo $RESDATA | awk '{print $(62)}') - # COMPRESS_RATE=$(echo $RESDATA | awk '{print $(58)}') - #elif [ $exe == $kaelz4_executable ]; then - # COMPRESS_SPEED=$(echo $RESDATA | rev | awk '{print $(5)}' | rev) - # UNCOMPRESS_SPEED=$(echo $RESDATA | rev | awk '{print $(3)}' | rev) - # COMPRESS_RATE=$(echo $RESDATA | rev | awk '{print $(6)}' | rev) - #fi - - # compressrate=$(echo "$COMPRESS_RATE" | awk -F'\[(x),]' '{print $2}') - compressrate=$(echo "$COMPRESS_RATE" | awk '{gsub(/[^0-9.]/, ""); print}') - - echo $exe $filename $compression_level $COMPRESS_SPEED $UNCOMPRESS_SPEED $compressrate -} - -function main(){ - # clear - echo "EXE文件 , 压缩文件 , Level , CompressSpeed(MB/s) , UncompressSpeed(MB/s) , CompressRate" - for exe in $EXE - do - for filename in $test_file_name - do - for level in $compression_level - do - for blksize in $blocksize - do - run_test $filename $exe $level $blksize - done - done - done - done -} - -main "$@" -exit $? diff --git a/KAESnappy/test/gtest/Makefile b/KAESnappy/test/gtest/Makefile deleted file mode 100644 index 25118a7..0000000 --- a/KAESnappy/test/gtest/Makefile +++ /dev/null @@ -1,91 +0,0 @@ -# -# Generic Makefile for C Program -# Description: -# compile for accelerator -# -# Usage: -# $ make compile and link the program. -# $ make rebuild rebuild the program. The same as make clean && make all. -# $ make clean clean the objective, dependent and executable files. -#============================================================================== -WORK_PATH := $(shell pwd) -SRC_CODE_PATH := $(shell pwd)/../../src -CC:=gcc -CXX:=g++ - -TARGET := kaelz4test - -ifndef SILENCE - SILENCE = @ -endif - -# Src -SRCDIRS := ${WORK_PATH}/src -SRCEXTS := .c .cpp # C program - -# header and lib -INCDIR := -I /usr/local/kaelz4/include/ -INCDIR += -I $(WORK_PATH)/../../open_source/lz4-1.9.4/lib -INCDIR += -I $(WORK_PATH)/../../src/utils -INCDIR += -I $(WORK_PATH)/../../src/v1 -INCDIR += -I ${WORK_PATH}/../../../uadk/include/ -LIBDIR := -L/usr/local/kaelz4/lib/ -LIBDIR += -L/usr/lib/ - -# Include -INCDIR += -I $(SRCDIRS)/ -# INCDIR += -I $(WORK_PATH)/../../uadk/include -INCDIR += -I $(WORK_PATH)/gtest-download/googletest-release-1.11.0/googletest/include - -# Include Libs. -LIBDIR += -L $(WORK_PATH)/test_tool_bins/gtest -LIBS += -lgtest_main -lgtest -lz -llz4 -lrt -lkaelz4 -lwd -lwd_comp - -# The flags -CFLAGS := -g -pipe -Wall -Wextra -Wshadow -fPIC -std=c++11 #-Wl,-rpath,/usr/local/kaezip/lib -LDFLAGS += $(LIBDIR) -LDFLAGS += $(LIBS) -LDFLAGS += -Wl,-rpath,/usr/local/kaelz4/lib - -# The command used to delete file. -RM = rm -f - -SRC_CODES = $(wildcard $(SRC_CODE_PATH)/v1/*.c) -SOURCES = $(wildcard $(SRCDIRS)/*.cpp) $(SRC_CODES) - -OBJS = $(foreach x,$(SRCEXTS), \ - $(patsubst %$(x), %.o, $(filter %$(x),$(SOURCES)))) - -.PHONY : all objs clean cleanall rebuild - -all : deps $(TARGET) - - -deps: - -# Rules for creating the dependency files (.d). -%.d : %.cpp - $(CXX) -MM -MD $(CFLAGS) $< - -# Rules for producing the objects. -objs : $(OBJS) -%.o : %.c - @echo compiling $(notdir $<) - $(SILENCE) $(CC) -c $(CFLAGS) $(INCDIR) $(LDFLAGS) -o $@ $< - -%.o : %.cpp - @echo compiling $(notdir $<) - $(SILENCE) $(CXX) -c $(CFLAGS) $(INCDIR) $(LDFLAGS) -o $@ $< - -$(TARGET): $(OBJS) - @echo Linking $@ - - $(SILENCE) $(CXX) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(INCDIR) -o ./$(TARGET) $(OBJS) - -rebuild: clean all - -clean : - @-$(RM) *.d *.a *.so $(TARGET) - @-$(RM) - @find ${WORK_PATH} -name '*.o' -exec $(RM) {} \; - @echo all clean diff --git a/KAESnappy/test/gtest/README.md b/KAESnappy/test/gtest/README.md deleted file mode 100644 index cd729a0..0000000 --- a/KAESnappy/test/gtest/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# KAELz4功能测试通过复用开源lz4测试功能实现 -# 具体通过执行kaelz4安装，而后至KAELz4/open_source/lz4-1.9.4/tests目录，并执行make -# 1.blcok模式的功能测试：通过执行生成的可执行文件fuzzer，以及脚本test_custom_blocksizes.sh实现 -# 2.frame模式的功能测试：通过执行生成的可执行文件frametest，以及脚本test-lz4-list.py实现 - -# 注：执行上述功能测试前，请确保kaelz4已安装完成 - -# unit test - -~~~ -sh run.sh -~~~ \ No newline at end of file diff --git a/KAESnappy/test/gtest/build.sh b/KAESnappy/test/gtest/build.sh deleted file mode 100644 index cec2d20..0000000 --- a/KAESnappy/test/gtest/build.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -BUILD_PATH=$(pwd) -gtest_download="${BUILD_PATH}/gtest-download" -GOOGLE_TEST_DIR="${gtest_download}/googletest-release-1.11.0" -set -ex - -function build_googletest() -{ - cd ${gtest_download}/googletest-release-1.11.0 - if [ ! -f "${BUILD_PATH}/test_tool_bins/gtest/libgtest.a" ]; then - mkdir -p "${BUILD_PATH}/test_tool_bins/gtest" - mkdir -p "${GOOGLE_TEST_DIR}/build" - cd "${GOOGLE_TEST_DIR}/build" - cmake ../ - make -j16 - cp ${GOOGLE_TEST_DIR}/build/lib/*.a ${BUILD_PATH}/test_tool_bins/gtest/ - fi -} - -function download_googletest() -{ - cd ${gtest_download} - if [ ! -d ${gtest_download}/googletest-release-1.11.0 ] - then - echo -e "\033[32m The googletest-release-1.11.0 directory not exists and need to be downloaded. \033[0m" - sh download.sh - fi - cd - -} - -function main() -{ - if [ "$1" == "clean" ]; then - echo "MakeClean" - make clean - exit 1 - fi - - # 下载gtest - download_googletest - - # 编译gtest - build_googletest - - # 编译测试代码 - cd ${BUILD_PATH} - make -} - -main $@ \ No newline at end of file diff --git a/KAESnappy/test/gtest/gtest-download/download.sh b/KAESnappy/test/gtest/gtest-download/download.sh deleted file mode 100644 index cf42a7b..0000000 --- a/KAESnappy/test/gtest/gtest-download/download.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -############################################################## -## Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. -## @Filename: download.sh -## @Usage: sh download.sh download googletest source code -############################################################## - -curd=$(pwd) -test=$curd/.. -root_path=$curd/../.. - -if [ -d ./googletest-release-1.11.0 ] -then - echo -e "\033[32m The googletest-release-1.11.0 directory already exists and does not need to be downloaded. \033[0m" - exit 0 -fi - -if [ -d ./googletest-release-1.11.0.zip ] -then - echo -e "\033[32m The googletest-release-1.11.0.zip was downloaded, now doing unzip. \033[0m" - unzip googletest-release-1.11.0.zip - exit 0 -fi - -if [ -d ~/.ArtGet/conf/Setting.xml ] -then - nohup artget config -fi - -#这里可以考虑通过wget的方式获取想要的gtest包,如果环境有googletest-release-1.11.0.zip，复制到此目录也行 - -unzip googletest-release-1.11.0.zip \ No newline at end of file diff --git a/KAESnappy/test/gtest/gtest-download/download.xml b/KAESnappy/test/gtest/gtest-download/download.xml deleted file mode 100644 index dfe641d..0000000 --- a/KAESnappy/test/gtest/gtest-download/download.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - opensource - googletest - 1.11.0 - googletest-release-1.11.0.zip - gtest-download - - - \ No newline at end of file diff --git a/KAESnappy/test/gtest/gtest-download/googletest-release-1.11.0.zip b/KAESnappy/test/gtest/gtest-download/googletest-release-1.11.0.zip deleted file mode 100644 index e69de29..0000000 diff --git a/KAESnappy/test/gtest/run.sh b/KAESnappy/test/gtest/run.sh deleted file mode 100644 index d4662a2..0000000 --- a/KAESnappy/test/gtest/run.sh +++ /dev/null @@ -1,7 +0,0 @@ -export LD_LIBRARY_PATH=/usr/local/kaelz4/lib/:/usr/local/kaezstd/lib/:/usr/local/kaezip/lib/:$LD_LIBRARY_PATH -export KAE_LZ4_WINTYPE=8 -export KAE_LZ4_COMP_TYPE=8 - -sh build.sh - -./kaelz4test \ No newline at end of file diff --git a/KAESnappy/test/gtest/src/base_case.cpp b/KAESnappy/test/gtest/src/base_case.cpp deleted file mode 100644 index 94c996d..0000000 --- a/KAESnappy/test/gtest/src/base_case.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -extern "C" { - #include "kaelz4.h" -#include -#include -} - -// // 随机生成指定长度的数据 -// static void generate_random_data(Bytef* data, unsigned long long length) { -// srand((unsigned int)time(NULL)); -// for (unsigned long long i = 0; i < length; ++i) { -// data[i] = rand() % 256; -// } -// } - -// struct compress_param { -// struct kaelz4_result result; -// unsigned char*dst; -// unsigned int dst_len; -// volatile unsigned int done; -// }; -// static void compress_async_callback(struct kaelz4_result *result) -// { -// struct compress_param *param = (struct compress_param *)result->user_data; -// param->dst_len = result->dst_len; -// param->done = 1; -// return; -// } - -// static void lz4_block_common_test() -// { -// uLong input_size = 1024UL * 48; // 48k -// Bytef *src = new Bytef[input_size]; -// ASSERT_NE(src, nullptr); -// generate_random_data(src, input_size); -// Bytef *dst = new Bytef[input_size * 250]; -// struct compress_param *user_data = (struct compress_param *)malloc(sizeof(struct compress_param)); -// user_data->dst = dst; -// user_data->dst_len = 0; -// user_data->done = 0; - -// struct kaelz4_result *result = (struct kaelz4_result *)malloc(sizeof(struct kaelz4_result));; -// result->src_size = input_size; -// result->user_data = user_data; -// printf("start--> \n"); -// int ret = LZ4_compress_async(src, dst, compress_async_callback, result); -// ASSERT_EQ(ret, 0); -// // 等待压缩操作完成 -// while (user_data->done == 0) { -// // 可选择添加延时或者使用条件变量进行更有效的等待 -// usleep(10); -// } -// printf("start-->2 \n"); -// // 异步压缩完成后，调用同步解压接口 -// int dst_len = input_size; -// Bytef *decompressed_data = new Bytef[dst_len]; // 用于存放解压后的数据 -// ASSERT_NE(decompressed_data, nullptr); - -// int src_len = user_data->dst_len; -// int decompressed_size = LZ4_decompress_safe((const char *)user_data->dst, (char *)decompressed_data, src_len, dst_len); -// ASSERT_GT(decompressed_size, 0); // 解压成功返回正数 -// ASSERT_EQ(decompressed_size, input_size); -// // 比较原始数据与解压后的数据是否一致 -// ASSERT_EQ(memcmp(src, decompressed_data, input_size), 0); // 比较原始数据和解压数据 - -// // 清理内存 -// delete[] src; -// delete[] dst; -// delete[] decompressed_data; -// free(user_data); -// } -TEST(LZ4Test, CompressAndDecompress_lz4) -{ - int ret = 0; - ASSERT_EQ(ret, 0); -} - diff --git a/KAESnappy/test/gtest/src/test2.cpp b/KAESnappy/test/gtest/src/test2.cpp deleted file mode 100644 index 5099b70..0000000 --- a/KAESnappy/test/gtest/src/test2.cpp +++ /dev/null @@ -1,49 +0,0 @@ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -extern "C" { - #include "kaelz4.h" - #include - #include - #include "kaelz4_ctx.h" - #include "kaelz4_comp.h" - #include "kaelz4_log.h" - extern void kaelz4_ctx_clear(void); -} - -extern __thread struct kaelz4_async_ctrl g_async_ctrl; - -TEST(kaelz4_ctx_clear_Test, is_test_g_async_ctrl_all_clean) -{ - int ret = 0; - kaelz4_ctx_t *kzctx = (kaelz4_ctx_t *)malloc(sizeof(kaelz4_ctx_t)); - g_async_ctrl.kz_ctx[0] = kzctx; - kaelz4_init_ctx(g_async_ctrl.kz_ctx[0]); - for (int i = 0; i < 2; i++) { - if (g_async_ctrl.kz_ctx[i] != NULL) { - ret++; - } - } - ASSERT_EQ(ret, 1); - - kaelz4_ctx_clear(); - - ret = 0; - for (int i = 0; i < 2; i++) { - if (g_async_ctrl.kz_ctx[i] != NULL) { - ret++; - } - } - ASSERT_EQ(ret, 0); -} diff --git a/KAESnappy/test/kzip/README.md b/KAESnappy/test/kzip/README.md deleted file mode 100644 index 6e50b48..0000000 --- a/KAESnappy/test/kzip/README.md +++ /dev/null @@ -1,27 +0,0 @@ - -# 压缩性能测试小工具 kzip - -## 构建 -1、安装依赖 -~~~ -cd KAE -# 安装 frame 相关头文件 -yum install lz4-devel -# 覆盖安装本次新增异步接口相关头文件 -sh build.sh lz4 -~~~ -2、打包 kzip -~~~ -# 在测试目录中 -sh build.sh -~~~ - -## 测试 - -~~~ -# 测试 alg/KAELz4/lz4.c -sh runPerf.sh -A kaelz4 -m 4 - -# 测试 alg/KAELz4/lz4Frame.c -sh runPerf.sh -A kaelz4_frame -m 12 -~~~ \ No newline at end of file diff --git a/KAESnappy/test/kzip/alg/KAELz4/lz4.c b/KAESnappy/test/kzip/alg/KAELz4/lz4.c deleted file mode 100644 index 9cb8022..0000000 --- a/KAESnappy/test/kzip/alg/KAELz4/lz4.c +++ /dev/null @@ -1,44 +0,0 @@ -#include "../manage.h" -#include -#include - -// LZ4 压缩实现 -static int lz4_compress(const unsigned char *src, unsigned int *src_len, unsigned char *dst, unsigned int *dst_len) -{ - int ret = LZ4_compress_default((const char *)src, (char *)dst, *src_len, *dst_len); - *dst_len = ret; // lz4 LZ4_compress_default的返回值才是压缩后的空间大小。 - return ret > 0 ? 0 : ret; -} - -// LZ4 解压实现 -static int lz4_decompress(const unsigned char *src, unsigned int *src_len, unsigned char *dst, unsigned int *dst_len) -{ - int ret = LZ4_decompress_safe((const char *)src, (char *)dst, *src_len, *dst_len); - *dst_len = ret; // lz4 LZ4_decompress_safe 的返回值才是解压的空间大小。 - return ret > 0 ? 0 : ret; -} - -static int lz4_bound(int src_len) { - return LZ4_compressBound(src_len); -} - -// LZ4 初始化 -static int lz4_init() { - printf("Initializing LZ4...\n"); - return 0; -} - -// LZ4 算法实例 -compression_algorithm_t lz4_algorithm = { - .name = "kaelz4", - .bound = lz4_bound, - .compress = lz4_compress, - .decompress = lz4_decompress, - .init = lz4_init -}; - -// 注册 LZ4 算法 -void register_lz4_algorithm(void) -{ - register_algorithm(&lz4_algorithm); -} \ No newline at end of file diff --git a/KAESnappy/test/kzip/alg/KAELz4/lz4Frame.c b/KAESnappy/test/kzip/alg/KAELz4/lz4Frame.c deleted file mode 100644 index 314af86..0000000 --- a/KAESnappy/test/kzip/alg/KAELz4/lz4Frame.c +++ /dev/null @@ -1,66 +0,0 @@ -#include "../manage.h" -#include -#include -#include - -static int g_custom_frameinfo_config = 0; // 是否 自定义 frameinfo 格式 - -// 单个 LZ4 frame 格式文件的压缩实现 -static int lz4_frame_compress(const unsigned char *src, unsigned int *src_len, unsigned char *dst, unsigned int *dst_len) -{ - int ret; - if (g_custom_frameinfo_config == 0) { - ret = LZ4F_compressFrame(dst, *dst_len, src, *src_len, NULL); - } else { - LZ4F_preferences_t preferences = {0}; - preferences.frameInfo.blockSizeID = LZ4F_max64KB; // 设定块大小 - preferences.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - preferences.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; - preferences.frameInfo.contentSize = *src_len; - ret = LZ4F_compressFrame(dst, *dst_len, src, *src_len, &preferences); - } - - *dst_len = ret; // lz4 LZ4_compress_default的返回值才是压缩后的空间大小。 - return ret > 0 ? 0 : ret; -} - -// 单个 LZ4 frame 格式文件的解压实现 -static int lz4_frame_decompress(const unsigned char *src, unsigned int *src_len, unsigned char *dst, unsigned int *dst_len) -{ - LZ4F_decompressionContext_t dctx; - LZ4F_createDecompressionContext(&dctx, 100); - size_t tmp_src_len = *src_len; - size_t tmp_dst_len = *dst_len; - int ret = LZ4F_decompress(dctx, dst, &tmp_dst_len, src, &tmp_src_len, NULL); - LZ4F_freeDecompressionContext(dctx); - *src_len = tmp_src_len; - *dst_len = tmp_dst_len; - return ret > 0 ? 0 : ret; -} -static int lz4_frame_bound(int src_len) { - int needlen = LZ4F_compressFrameBound(src_len, NULL); - if (g_custom_frameinfo_config == 1) { - needlen += 1024; - } - return needlen; -} -// LZ4 frame 初始化 -static int lz4_frame_init() { - printf("Initializing LZ4...\n"); - return 0; -} - -// LZ4 frame 算法实例 -compression_algorithm_t lz4_frame_algorithm = { - .name = "kaelz4_frame", - .bound = lz4_frame_bound, - .compress = lz4_frame_compress, - .decompress = lz4_frame_decompress, - .init = lz4_frame_init -}; - -// 注册 LZ4 frame 算法 -void register_lz4_frame_algorithm(void) -{ - register_algorithm(&lz4_frame_algorithm); -} \ No newline at end of file diff --git a/KAESnappy/test/kzip/alg/KAELz4Async/lz4AsyncBlock.c b/KAESnappy/test/kzip/alg/KAELz4Async/lz4AsyncBlock.c deleted file mode 100644 index d4ff287..0000000 --- a/KAESnappy/test/kzip/alg/KAELz4Async/lz4AsyncBlock.c +++ /dev/null @@ -1,42 +0,0 @@ -#include "../manage.h" -#include -#include - -// LZ4 压缩实现 -static int lz4async_block_compress(const unsigned char *src, unsigned char *dst, lz4_async_callback cb, struct kaelz4_result *result) -{ - int ret = LZ4_compress_async(src, dst, cb, result); - return ret; -} - - -// LZ4 解压实现 -static int lz4async_block_decompress_sync(const unsigned char *src, unsigned int *src_len, unsigned char *dst, unsigned int *dst_len) -{ - int ret = LZ4_decompress_safe((const char *)src, (char *)dst, *src_len, *dst_len); - *dst_len = ret; // lz4 LZ4_decompress_safe 的返回值才是解压的空间大小。 - return ret > 0 ? 0 : ret; -} -static int lz4_bound(int src_len) { - return LZ4_compressBound(src_len); -} -// LZ4 初始化 -static int lz4_async_block_init() { - printf("Initializing LZ4...\n"); - return 0; -} - -// LZ4 算法实例 -compression_algorithm_t lz4async_block_algorithm = { - .name = "kaelz4async_block", - .async_compress = lz4async_block_compress, - .bound = lz4_bound, - .decompress = lz4async_block_decompress_sync, - .init = lz4_async_block_init -}; - -// 注册 LZ4 算法 -void register_lz4async_block_algorithm(void) -{ - register_algorithm(&lz4async_block_algorithm); -} \ No newline at end of file diff --git a/KAESnappy/test/kzip/alg/KAELz4Async/lz4AsyncFrame.c b/KAESnappy/test/kzip/alg/KAELz4Async/lz4AsyncFrame.c deleted file mode 100644 index 4980ef2..0000000 --- a/KAESnappy/test/kzip/alg/KAELz4Async/lz4AsyncFrame.c +++ /dev/null @@ -1,63 +0,0 @@ -#include "../manage.h" -#include -#include -#include - -static int g_has_custom_frameinfo_config = 0; // 是否 自定义 frameinfo 格式 - -static int lz4_async_frame_compress(const unsigned char *src, unsigned char *dst, lz4_async_callback cb, struct kaelz4_result *result) -{ - int ret; - if (g_has_custom_frameinfo_config == 0) { - ret = LZ4F_compressFrame_async(src, dst, cb, result, NULL); - } else { - // 初始化LZ4F压缩的参数 - LZ4F_preferences_t preferences = {0}; - preferences.frameInfo.blockSizeID = LZ4F_max64KB; // 设定块大小 - preferences.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - preferences.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; - preferences.frameInfo.contentSize = result->src_size; - ret = LZ4F_compressFrame_async(src, dst, cb, result, &preferences); - } - return ret; -} - -// 单个 LZ4 frame 格式文件的解压实现 -static int lz4_async_frame_decompress(const unsigned char *src, unsigned int *src_len, unsigned char *dst, unsigned int *dst_len) -{ - LZ4F_decompressionContext_t dctx; - LZ4F_createDecompressionContext(&dctx, 100); - size_t tmp_src_len = *src_len; - size_t tmp_dst_len = *dst_len; - int ret = LZ4F_decompress(dctx, dst, &tmp_dst_len, src, &tmp_src_len, NULL); - LZ4F_freeDecompressionContext(dctx); - *src_len = tmp_src_len; - *dst_len = tmp_dst_len; - return ret > 0 ? 0 : ret; -} -static int lz4_frame_bound(int src_len) { - // if (g_has_custom_frameinfo_config == 1) { - // return LZ4F_compressFrameBound(src_len, NULL) * 1.2; - // } - return LZ4F_compressFrameBound(src_len, NULL); -} -// LZ4 frame 初始化 -static int lz4_frame_init() { - printf("Initializing LZ4...\n"); - return 0; -} - -// LZ4 frame 算法实例 -compression_algorithm_t lz4_async_frame_algorithm = { - .name = "kaelz4async_frame", - .bound = lz4_frame_bound, - .async_compress = lz4_async_frame_compress, - .decompress = lz4_async_frame_decompress, - .init = lz4_frame_init -}; - -// 注册 LZ4 frame 算法 -void register_lz4async_frame_algorithm(void) -{ - register_algorithm(&lz4_async_frame_algorithm); -} \ No newline at end of file diff --git a/KAESnappy/test/kzip/alg/manage.c b/KAESnappy/test/kzip/alg/manage.c deleted file mode 100644 index bcb4e51..0000000 --- a/KAESnappy/test/kzip/alg/manage.c +++ /dev/null @@ -1,51 +0,0 @@ - -#include "manage.h" - -#include -#include -#include - - -#define MAX_ALGORITHMS 10 // 支持最多 10 种算法 - -static compression_algorithm_t *algorithm_list[MAX_ALGORITHMS]; -static int algorithm_count = 0; - -// 注册算法 -void register_algorithm(compression_algorithm_t *algorithm) { - if (algorithm_count < MAX_ALGORITHMS) { - algorithm_list[algorithm_count++] = algorithm; - } else { - fprintf(stderr, "Error: Too many compression algorithms registered.\n"); - } -} - -// 查找算法 -compression_algorithm_t *get_algorithm(const char *name) -{ - for (int i = 0; i < algorithm_count; i++) { - if (strcmp(algorithm_list[i]->name, name) == 0) { - return algorithm_list[i]; - } - } - return NULL; -} -int vaild_algorithm(const char *name) -{ - for (int i = 0; i < algorithm_count; i++) { - if (strcmp(algorithm_list[i]->name, name) == 0) { - return 0; - break; - } - } - printf("Error: Invalid compression algorithm: %s\n", name); - return -1; -} - -// 初始化所有算法 -void initialize_algorithms(void) { - register_lz4_algorithm(); - register_lz4_frame_algorithm(); - register_lz4async_block_algorithm(); - register_lz4async_frame_algorithm(); -} \ No newline at end of file diff --git a/KAESnappy/test/kzip/alg/manage.h b/KAESnappy/test/kzip/alg/manage.h deleted file mode 100644 index f388030..0000000 --- a/KAESnappy/test/kzip/alg/manage.h +++ /dev/null @@ -1,54 +0,0 @@ - -/* - * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. - * @Description: contain and manage all support algorithms - * @Author: Ma Xiaofeng - * @Date: 2025-3-31 - * @LastEditTime: 2025-3-31 - */ - - -#ifndef MANAGE_H -#define MANAGE_H -#include -#include - -typedef struct { - const char *name; - // 同步接口 - int (*init)(); - int (*bound)(int src_len); - // 我们约定：由框架统一读取待处理的数据以及大小。统一申请待存储的空间以及大小。 - // 压缩解压算法需要输出正确的处理后产物，输出正确的 dst_len。 - // 统一返回 0 表示算法OK - // 返回其他表示压缩解压异常 - int (*compress)(const unsigned char *src, unsigned int *src_len, - unsigned char *dst, unsigned int *dst_len); - int (*decompress)(const unsigned char *src, unsigned int *src_len, - unsigned char *dst, unsigned int *dst_len); - void (*cleanup)(); - - // 异步接口 - int (*async_compress)(const unsigned char *src, unsigned char *dst, - lz4_async_callback cb, struct kaelz4_result *result); - - int (*async_decompress)(const unsigned char* src, unsigned char *dst, - lz4_async_callback cb, struct kaelz4_result *result); -} compression_algorithm_t; - -// 注册算法 -void register_algorithm(compression_algorithm_t *algorithm); - -// 根据名称查找算法 -compression_algorithm_t *get_algorithm(const char *name); - -int vaild_algorithm(const char *name); - -// 初始化所有算法（自动注册） -void initialize_algorithms(void); - -void register_lz4_algorithm(void); -void register_lz4_frame_algorithm(void); -void register_lz4async_block_algorithm(void); -void register_lz4async_frame_algorithm(void); -#endif diff --git a/KAESnappy/test/kzip/build.sh b/KAESnappy/test/kzip/build.sh deleted file mode 100644 index 2c00308..0000000 --- a/KAESnappy/test/kzip/build.sh +++ /dev/null @@ -1,19 +0,0 @@ -export LD_LIBRARY_PATH=/usr/local/kaelz4/lib:/usr/local/kaezstd/lib:/usr/local/kaezip/lib:$LD_LIBRARY_PATH -export C_INCLUDE_PATH=/usr/local/kaelz4/include:$C_INCLUDE_PATH - -# echo "测试kaezip 和 kaelz4，请使用 sh build.sh kaelz4, 默认kaelz4" -# echo "测试kaezip 和 kaezstd，请使用 sh build.sh kaezstd" -# echo "测试qat，请使用 sh build.sh qat" - -TestEnv=$1 -TestEnv=${TestEnv:=kaelz4} -echo "build kzip for $TestEnv..." -case "$TestEnv" in - kaelz4) - gcc -g -o kzip main.c delayRecord.c datagen.c alg/manage.c alg/*/*.c scene_test_functions/*c -lz -lnuma -lrt -L/usr/local/kaelz4/lib -llz4 -DBUILD_ENV=$TestEnv -DBUILD_ENV_KAELZ4=1 -O3 -fstack-protector-all -Wall -Werror - ;; - *) - ;; -esac - -echo "build kzip done" diff --git a/KAESnappy/test/kzip/compress_ctx.h b/KAESnappy/test/kzip/compress_ctx.h deleted file mode 100644 index 36f533c..0000000 --- a/KAESnappy/test/kzip/compress_ctx.h +++ /dev/null @@ -1,86 +0,0 @@ -#ifndef _COMPRESS_CTX_H -#define _COMPRESS_CTX_H - -#include -#include -#include "alg/manage.h" -#include "scene_test_functions/entry.h" - -struct fragment_metadata { - unsigned int offset; // 分片的起始偏移量 - unsigned int len; // 分片的长度 -}; - -struct compress_out_buf { - void *buf_addr; - unsigned int len; - unsigned int sn; - struct compress_out_buf *next; - void *src; - unsigned int src_len; - uint32_t ibuf_crc; - uint32_t obuf_crc; -}; - - -struct compress_ctx; - -struct compress_param { - struct kaelz4_result result; - struct compress_ctx *ctx; - struct timeval start_time; - uint32_t ibuf_crc; - uint32_t obuf_crc; - unsigned int sn; - unsigned int loop_index; - unsigned char*src; - unsigned int src_len; - unsigned char*dst; - unsigned int dst_len; - volatile unsigned int done; -}; - -struct compress_ctx { - unsigned int loop_times; - unsigned int inflight_num; - unsigned int loop_index; - unsigned int sn; - volatile unsigned int finish_num; - void *src_buf; - unsigned long src_len; - unsigned long out_total_len; - unsigned int chunk_len; - compression_algorithm_t *algorithm; - int compress_or_decompress; - struct compress_out_buf *out_buf_list; - struct compress_out_buf *out_buf_tail; - int thread_id; - struct compress_param param_buf[1024]; - unsigned int param_index; -}; - - -struct thread_compress_args { - struct compress_ctx ctx; - const char* in_filename; - const char* out_filename; - int multi; - int window_bits; - int level; -}; - -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) -#define MIN(x, y) ((x) < (y) ? (x) : (y)) - -#if defined(__AARCH64_CMODEL_SMALL__) && __AARCH64_CMODEL_SMALL__ -#define dsb(opt) { asm volatile("dsb " #opt : : : "memory"); } -#define rmb() dsb(ld) /* read fence */ -#define wmb() dsb(st) /* write fence */ -#define mb() dsb(sy) /* rw fence */ -#else -#define rmb() __sync_synchronize() /* read fence */ -#define wmb() __sync_synchronize() /* write fence */ -#define mb() __sync_synchronize() /* rw fence */ -#endif -#endif diff --git a/KAESnappy/test/kzip/datagen.c b/KAESnappy/test/kzip/datagen.c deleted file mode 100644 index f448640..0000000 --- a/KAESnappy/test/kzip/datagen.c +++ /dev/null @@ -1,189 +0,0 @@ -/* - datagen.c - compressible data generator test tool - Copyright (C) Yann Collet 2012-2020 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/************************************** -* Includes -**************************************/ -#include "platform.h" /* Compiler options, SET_BINARY_MODE */ -#include "util.h" /* U32 */ -#include /* malloc */ -#include /* FILE, fwrite */ -#include /* memcpy */ -#include - - -/************************************** -* Constants -**************************************/ -#define KB *(1 <<10) - -#define PRIME1 2654435761U -#define PRIME2 2246822519U - - -/************************************** -* Local types -**************************************/ -#define LTLOG 13 -#define LTSIZE (1<> (32 - r))) -static unsigned int RDG_rand(U32* src) -{ - U32 rand32 = *src; - rand32 *= PRIME1; - rand32 ^= PRIME2; - rand32 = RDG_rotl32(rand32, 13); - *src = rand32; - return rand32; -} - - -static void RDG_fillLiteralDistrib(litDistribTable lt, double ld) -{ - BYTE const firstChar = ld <= 0.0 ? 0 : '('; - BYTE const lastChar = ld <= 0.0 ? 255 : '}'; - BYTE character = ld <= 0.0 ? 0 : '0'; - U32 u = 0; - - while (u lastChar) character = firstChar; - } -} - - -static BYTE RDG_genChar(U32* seed, const litDistribTable lt) -{ - U32 id = RDG_rand(seed) & LTMASK; - return (lt[id]); -} - - -#define RDG_DICTSIZE (32 KB) -#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767) -#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) -void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr) -{ - BYTE* buffPtr = (BYTE*)buffer; - const U32 matchProba32 = (U32)(32768 * matchProba); - size_t pos = prefixSize; - U32* seed = seedPtr; - - /* special case */ - while (matchProba >= 1.0) { - size_t size0 = RDG_rand(seed) & 3; - size0 = (size_t)1 << (16 + size0 * 2); - size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/ - if (buffSize < pos + size0) { - memset(buffPtr+pos, 0, buffSize-pos); - return; - } - memset(buffPtr+pos, 0, size0); - pos += size0; - buffPtr[pos-1] = RDG_genChar(seed, lt); - } - - /* init */ - if (pos==0) { - buffPtr[0] = RDG_genChar(seed, lt); - pos=1; - } - - /* Generate compressible data */ - while (pos < buffSize) { - /* Select : Literal (char) or Match (within 32K) */ - if (RDG_RAND15BITS < matchProba32) { - /* Copy (within 32K) */ - size_t match; - size_t d; - int length = RDG_RANDLENGTH + 4; - U32 offset = RDG_RAND15BITS + 1; - if (offset > pos) offset = (U32)pos; - match = pos - offset; - d = pos + length; - if (d > buffSize) d = buffSize; - while (pos < d) buffPtr[pos++] = buffPtr[match++]; - } else { - /* Literal (noise) */ - size_t d; - size_t length = RDG_RANDLENGTH; - d = pos + length; - if (d > buffSize) d = buffSize; - while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt); - } - } -} - - -void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) -{ - litDistribTable lt; - if (litProba==0.0) litProba = matchProba / 4.5; - RDG_fillLiteralDistrib(lt, litProba); - RDG_genBlock(buffer, size, 0, matchProba, lt, &seed); -} - - -#define RDG_BLOCKSIZE (128 KB) -void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed) -{ - BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE]; - U64 total = 0; - size_t genBlockSize = RDG_BLOCKSIZE; - litDistribTable lt; - - /* init */ - if (litProba==0.0) litProba = matchProba / 4.5; - RDG_fillLiteralDistrib(lt, litProba); - SET_BINARY_MODE(stdout); - - /* Generate dict */ - RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed); - - /* Generate compressible data */ - while (total < size) { - RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed); - if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total); - total += genBlockSize; - fwrite(buff, 1, genBlockSize, stdout); /* should check potential write error */ - /* update dict */ - memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); - } -} diff --git a/KAESnappy/test/kzip/datagen.h b/KAESnappy/test/kzip/datagen.h deleted file mode 100644 index c20c9c7..0000000 --- a/KAESnappy/test/kzip/datagen.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - datagen.h - compressible data generator header - Copyright (C) Yann Collet 2012-2020 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : https://github.com/lz4/lz4 - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - - -#include /* size_t */ - -void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed); -void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); -/* RDG_genOut - Generate 'size' bytes of compressible data into stdout. - Compressibility can be controlled using 'matchProba'. - 'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used. - Generated data can be selected using 'seed'. - If (matchProba, litProba and seed) are equal, the function always generate the same content. - - RDG_genBuffer - Same as RDG_genOut, but generate data into provided buffer -*/ diff --git a/KAESnappy/test/kzip/delayRecord.c b/KAESnappy/test/kzip/delayRecord.c deleted file mode 100644 index c2c2b4a..0000000 --- a/KAESnappy/test/kzip/delayRecord.c +++ /dev/null @@ -1,61 +0,0 @@ -#include -// #include - -// 定义哈希表条目的最大数量 -#define MAX_LATENCY_COUNT 3000 -// 用于存储每个时延的结构体 -typedef struct { - float latency; - int count; -} LatencyEntry; -// 哈希表，用于存储时延值及其出现次数 -LatencyEntry latencies[MAX_LATENCY_COUNT]; -int latency_count = 0; // 当前存储的时延数 -// 记录时延数据 -void record_latency(float latency) { - // 查找是否已有该时延值 - for (int i = 0; i < latency_count; i++) { - if (latencies[i].latency == latency) { - latencies[i].count++; - return; - } - } - // 如果没有找到该时延值，插入新条目 - if (latency_count < MAX_LATENCY_COUNT) { - latencies[latency_count].latency = latency; - latencies[latency_count].count = 1; - latency_count++; - } -} -// 比较函数，用于排序（按时延出现的次数从大到小） -int compare(const void *a, const void *b) { - return ((LatencyEntry *)b)->count - ((LatencyEntry *)a)->count; -} -// 获取出现次数最多的前三个时延 -void get_top_latencies(float top_latencies[3], int top_counts[3]) { - // 如果没有足够的数据，返回默认值 - if (latency_count < 1) { - top_latencies[0] = top_latencies[1] = top_latencies[2] = 0; - top_counts[0] = top_counts[1] = top_counts[2] = 0; - return; - } - - // 排序时延数据，根据 count 值降序排列 - qsort(latencies, latency_count, sizeof(LatencyEntry), compare); - - // 获取前三个时延和它们的出现次数 - for (int i = 0; i < 3 && i < latency_count; i++) { - top_latencies[i] = latencies[i].latency; - top_counts[i] = latencies[i].count; - } - - // 如果总数不足3个，剩下的用默认值填充 - for (int i = latency_count; i < 3; i++) { - top_latencies[i] = 0; - top_counts[i] = 0; - } -} - -int get_all_data_count() { - return latency_count; -} diff --git a/KAESnappy/test/kzip/delayRecord.h b/KAESnappy/test/kzip/delayRecord.h deleted file mode 100644 index 875674d..0000000 --- a/KAESnappy/test/kzip/delayRecord.h +++ /dev/null @@ -1,15 +0,0 @@ -/* - * @Copyright: Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. - * @Description: Recording the statistical delay data - * @Author: Ma Xiaofeng - * @Date: 2025-3-31 - * @LastEditTime: 2025-3-31 - */ -#ifndef DELAY_RECORD_H -#define DELAY_RECORD_H - -void record_latency(float latency); -void get_top_latencies(float top_latencies[3], int top_counts[3]); -int get_all_data_count(); - -#endif diff --git a/KAESnappy/test/kzip/kaelz4.cnf b/KAESnappy/test/kzip/kaelz4.cnf deleted file mode 100644 index 2e8a22e..0000000 --- a/KAESnappy/test/kzip/kaelz4.cnf +++ /dev/null @@ -1,2 +0,0 @@ -[LogSection] -debug_level=debug diff --git a/KAESnappy/test/kzip/main.c b/KAESnappy/test/kzip/main.c deleted file mode 100644 index dc12975..0000000 --- a/KAESnappy/test/kzip/main.c +++ /dev/null @@ -1,1220 +0,0 @@ -#define _GNU_SOURCE -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "lz4.h" -#include "alg/manage.h" -#include "delayRecord.h" -#include "compress_ctx.h" - -#include "datagen.h" /* RDG_generate */ - -#define KB *(1 << 10) -#define MB *(1 << 20) -#define GB *(1U << 30) - -int g_file_chunk_size = 0; // 测试分片大小。 默认 0kb 不分片 -int g_log_level = 0; // 打印日志级别。 0：不开启打印。 1: 压缩率打印。 2：单次时延打印 -int g_cpu_threads_per_core = 1; // 是否开启超线程。 1: 未开启。 2:开启 - -static uLong read_inputFile(const char* fileName, void** input) -{ - FILE* sourceFile = fopen(fileName, "r"); - if (sourceFile == NULL) { - fprintf(stderr, "%s not exist!\n", fileName); - return 0; - } - int fd = fileno(sourceFile); - struct stat fs; - (void)fstat(fd, &fs); - - uLong input_size = fs.st_size; - *input = malloc(input_size * sizeof(Bytef)); - if (*input == NULL) { - return 0; - } - (void)fread(*input, 1, input_size, sourceFile); - fclose(sourceFile); - - return input_size; -} -static void save_metadata_to_file( - const char *metadata_filename, struct fragment_metadata *fragments, unsigned int fragment_count) -{ - FILE *file = fopen(metadata_filename, "wb"); - if (file == NULL) { - perror("Failed to open metadata file"); - return; - } - // 写入分片数量 - fwrite(&fragment_count, sizeof(unsigned int), 1, file); - - // 写入每个分片的元数据（偏移量和长度） - for (unsigned int i = 0; i < fragment_count; i++) { - fwrite(&fragments[i], sizeof(struct fragment_metadata), 1, file); - } - - fclose(file); -} -// 从文件中读取元数据 -static void load_metadata_from_file( - const char *in_filename, struct fragment_metadata **fragments, unsigned int *fragment_count) -{ - size_t meta_file_length = strlen(in_filename) + strlen(".meta") + 1; // +1 用于'\0'结束符 - char *metadata_filename = (char *)malloc(meta_file_length); // 分配内存 - if (metadata_filename == NULL) { - perror("Failed to allocate memory"); - exit(-1); - } - // 拼接文件名 - snprintf(metadata_filename, meta_file_length, "%s.meta", in_filename); - - FILE *file = fopen(metadata_filename, "rb"); - if (file == NULL) { - perror("Failed to open metadata file"); - return; - } - // 读取分片数量 - fread(fragment_count, sizeof(unsigned int), 1, file); - // 为分片元数据分配内存 - *fragments = (struct fragment_metadata *)malloc(*fragment_count * sizeof(struct fragment_metadata)); - if (*fragments == NULL) { - perror("Failed to allocate memory for fragment metadata"); - fclose(file); - return; - } - - // 读取每个分片的元数据 - fread(*fragments, sizeof(struct fragment_metadata), *fragment_count, file); - - fclose(file); - free(metadata_filename); -} - -static struct compress_out_buf *get_buf_node_and_del_it(struct compress_out_buf **out_buf_list, unsigned int sn) -{ - struct compress_out_buf *prev = NULL; - struct compress_out_buf *current = *out_buf_list; - while (current != NULL) { - if (current->sn == sn) { - // 如果是第一个节点 - if (prev == NULL) { - // 移动头指针 - *out_buf_list = current->next; - } else { - // 删除当前节点，调整前一个节点的 next 指针 - prev->next = current->next; - } - - // 返回匹配的节点（如果需要） - return current; - } - - prev = current; - current = current->next; - } - - return NULL; -} - -static size_t write_outputFile_2(const char* outFileName, void* output, uLong output_size) -{ - FILE* outputFile = fopen(outFileName, "w"); - if (!outputFile) { - fprintf(stderr, "%s create failed!\n", outFileName); - return 0; - } - size_t count = fwrite(output, sizeof(Bytef), output_size, outputFile); - fclose(outputFile); - return count; -} - -static size_t write_outputFile(const char* outFileName, struct compress_out_buf **out_buf_list, unsigned int output_num) -{ - FILE* outputFile = fopen(outFileName, "w"); - if (!outputFile) { - fprintf(stderr, "%s create failed!\n", outFileName); - return 0; - } - - struct fragment_metadata *fragments = (struct fragment_metadata *)malloc(output_num * sizeof(struct fragment_metadata)); - if (fragments == NULL) { - perror("Failed to allocate memory"); - return -1; - } - unsigned int base_offset = 0; // 记录每个分片的偏移量。 - - size_t count = 0; - unsigned int num = 0; - while (output_num > num) { - struct compress_out_buf *out_buf_node = get_buf_node_and_del_it(out_buf_list, num); - if (out_buf_node == NULL) - break; - - fragments[num].offset = base_offset; - fragments[num].len = out_buf_node->len; // 假设每个分片固定大小 100 - base_offset += fragments[num].len; // 更新下一个分片的偏移量 - uint32_t tmp_crc = crc32(0, out_buf_node->buf_addr, out_buf_node->len); - if (out_buf_node->obuf_crc != 0 && out_buf_node->obuf_crc != tmp_crc) { - printf("Obuf crc err ! expected 0x%x recv 0x%x\n", tmp_crc, out_buf_node->obuf_crc); - return -1; - } - tmp_crc = crc32(0, out_buf_node->src, out_buf_node->src_len); - if (out_buf_node->ibuf_crc != 0 && out_buf_node->ibuf_crc != tmp_crc) { - printf("Ibuf crc err ! expected 0x%x recv 0x%x\n", tmp_crc, out_buf_node->ibuf_crc); - return -1; - } - - count += fwrite(out_buf_node->buf_addr, sizeof(Bytef), out_buf_node->len, outputFile); - num++; - - free(out_buf_node); - } - - size_t meta_file_length = strlen(outFileName) + strlen(".meta") + 1; // +1 用于'\0'结束符 - char *meta_filename = (char *)malloc(meta_file_length); // 分配内存 - if (meta_filename == NULL) { - perror("Failed to allocate memory"); - return -1; - } - // 拼接文件名 - snprintf(meta_filename, meta_file_length, "%s.meta", outFileName); - save_metadata_to_file(meta_filename, fragments, output_num); - - free(meta_filename); - fclose(outputFile); - free(fragments); - return count; -} - -static uint8_t *get_compress_input(size_t input_sz) -{ - uint8_t *inbuf = (uint8_t *)malloc(input_sz * sizeof(uint8_t)); - if (inbuf == NULL) { - return NULL; - } - - memset(inbuf, 0, input_sz); - srand((unsigned int)time(NULL)); - int i = 0; - for (i = 0; i < input_sz; i++) { - inbuf[i] = (uint8_t)rand() % 254 + 1; - } - - return inbuf; -} - -static void compress_async_polling(struct compress_param *param) -{ - struct compress_ctx *ctx = param->ctx; - - while (param->done != 1) { - // usleep(1); - } - param->done = 0; - rmb(); - if (param->loop_index > 0) { - ctx->out_total_len += param->dst_len; - ctx->finish_num++; - return; - } - - if (param->result.status != 0 && ctx->chunk_len <= 64 * 1024) { - printf("Async compress callback status not ok!\n============== exit =============\n"); - exit(-1); - } - - struct compress_out_buf *out_buf = (struct compress_out_buf *)malloc(sizeof(struct compress_out_buf)); - if (out_buf == NULL) { - return; - } - - out_buf->len = param->dst_len; - out_buf->buf_addr = param->dst; - out_buf->src_len = param->src_len; - out_buf->src = param->src; - out_buf->next = NULL; - out_buf->sn = param->sn; - out_buf->ibuf_crc = param->ibuf_crc; - out_buf->obuf_crc = param->obuf_crc; - - ctx->out_total_len += param->dst_len; - if (ctx->out_buf_tail) { - ctx->out_buf_tail->next = out_buf; - } else { - ctx->out_buf_list = out_buf; - } - ctx->out_buf_tail = out_buf; - ctx->finish_num++; -} - -static void compress_async_callback(struct kaelz4_result *result) -{ - // printf("[user]异步 callback 了！！\n"); - if (result->status != 0) { - printf("[user]回调压缩异常 : %d\n", result->status); - } - struct compress_param *param = (struct compress_param *)result->user_data; - struct timeval stop_time; - - if (unlikely(g_log_level == 1)) { - gettimeofday(&stop_time, NULL); - uLong timeonce = (stop_time.tv_sec - param->start_time.tv_sec) * 1000000 + stop_time.tv_usec - - param->start_time.tv_usec; - // printf("单次 接口调用 delay: %.3f milliseconds\n", timeonce / 1000.0); - float rounded_time_delay = round(timeonce) / 1000.0; - record_latency(rounded_time_delay); - } - param->dst_len = result->dst_len; - wmb(); - param->done = 1; - return; -} - -static int do_real_compression(struct compress_ctx *ctx, const unsigned char *src, unsigned int *src_len, - unsigned char *dst, unsigned int *dst_len, void *param) -{ - if (ctx->compress_or_decompress) { // 压缩流程。 - if (ctx->algorithm->async_compress) { - return ctx->algorithm->async_compress(src, dst, compress_async_callback, param); - } else { - return ctx->algorithm->compress(src, src_len, dst, dst_len); - } - } else { // 解压逻辑 - if (ctx->algorithm->async_decompress) { - // algorithm->async_decompress() - } else { - return ctx->algorithm->decompress(src, src_len, dst, dst_len); - } - } - return 0; -} - -static void compress_ctx_init(struct compress_ctx *ctx, int compress_or_decompress, unsigned int inflight_num, - unsigned int chunk_len, compression_algorithm_t *algorithm) -{ - ctx->algorithm = algorithm; - ctx->chunk_len = chunk_len; - ctx->sn = 0; - ctx->finish_num = 0; - ctx->compress_or_decompress = compress_or_decompress; - ctx->out_buf_list = NULL; - ctx->out_buf_tail = NULL; - ctx->out_total_len = 0; - ctx->thread_id = 0; - ctx->inflight_num = inflight_num; - if (algorithm->async_compress == NULL || compress_or_decompress == 0) { - ctx->inflight_num = 1; - } - memset(ctx->param_buf, 0, ctx->inflight_num * sizeof(struct compress_param)); - ctx->param_index = 0; -} - -static void compress_ctx_destory(struct compress_ctx *ctx) -{ - struct compress_out_buf *out_buf_list = ctx->out_buf_list; - while (out_buf_list != NULL) { - ctx->out_buf_list = ctx->out_buf_list->next; - free(out_buf_list); - out_buf_list = ctx->out_buf_list; - } - ctx->out_buf_list = NULL; - ctx->out_buf_tail = NULL; -} -#define MAX_CPUS 512 // 最大可绑核数量。 -static int g_taskset_cpus_arr[MAX_CPUS]; -static int g_config_bind_cpus_count = 0; // 配置文件中传入的绑定的cpu核心数量 -/** -* 绑核参数解析。将-b 参数解析为实际可用的cpu核数组 -* 支持逗号分割和横线范围绑核。 -* 最大支持 MAX_CPUS 个绑核。 -* @params [in] char * range 实际用户输入的-b参数。例如： 3,4,6-10,12 -* @params [out] int * tasksetCpuArr。实际解析出来的cpu绑核列表。 例如：[3,4,6,7,8,9,10,12] -* @params [out] int * bindCpusCount。实际解析出来的cpu绑核个数。默认从0开始 -* @return void -*/ -void parseTasksetLists(char *range, int *tasksetCpuArr, int *bindCpusCount){ - char *token; - token = strtok(range, ","); // 拆分解析逗号 - while (token != NULL) { - int start, end; - char *dash = strchr(token, '-'); // 查找横杠表示范围 - if (dash) { - // 解析范围，如 "1-3" - *dash = '\0'; - start = atoi(token); - end = atoi(dash + 1); - - // 将范围内的所有 CPU 加入数组 - for (int i = start; i <= end; i++) { - tasksetCpuArr[(*bindCpusCount)++] = i; - if (*bindCpusCount > MAX_CPUS) { - goto parse_config_end; - } - } - } else { - // 单个 CPU 核，如 "2" - tasksetCpuArr[(*bindCpusCount)++] = atoi(token); - if (*bindCpusCount > MAX_CPUS) { - goto parse_config_end; - } - } - token = strtok(NULL, ","); - } - parse_config_end: - *bindCpusCount = MAX_CPUS; -} -/** - * 绑定CPU亲和 - * @params [in] i 并发的进程或线程数 - * @params [in] process_or_pthread 当前启动的是进程还是线程。 1: 进程。 其他: 线程 - */ -static void set_cpu_affinity_for_child(int i, int process_or_pthread) -{ - cpu_set_t mask; - CPU_ZERO(&mask); // 清空 CPU 集合 - int use_engine_nums = 2; // 要使用的加速器数量； 可参数控制 - int super_thread_rate = g_cpu_threads_per_core; // 当前机器超线程的超级倍数；可自动读取 - int cpus_per_numa = 40 * super_thread_rate; // 每个numa对应的cpu核心数量。920 默认 40，开超线程就80。 - int coreid = (int)(i/use_engine_nums)*super_thread_rate + (cpus_per_numa * (i % use_engine_nums)) ; - if (g_config_bind_cpus_count > 0) { - if (i > g_config_bind_cpus_count) { // 如果并发数大于配置的绑核数量，直接不管多出来的并发了。 - return; - } - coreid = g_taskset_cpus_arr[i]; // 每个线程或进程使用一个固定的绑核。 - } - CPU_SET(coreid, &mask); - if (process_or_pthread == 1) { // 为进程绑核 - if (sched_setaffinity(0, sizeof(mask), &mask) == -1) { - perror("sched_setaffinity failed"); - exit(EXIT_FAILURE); - } - } else { // 为线程绑核 - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) != 0) { - perror("pthread_setaffinity_np failed"); - pthread_exit(NULL); // 退出线程 - } - } -} - -static uLong get_src_content(struct compress_ctx *ctx, const char* in_filename, void **inbuf) -{ - uLong src_len; - if (in_filename) { - fprintf(stdout, "compress filename : %s\n", in_filename); - src_len = read_inputFile(in_filename, inbuf); - } else { - *inbuf = get_compress_input(ctx->chunk_len); - src_len = ctx->chunk_len; - } - if (!*inbuf) { - fprintf(stderr, "inbuf is NULL!\n"); - return -1; - } - return src_len; -} - -static void printf_perf_data(struct compress_ctx *ctx, struct timeval start, struct timeval stop, uLong src_len, - const char* in_filename, const char* out_filename,int multi) -{ - gettimeofday(&stop, NULL); - uLong time1 = (stop.tv_sec - start.tv_sec) * 1000000 + stop.tv_usec - start.tv_usec; - - uLong stream_len = ctx->compress_or_decompress ? src_len * ctx->loop_times : ctx->out_total_len; - float speed1 = 1000000.0 / time1 * multi * stream_len / (1 << 30); - float iops = 1000.0 * ctx->sn / time1; - - printf("%s %s perf result when loop %d times: ", ctx->algorithm->name, ctx->compress_or_decompress ? - "compress" : "decompress", ctx->loop_times); - printf( - "file:%s. chunk %d kb. time used: %lu us, speed = %.3f GB/s ", in_filename, g_file_chunk_size, time1, speed1); - printf("iops = %.3fk, %s latency avg = %.3fus, latency avg per io = %.3fus\n", - iops, - ctx->compress_or_decompress ? "compress" : "decompress", - 1.0 * time1 / ctx->sn, - 1.0 * time1 / ctx->sn * ctx->inflight_num); - if (g_log_level) { - float top_latencies[3]; - int top_counts[3]; - get_top_latencies(top_latencies, top_counts); - int all_counts_num = get_all_data_count(); - printf("%s delay result: chunk %d kb. total test %d times. delay is %.3fms - %.3fms - %.3fms \n", - ctx->algorithm->name, g_file_chunk_size, all_counts_num, top_latencies[0], top_latencies[1], - top_latencies[2]); - } - - double compress_rate = (double)src_len * ctx->loop_times / ctx->out_total_len; - fprintf(stdout, "compress_size is %luB = %.3lfMB, compress_rate is %.3f\n", - ctx->out_total_len, 1.0 * (float)ctx->out_total_len / (1 << 20), compress_rate); - if (out_filename && ctx->thread_id == 0) { - write_outputFile(out_filename, &(ctx->out_buf_list), ctx->sn / ctx->loop_times); - } -} - -static int wait_for_all_fork_done() -{ - int ret = -1; - while (1) { - ret = wait(NULL); - if (ret == -1) { - if (errno == EINTR) { - continue; - } - break; - } - } - return 0; -} - -static int do_comp_and_decomp_with_full_file( - struct compress_ctx *ctx, void *inbuf, uLong src_len, void *outbuf, uLong output_sz, unsigned long *out_offset) -{ - struct compress_param *param = NULL; - - while (ctx->param_buf[ctx->param_index].done != 0) { - compress_async_polling(&ctx->param_buf[ctx->param_index]); - } - param = &ctx->param_buf[ctx->param_index]; - ctx->param_index = (ctx->param_index + 1) % ctx->inflight_num; - // 单次接口调用时延 - if (g_log_level == 1) { - gettimeofday(¶m->start_time, NULL); - } - param->done = 2; - param->ctx = ctx; - param->sn = ctx->sn; - param->loop_index = ctx->loop_index; - - uLong output_sz_tmp = output_sz; - void *dst_start = outbuf + *out_offset; - param->dst = dst_start; - param->src = inbuf; - param->src_len = src_len; - param->ibuf_crc = 0; - param->obuf_crc = 0; - if (ctx->loop_index == 0) { - param->result.ibuf_crc = ¶m->ibuf_crc; - param->result.obuf_crc = ¶m->obuf_crc; - } else { - param->result.ibuf_crc = NULL; - param->result.obuf_crc = NULL; - } - param->result.user_data = param; - param->result.src_size = src_len; - param->result.dst_len = output_sz_tmp; - int ret = do_real_compression(ctx, inbuf, (unsigned int *)&src_len, dst_start, (unsigned int *)&output_sz_tmp, ¶m->result); - if (ctx->algorithm->async_compress == NULL || ctx->compress_or_decompress == 0) { - param->result.dst_len = output_sz_tmp; - compress_async_callback(¶m->result); - } - ctx->sn++; - return ret; -} -static int do_comp_with_split_file( - struct compress_ctx *ctx, void *inbuf, uLong src_len, void *outbuf, uLong output_sz, unsigned long *out_offset) -{ - int ret = 0; - unsigned int remaining = src_len; - int chunk_size = g_file_chunk_size * 1024; - - void *start_buf = inbuf; - while (remaining > 0) { - struct compress_param *param = NULL; - - while (ctx->param_buf[ctx->param_index].done != 0) { - compress_async_polling(&ctx->param_buf[ctx->param_index]); - } - param = &ctx->param_buf[ctx->param_index]; - ctx->param_index = (ctx->param_index + 1) % ctx->inflight_num; - // 单次接口调用时延 - if (g_log_level == 1) { - gettimeofday(¶m->start_time, NULL); - } - param->done = 2; - param->ctx = ctx; - param->sn = ctx->sn; - param->loop_index = ctx->loop_index; - unsigned int chunk_src_len = (remaining > chunk_size) ? chunk_size : remaining; - unsigned int chunk_len_this_loop = chunk_src_len; // chunk_src_len 作为 src_len 传入压缩解压函数，可能会被修改 - - unsigned int output_size_chunk = ctx->algorithm->bound(chunk_size); // 实际本次压缩后产物的长度 - output_size_chunk += 4; - output_size_chunk -= output_size_chunk % 4; - uLong output_sz_tmp = output_size_chunk; - void *dst_start = outbuf + *out_offset; // 使用总内存里面的部分空间 - param->dst = dst_start; - param->src = start_buf; - param->src_len = chunk_len_this_loop; - param->ibuf_crc = 0; - param->obuf_crc = 0; - if (ctx->loop_index == 0) { - param->result.ibuf_crc = ¶m->ibuf_crc; - param->result.obuf_crc = ¶m->obuf_crc; - } else { - param->result.ibuf_crc = NULL; - param->result.obuf_crc = NULL; - } - param->result.user_data = param; - param->result.src_size = chunk_len_this_loop; - param->result.dst_len = output_sz_tmp; - ret = do_real_compression( - ctx, start_buf, (unsigned int *)&chunk_len_this_loop, dst_start, (unsigned int *)&output_sz_tmp, ¶m->result); - if (ret != 0) { - printf("Error: do_real_compression error. ret = %d \nexit\n", ret); - return ret; - } - if (ctx->algorithm->async_compress == NULL || ctx->compress_or_decompress == 0) { - param->result.dst_len = output_sz_tmp; - compress_async_callback(¶m->result); - } - ctx->sn++; - - // 更新输入和剩余长度 - start_buf += chunk_len_this_loop; - remaining -= chunk_len_this_loop; - *out_offset += output_size_chunk; - } - return ret; -} - -static int start_work(struct compress_ctx *ctx, const char* in_filename, const char* out_filename, int multi, - int window_bits, int level) -{ - void *inbuf = NULL; - uLong src_len = get_src_content(ctx, in_filename, &inbuf); - // fprintf(stdout, "input_size is %luB\n", src_len); - ctx->src_buf = inbuf; - ctx->src_len = src_len; - if (multi == 0) { multi = 1; } - - int ret = 0; - int i, j; - pid_t pid_child = 1; - fflush(stdout); - fflush(stderr); - - for (i = 0; i < multi - 1; i++) { - pid_child = fork(); - if (pid_child == 0 || pid_child == -1) { - if (g_config_bind_cpus_count > 0) { - set_cpu_affinity_for_child(i, 1); - } - break; - } - } - // 单进程下测试时不给唯一的父进程绑核。 - if (pid_child > 0) { - if (g_config_bind_cpus_count > 0 && multi > 1) { - set_cpu_affinity_for_child(multi - 1, 1); - } - } - LZ4_async_compress_init(); - - uLong output_sz; - if(ctx->compress_or_decompress) { // 压缩空间预估理论上不同算法各有自己的计算规则 - if (ctx->algorithm->bound) { - output_sz = ctx->algorithm->bound(src_len); - } else { - output_sz = compressBound(src_len) * 2; // 这是zlib的压缩后空间测算 - } - output_sz *= 2; - } else { // 解压极端情况：1、压缩率超高，需要最大250倍以上的空间。 2、压缩率超低，需要的空间最大不超过1G - output_sz = MIN(src_len * 300, 1*1024*1024*1024); - } - // tips: 对文件分片压缩时,此处针对整体文件预估的空间是不足的。在循环次数较多时，空间会不够，可以继续增大。 - void *outbuf = malloc(output_sz * sizeof(uint8_t) * 2); - if (outbuf == NULL) { - printf("Error: 申请解压空间失败: %.3f G \n", 1.0 * (output_sz * sizeof(uint8_t) * ctx->loop_times)/(1 << 30) ); - return -1; - } - - struct timeval start, stop; - gettimeofday(&start, NULL); - unsigned long out_offset = 0; // 用于选择 outbuf 填充数据的偏移值。 - // 要循环压缩解压多少次 - for (j = 0; j < ctx->loop_times; j++) { - ctx->loop_index = j; - if (j > 0) { // 为第1次之后的循环的产物复用空间 - out_offset = output_sz; - } - if (g_file_chunk_size != 0) { // 分片逻辑 - ret = do_comp_with_split_file(ctx, inbuf, src_len, outbuf, output_sz, &out_offset); - } else { // 原始文件整体丢入 - ret = do_comp_and_decomp_with_full_file(ctx, inbuf, src_len, outbuf, output_sz, &out_offset); - } - if(ret < 0) { - printf("Error: 压缩解压失败 ret=%d \n", ret); - } - } - - while (ctx->sn != ctx->finish_num) { - while (ctx->param_buf[ctx->param_index].done != 0) { - compress_async_polling(&ctx->param_buf[ctx->param_index]); - } - - ctx->param_index = (ctx->param_index + 1) % ctx->inflight_num; - } - - if (pid_child > 0 && ret >= 0) { - ret = wait_for_all_fork_done(); - - printf_perf_data(ctx, start, stop, src_len, in_filename, out_filename, multi); - printf("\nall process done====================。 \n \n"); - } - - free(inbuf); - free(outbuf); - - return ret; -} -static void *start_work_thread(void *arg) -{ - struct thread_compress_args *args = (struct thread_compress_args *)arg; - struct compress_ctx *ctx = &args->ctx; - const char *in_filename = args->in_filename; - const char *out_filename = args->out_filename; - int multi = args->multi; - int window_bits = args->window_bits; - int level = args->level; - start_work(ctx, in_filename, out_filename, multi, window_bits, level); - - compress_ctx_destory(&args->ctx); - free(args); // 释放 args - return NULL; -} -// 分片解压。 -// 读取本地源数据信息。读取本地总文件信息。逐个块解压 --> 写文件拼接到一起 -static int start_work_decompress( - struct compress_ctx *ctx, const char *in_filename, const char *out_filename, int multi, int window_bits, int level) -{ - void *inbuf = NULL; - uLong src_len = get_src_content(ctx, in_filename, &inbuf); - - // 从文件读取元数据 - struct fragment_metadata *loaded_fragments = NULL; - unsigned int fragment_count; - load_metadata_from_file(in_filename, &loaded_fragments, &fragment_count); - // // // 打印读取的元数据 - // for (unsigned int i = 0; i < fragment_count; i++) { - // printf("Fragment %d: Offset = %u, Length = %u\n", i + 1, loaded_fragments[i].offset, loaded_fragments[i].len); - // } - - ctx->src_buf = inbuf; - ctx->src_len = src_len; - if (multi == 0) { multi = 1; } - - uLong output_sz = MIN(src_len * 300, 1*1024*1024*1024); - void *outbuf = malloc(output_sz * sizeof(uint8_t) * 2); - if (outbuf == NULL) { - printf("Error: 申请解压空间失败: %.3f G \n", 1.0 * (output_sz * sizeof(uint8_t) * ctx->loop_times)/(1 << 30) ); - return -1; - } - - int ret; - int i, j; - pid_t pid_child = 1; - fflush(stdout); - fflush(stderr); - - for (i = 0; i < multi - 1; i++) { - pid_child = fork(); - if (pid_child == 0 || pid_child == -1) { - break; - } - } - - struct timeval start, stop; - gettimeofday(&start, NULL); - - size_t out_offset = 0; // 总内存中的偏移，每一小块儿使用不同的偏移。 - // 解压仅1次 - for (j = 0; j < ctx->loop_times; j++) { - if (j > 0) { // 为第1次之后的循环的产物复用空间 - out_offset = output_sz; - } - if (g_file_chunk_size != 0) { // 分片逻辑 - for (int k = 0; k < fragment_count; k++) { - size_t this_offset = loaded_fragments[k].offset; - size_t this_src_len = loaded_fragments[k].len; - - struct compress_param *param = NULL; - - while (ctx->param_buf[ctx->param_index].done != 0) { - compress_async_polling(&ctx->param_buf[ctx->param_index]); - } - param = &ctx->param_buf[ctx->param_index]; - ctx->param_index = (ctx->param_index + 1) % ctx->inflight_num; - // 单次接口调用时延 - if (g_log_level == 1) { - gettimeofday(¶m->start_time, NULL); - } - param->done = 2; - param->ctx = ctx; - param->sn = ctx->sn; - param->loop_index = j; - - size_t output_size_chunk = MIN(this_src_len * 300, 1*1024*1024*1024); // 预估本次压缩后产物的长度 - void *dst_start = outbuf + out_offset; // 使用总内存里面的部分空间 - param->dst = dst_start; - param->src = inbuf + this_offset; - param->src_len = this_src_len; - param->ibuf_crc = 0; - param->obuf_crc = 0; - if (ctx->loop_index == 0) { - param->result.ibuf_crc = ¶m->ibuf_crc; - param->result.obuf_crc = ¶m->obuf_crc; - } else { - param->result.ibuf_crc = NULL; - param->result.obuf_crc = NULL; - } - param->result.user_data = param; - param->result.src_size = this_src_len; - param->result.dst_len = output_size_chunk; - ret = do_real_compression(ctx, inbuf + this_offset, (unsigned int *)&this_src_len, - dst_start, (unsigned int *)&output_size_chunk, ¶m->result); - - if(ret != 0) { - printf("Error: sn %d len=%d;offset=%lx. end=%lx.do_real_compression decomp error. ret = %d \nexit\n ", - param->sn, loaded_fragments[k].len, this_offset, this_offset + loaded_fragments[k].len, ret); - write_outputFile_2( - "./error-split-file-compressed-data.compressed", inbuf + this_offset, loaded_fragments[k].len); - return ret; - } - if (ctx->algorithm->async_compress == NULL || ctx->compress_or_decompress == 0) { - param->result.dst_len = output_size_chunk; - compress_async_callback(¶m->result); - } - ctx->sn++; - - out_offset += output_size_chunk; // 偏移本次解压实际使用的空间 - } - } - } - - while (ctx->sn != ctx->finish_num) { - while (ctx->param_buf[ctx->param_index].done != 0) { - compress_async_polling(&ctx->param_buf[ctx->param_index]); - } - - ctx->param_index = (ctx->param_index + 1) % ctx->inflight_num; - usleep(10); - } - - if (pid_child > 0) { - ret = wait_for_all_fork_done(); - - printf_perf_data(ctx, start, stop, src_len, in_filename, out_filename, multi); - printf("\nall decompress done==================== \n \n"); - } - free(inbuf); - free(outbuf); - - return ret; -} -static void format_cpu_env(char *str) -{ - char *token = strtok(str, "-"); - int count = 0; - - while (token != NULL) { - count++; - if (count == 3) { - // 获取第3个字段并转换为整数 - g_cpu_threads_per_core = atoi(token); - break; // 一旦获取到第3个字段，跳出循环 - } - token = strtok(NULL, "-"); - } -} -static void auto_get_parent_cpu_affinity(int *arr, int *count) -{ - // 子线程中获取父进程的亲和性 - cpu_set_t parent_affinity; - if (sched_getaffinity(0, sizeof(cpu_set_t), &parent_affinity) == -1) { - perror("sched_getaffinity"); - return; - } - // 打印父进程的亲和性 - for (int i = 0; i < CPU_SETSIZE; i++) { - if (CPU_ISSET(i, &parent_affinity)) { - arr[(*count)++] = i; - } - } -} - -static void init_env_config() -{ - char *taskset_cpu_config = getenv("USER_BIND_CPU_CONFIG"); - if (taskset_cpu_config != NULL) { - parseTasksetLists(taskset_cpu_config, g_taskset_cpus_arr, &g_config_bind_cpus_count); - } else { - auto_get_parent_cpu_affinity(g_taskset_cpus_arr, &g_config_bind_cpus_count); - } -} - -int compare_files(char *file_un, char *file_de) { - FILE* file1 = fopen(file_un, "rb"); - FILE* file2 = fopen(file_de, "rb"); - int result = 0; - while (1) { - char byte1 = fgetc(file1); // 从文件中读取一个字节 - char byte2 = fgetc(file2); - - if (byte1 != byte2) { // 读出字节不同，文件内容不同 - result = 1; - break; - } - - if (feof(file1) || feof(file2)) { - break; // 任意一个文件到达末尾 - } - } - // 检查两个文件是否同时到达末尾 - if (!feof(file1) || !feof(file2)) { - result = 1; // 文件长度不同，内容不同 - } - - fclose(file1); - fclose(file2); - return result; -} - -int round_trip_fuzztest(uint32_t RDGseed) -{ - uint64_t RDGsize = 64 * 1024; // 默认值64K - double RDGlitProba = 0.0; // lit分布概率，一般不用改 - int RGDproba = 50; // 数据可压缩程度 0-100 - - int compress = 1; - int ret = 0; - int multi = 1; - int threadNum = 1; - int loop_times = 1000; - int inflight_num = 256; - uLong chunk_len = 1024; - int window_bits = 15; - int level = 1; - - char algorithm_name[25] = {0}; - - // Fuzz 测试循环次数 - uint64_t run_times = 104104 * 2; - - char *algorithm_options[2] = {"kaelz4async_block", "kaelz4async_frame"}; - int file_chunk_size_options[13] = {2 KB, 4 KB, 8 KB, 16 KB, 32 KB, 64 KB, 128 KB, 256 KB, 512 KB, 1 MB, 2 MB, 4 MB, 8 MB}; - int random_data_size_options[13] = {16 KB, 32 KB, 64 KB, 64 KB + 5, 96 KB, 128 KB, 128 KB + 5, 512 KB, 1 MB, 4 MB, 16 MB, 64 MB, 128 MB}; - int compress_proba_options[11] = {0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}; - int muti_options[1] = {1}; - int threadNum_options[1] = {1}; - int loop_times_options[4] = {1, 1000, 2000, 5000}; - int inflight_num_options[7] = {24, 48, 64, 128, 256, 512, 1024}; - - int params_num = 8; - int option_counts[8] = {2, 13, 13, 11, 1, 1, 4, 7}; // 每个参数的选项数量 - int indices[8] = {0}; // 每个参数的当前选项索引 - - for (int i = 0; i < run_times; ++i) { - - compress = (i + 1) % 2; // 先压缩后解压 - char* input_filename = (compress == 1) ? "uncomp_data" : "comped_data"; - char* output_filename = (compress == 1) ? "comped_data" : "decomp_data"; - - // 迭代法遍历各种参数组合 - if (compress == 1) { // 每次测试压缩时读入参数，解压时维持压缩时参数 - strcpy(algorithm_name, algorithm_options[indices[0]]); - g_file_chunk_size = file_chunk_size_options[indices[1]]; - g_file_chunk_size /= 1024; // 转换成KB - RDGsize = random_data_size_options[indices[2]]; - RGDproba = compress_proba_options[indices[3]]; - multi = muti_options[indices[4]]; - threadNum = threadNum_options[indices[5]]; - loop_times = loop_times_options[indices[6]]; - inflight_num = inflight_num_options[indices[7]]; - - // 生成随机数据 - char *RDGbuffer = (char *)malloc(RDGsize); - - RDG_genBuffer(RDGbuffer, RDGsize, (double)(RGDproba / 100), RDGlitProba, RDGseed); - FILE *file_in = fopen(input_filename, "wb"); - - size_t written = fwrite(RDGbuffer, sizeof(char), RDGsize, file_in); - if (written != RDGsize) { - printf("Error: write file failed\n"); - return 0; - } - fclose(file_in); - free(RDGbuffer); - // 更新迭代索引 - int j = params_num - 1; - while(j >= 0) { - indices[j]++; - if (indices[j] < option_counts[j]) { - break; - } - indices[j] = 0; - j--; - } - if (j < 0) { - printf("seed : %d Fuzz test done, seed + 1.\n", RDGseed); - ++RDGseed; - } - } - - const char* in_filename = input_filename; - const char* out_filename = output_filename; - chunk_len *= 1024; - printf("kzip fuzz test: algorithm: %s, multi process %d, threadNum %d, data length: %lu(KB), loop times: %d, window_bits : %d, level : %d, chunk: %d(KB), %s, in_file : %s, out_file : %s,\n", - algorithm_name, multi, threadNum, RDGsize/1024, loop_times, window_bits, level, g_file_chunk_size, compress == 1 ? "compress" : "decompress", in_filename, out_filename); - - compression_algorithm_t *algorithm = get_algorithm(algorithm_name); - if (!algorithm) { - printf("Error: Algorithm %s not found.\n", algorithm_name); - return -1; - } - struct compress_ctx ctx; - compress_ctx_init(&ctx, compress, inflight_num, chunk_len, algorithm); - ctx.loop_times = loop_times; - if (!ctx.compress_or_decompress) { - ctx.loop_times = 1; - multi = 1; - } - // LZ4_async_compress_init(); - if (!ctx.compress_or_decompress && g_file_chunk_size > 0) { // 如果是分片解压，单独处理 - ret = start_work_decompress(&ctx, in_filename, out_filename, multi, window_bits, level); - } else { - if (threadNum > 1) { - pthread_t threads[threadNum]; - int j; - for (j = 0; j < threadNum; j++) { - struct thread_compress_args *args = malloc(sizeof(struct thread_compress_args)); - compress_ctx_init(&args->ctx, compress, inflight_num, chunk_len, algorithm); - args->ctx.thread_id = j; - args->ctx.loop_times = loop_times; - args->in_filename = in_filename; - args->out_filename = out_filename; - args->multi = multi; - args->window_bits = window_bits; - args->level = level; - if (pthread_create(&threads[j], NULL, start_work_thread, args) != 0) { - perror("pthread_create failed"); - exit(EXIT_FAILURE); - } - } - for (j = 0; j < threadNum; j++) { - pthread_join(threads[j], NULL); - } - } else { - ret = start_work(&ctx, in_filename, out_filename, multi, window_bits, level); - } - } - if (ret < 0) { - printf("kzip fuzz failed ! seed : %u, %s, file : %s\n", RDGseed, compress == 1 ? "compress" : "decompress", in_filename); - return ret; - } - if (!compress) { // 解压后比较两个文件内容是否一致 - ret = compare_files("uncomp_data", "decomp_data"); // 返回值1时表示文件不同，返回值0时表示文件相同 - if (ret != 0) { - printf("Error: find difference between uncomp_data and decomp_data! \n"); - compress_ctx_destory(&ctx); - return ret; - } - } - compress_ctx_destory(&ctx); - } - printf("kzip fuzz test done. all test succeess.\n"); - return ret; -} - -static void usage(void) -{ - printf("usage: \n"); - printf(" -A: set algorithm type(kaelz4|kaelz4_frame|kaelz4async_block|kaelz4async_frame). default is kaelz4\n"); - printf(" -m: multi process. default is 2. use fork() to start multi process\n"); - printf(" -t: thread num. default is 1. if thread num > 1, use pthread_create for multi compression. \n"); - printf(" -l: stream length(KB)\n"); - printf(" -i: inflight num for calling async compression at same time. default 16\n"); - printf(" -w: window_bits\n"); - printf(" -v: compress level(1~9)\n"); - printf(" -f: input filename(-l useless if this work)\n"); - printf(" -o: output filename\n"); - printf(" -n: loop times\n"); - printf(" -s: input file split chunk size(KB)\n"); - printf(" -d: compress or decompress\n"); - printf(" -F: start fuzz test, and set random data seed, default seed : 0 \n"); - printf(" example: ./kzip -A kaelz4 -m 2 -f ./kzip -o ./kzip.compressd -n 1000\n"); - printf(" ./kzip -A kaelz4 -d -m 2 -f ./kzip.compressd -o ./kzip.origin -n 1000\n"); -} -int main(int argc, char **argv) -{ - // 初始化所有算法 - initialize_algorithms(); - init_env_config(); - - const char *optstring = "dm:l:n:w:f:o:v:A:hg:s:c:i:t:T:F:"; - int ret = 0; - int o = 0; - int multi = 2; - int level = 6; - uLong chunk_len = 1024; - int loop_times = 1000; - int compress = 1; - int window_bits = 15; - char input_filename[64] = {0}; - char output_filename[64] = {0}; - char algorithm_name[25] = "kaelz4"; - char cpuConfigStr[20] = "4-40-1-4"; // 默认的920B配置：未开超线程，使用全部的4个加速器。 - int inflight_num = 256; - int threadNum = 1; - int isSceneTest = 0; - // Fuzz RDG - int fuzztest = 0; - uint32_t RDGseed = 0; // 随机数据生成种子 - - while ((o = getopt(argc, argv, optstring)) != -1) { - if(optstring == NULL) continue; - switch (o) { - case 'T': - isSceneTest = atoi(optarg); - break; - case 'A': - strcpy(algorithm_name, optarg); - break; - case 'm': - multi = atoi(optarg); - break; - case 't': - threadNum = atoi(optarg); - break; - case 'i': - inflight_num = atoi(optarg); - if (inflight_num > 1024) - inflight_num = 1024; - break; - case 'g': - g_log_level = atoi(optarg); - break; - case 's': - g_file_chunk_size = atoi(optarg); - break; - case 'l': - chunk_len = atoi(optarg); - break; - case 'v': - level = atoi(optarg); - break; - case 'n': - loop_times = atoi(optarg); - break; - case 'w': - window_bits = atoi(optarg); - break; - case 'd': - compress = 0; - break; - case 'f': - strcpy(input_filename, optarg); - break; - case 'o': - strcpy(output_filename, optarg); - break; - case 'c': - strcpy(cpuConfigStr, optarg); - break; - case 'F': - fuzztest = 1; - RDGseed = (uint32_t)atoi(optarg); - break; - case 'h': - usage(); - return 0; - } - } - - ret = vaild_algorithm(algorithm_name); - if(ret != 0) { return ret; } - - if (argc <= 1) { - usage(); - printf("\ndefault input parameter used\n"); - } - format_cpu_env(cpuConfigStr); - - const char* in_filename = input_filename[0] == 0 ? NULL : input_filename; - const char* out_filename = output_filename[0]== 0 ? NULL : output_filename; - chunk_len *= 1024; - - if (isSceneTest == 1) { - return scene_tests_run(); - } - if (fuzztest == 1) { - ret = round_trip_fuzztest(RDGseed); - return ret; - } - - printf("kzip perf parameter: algorithm: %s, multi process %d, threadNum %d, stream length: %lu(KB), loop times: %d, window_bits : %d, level : %d, chunk: %d\n", - algorithm_name, multi, threadNum, chunk_len/1024, loop_times, window_bits, level, g_file_chunk_size); - - // 获取用户指定的算法 - compression_algorithm_t *algorithm = get_algorithm(algorithm_name); - - if (!algorithm) { - printf("Error: Algorithm %s not found.\n", algorithm_name); - return -1; - } - - struct compress_ctx ctx; - compress_ctx_init(&ctx, compress, inflight_num, chunk_len, algorithm); - ctx.loop_times = loop_times; - // if (!ctx.compress_or_decompress) { - // ctx.loop_times = 1; - // multi =1; - // } - - if (!ctx.compress_or_decompress && g_file_chunk_size > 0) { // 如果是分片解压，单独处理 - ret = start_work_decompress(&ctx, in_filename, out_filename, multi, window_bits, level); - } else { - if (threadNum > 1) { - pthread_t threads[threadNum]; - int j; - for (j = 0; j < threadNum; j++) { - struct thread_compress_args *args = malloc(sizeof(struct thread_compress_args)); - compress_ctx_init(&args->ctx, compress, inflight_num, chunk_len, algorithm); - args->ctx.thread_id = j; - args->ctx.loop_times = loop_times; - args->in_filename = in_filename; - args->out_filename = out_filename; - args->multi = multi; - args->window_bits = window_bits; - args->level = level; - if (pthread_create(&threads[j], NULL, start_work_thread, args) != 0) { - perror("pthread_create failed"); - exit(EXIT_FAILURE); - } - } - for (j = 0; j < threadNum; j++) { - pthread_join(threads[j], NULL); - } - } else { - ret = start_work(&ctx, in_filename, out_filename, multi, window_bits, level); - } - } - LZ4_teardown_async_compress(); - compress_ctx_destory(&ctx); - return ret; -} diff --git a/KAESnappy/test/kzip/platform.h b/KAESnappy/test/kzip/platform.h deleted file mode 100644 index 43a171b..0000000 --- a/KAESnappy/test/kzip/platform.h +++ /dev/null @@ -1,155 +0,0 @@ -/* - platform.h - compiler and OS detection - Copyright (C) 2016-2020, Przemyslaw Skibinski, Yann Collet - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ - -#ifndef PLATFORM_H_MODULE -#define PLATFORM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/* ************************************** -* Compiler Options -****************************************/ -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ -# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ -# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -# endif -#endif - - -/* ************************************** -* Detect 64-bit OS -* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros -****************************************/ -#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ - || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ - || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ - || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ - || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ - || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ - || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ - || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ -# if !defined(__64BIT__) -# define __64BIT__ 1 -# endif -#endif - - -/* ********************************************************* -* Turn on Large Files support (>4GB) for 32-bit Linux/Unix -***********************************************************/ -#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ -# if !defined(_FILE_OFFSET_BITS) -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# endif -# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ -# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ -# endif -# if defined(_AIX) || defined(__hpux) -# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ -# endif -#endif - - -/* ************************************************************ -* Detect POSIX version -* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows -* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX -* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION -************************************************************** */ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ - || defined(__midipix__) || defined(__VMS)) -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \ - || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__MidnightBSD__) /* BSD distros */ \ - || defined(__HAIKU__) -# define PLATFORM_POSIX_VERSION 200112L -# else -# if defined(__linux__) || defined(__linux) -# ifndef _POSIX_C_SOURCE -# define _POSIX_C_SOURCE 200809L /* use feature test macro */ -# endif -# endif -# include /* declares _POSIX_VERSION */ -# if defined(_POSIX_VERSION) /* POSIX compliant */ -# define PLATFORM_POSIX_VERSION _POSIX_VERSION -# else -# define PLATFORM_POSIX_VERSION 0 -# endif -# endif -#endif -#if !defined(PLATFORM_POSIX_VERSION) -# define PLATFORM_POSIX_VERSION -1 -#endif - - -/*-********************************************* -* Detect if isatty() and fileno() are available -*********************************************** */ -#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__) -# include /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#elif defined(WIN32) || defined(_WIN32) -# include /* _isatty */ -# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include /* FILE */ -static __inline int IS_CONSOLE(FILE* stdStream) -{ - DWORD dummy; - return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); -} -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - -/****************************** -* OS-specific Includes -***************************** */ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) -# include /* _O_BINARY */ -# include /* _setmode, _fileno, _get_osfhandle */ -# if !defined(__DJGPP__) -# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ -# include /* FSCTL_SET_SPARSE */ -# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } -# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } -# else -# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) -# define SET_SPARSE_FILE_MODE(file) -# endif -#else -# define SET_BINARY_MODE(file) -# define SET_SPARSE_FILE_MODE(file) -#endif - - - -#if defined (__cplusplus) -} -#endif - -#endif /* PLATFORM_H_MODULE */ diff --git a/KAESnappy/test/kzip/runAffinityFunc.sh b/KAESnappy/test/kzip/runAffinityFunc.sh deleted file mode 100644 index 87ccb9b..0000000 --- a/KAESnappy/test/kzip/runAffinityFunc.sh +++ /dev/null @@ -1,77 +0,0 @@ -export LD_LIBRARY_PATH=/usr/local/kaelz4/lib/:/usr/local/kaezstd/lib/:/usr/local/kaezip/lib/:$LD_LIBRARY_PATH -export KAE_LZ4_WINTYPE=8 -export KAE_LZ4_COMP_TYPE=8 -export KAE_LZ4_ASYNC_THREAD_NUM=6 - -echo "亲和性测试开始" -sh build.sh kaelz4 - -Algthm=("kaelz4async_block" "kaelz4async_frame") -Datasets=("calgary" "itemdata" "ooffice" "osdb" "samba" "webster" "xml" "x-ray") -BlockSize=("4" "8" "60""128" "512" "1024") -Cpu_core_ranges=("0-20" "40-79" "0-79" "120-159") - -current_time=$(date +"%Y-%m-%d_%H-%M-%S") -LogFile=kaelz4-function.log.$current_time -testFilePath=../../../scripts/compressTestDataset - - -for da in "${Datasets[@]}"; do - for alg in "${Algthm[@]}"; do - for bs in "${BlockSize[@]}"; do - echo "Executing: $da $alg $bs testing" - testFile="$testFilePath/$da" - testFileComped="$testFile.compressed" - testFileOrigin="$testFile.origin" - rm -rf $testFileComped - rm -rf $testFileOrigin - rm -rf $testFileComped.meta - rm -rf $testFileOrigin.meta - for cpu_cores in "${Cpu_core_ranges[@]}"; do - echo "Binding CPU : $cpu_cores Testing" - taskset -c "$cpu_cores" ./kzip -A $alg -m 1 -f $testFile -n 30000 -s $bs -i 16 >> $LogFile & # 压缩测试 - disown # 丢弃后台进程在被kill时的bash报错 - sleep 0.5 - pid=$! - numa0=0 # 自适应亲和性测试记录KAE线程所处的numa - numa1=0 - for thread in $(ls /proc/$pid/task); do - thread_id=$(basename $thread) - current_core=$(cat "/proc/$pid/task/$thread_id/stat" | awk '{print $39}') - - start_core=$(echo "$cpu_cores" | cut -d'-' -f1) - end_core=$(echo "$cpu_cores" | cut -d'-' -f2) - if [[ "$current_core" -ge "$start_core" && "$core" -le "$end_core" ]]; then - echo "$thread running in $current_core, pass" - else - echo "Error: $thread set in $cpu_cores ,runing in $current_core" - echo "$alg, Dataset : $da, BlockSize : $bs" - exit 1 - fi - # 自适应亲和性测试。计算0-79绑核时，KAE线程落在2个numa上的数量 - if [[ "$cpu_cores" == "0-79" ]]; then - # 主进程不参与计算 - if [[ "$thread" -ne "$pid" && $current_core -ge 0 && $current_core -le 39 ]]; then - ((numa0++)) - elif [[ "$thread" -ne "$pid" && $current_core -ge 40 && $current_core -le 79 ]]; then - ((numa1++)) - fi - fi - done - if [[ "$cpu_cores" == "0-79" ]]; then - echo "SUCCESS: all threads running in $cpu_cores." - if [[ "$numa0" -eq "$numa1" ]]; then - echo "SUCCESS: KAE threads average test: $numa0 in numa0, $numa1 in numa1." - else - echo "Warning: KAE threads: $numa0 in numa0, $numa1 in numa1." - fi - else - echo "SUCCESS: all threads running in $cpu_cores" - fi - kill -9 $pid - done - done - done -done - -echo "亲和性测试结束" \ No newline at end of file diff --git a/KAESnappy/test/kzip/runError.sh b/KAESnappy/test/kzip/runError.sh deleted file mode 100644 index 3a86a0c..0000000 --- a/KAESnappy/test/kzip/runError.sh +++ /dev/null @@ -1,113 +0,0 @@ -export LD_LIBRARY_PATH=/usr/local/kaelz4/lib/:/usr/local/kaezstd/lib/:/usr/local/kaezip/lib/:$LD_LIBRARY_PATH -export KAE_LZ4_WINTYPE=8 -export KAE_LZ4_COMP_TYPE=8 - -sh build.sh kaelz4 - -# 异常以及可靠性测试 -# 1、KAE硬件队列资源消耗完时，自动切软算。 -# 2、K异步接口在KAE资源不可用时，自动切软算。 - -uninstall_driver() -{ - rmmod hisi_zip -} -install_driver() -{ - modprobe hisi_zip perf_mode=1 uacce_mode=2 pf_q_num=256 -} - -set_driver_less_and_runout() -{ - rmmod hisi_zip - sleep 1 - modprobe hisi_zip uacce_mode=2 pf_q_num=2 - sleep 1 - taskset -c 300-312 ./kzip -A kaelz4 -m 8 -n 400000000 -s 4 1>/dev/null 2>/dev/null & -} -reset_driver() -{ - rmmod hisi_zip - modprobe hisi_zip perf_mode=1 uacce_mode=2 pf_q_num=256 -} - -do_compress_and_decompress_and_diff() -{ - Algthm=("kaelz4async_block" "kaelz4async_frame") - Datasets=("calgary" "itemdata" "ooffice" "osdb" "samba" "webster" "xml" "x-ray") - Datasets=("calgary" "itemdata") - BlockSize=("4" "8" "16" "60" "64") - - current_time=$(date +"%Y-%m-%d_%H-%M-%S") - LogFile=kaelz4-function.log.$current_time - testFilePath=../../../scripts/compressTestDataset - - diffFile() { - local testFile=$1 - local testFileOrigin=$2 - if [[ ! -f "$testFile" ]]; then - echo "Error: 压缩异常!未成功压缩文件" - fi - if [[ ! -f "$testFileOrigin" ]]; then - echo "Error: 解压异常!未成功解压文件" - fi - diffRes=$(diff $testFile $testFileOrigin) - if [[ -n "$diffRes" ]] ; then - echo "Error: 解压后数据与原始数据比对不通过！！" - else - echo "Success: 测试通过 解压数据校验通过" - fi - } - - for da in "${Datasets[@]}"; do - for alg in "${Algthm[@]}"; do - for bs in "${BlockSize[@]}"; do - echo "Executing: $da $alg $bs kb testing" - testFile="$testFilePath/$da" - testFileComped="$testFile.compressed" - testFileOrigin="$testFile.origin" - rm -rf $testFileComped - rm -rf $testFileOrigin - rm -rf $testFileComped.meta - rm -rf $testFileOrigin.meta - ./kzip -A $alg -m 2 -f $testFile -o $testFileComped -n 2 -s $bs -i 256 >> $LogFile # 压缩测试 - ./kzip -d -A $alg -m 1 -f $testFileComped -o $testFileOrigin -n 2 -s $bs -i 256 >> $LogFile # 压缩测试 - diffFile $testFile $testFileOrigin - done - done - done -} - -test_func_ok_when_run_out_of_KAE() -{ - echo "start testing async lz4 compress when hardware queue is run out>>>>>>>>>>>>>>>" - set_driver_less_and_runout - sleep 2 - echo "hardware queue left 0. start tesing when data is less than 64k>>>>>>>>>>>>>>>" - do_compress_and_decompress_and_diff 2>/dev/null - echo "all testing done>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - sleep 2 - killall kzip - reset_driver - echo "env is reset." -} -test_func_ok_when_unavaliable_of_KAE() { - echo "start testing async lz4 compress when hardware queue is unavaliable>>>>>>>>>>>>>" - uninstall_driver - echo "hardware queue no exists. start tesing when data is less than 64k>>>>>>>>>>>>>>>" - do_compress_and_decompress_and_diff 2>/dev/null - echo "all testing done>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - install_driver - echo "env is reset." -} - -main() -{ - test_func_ok_when_run_out_of_KAE - - sleep 5 - - test_func_ok_when_unavaliable_of_KAE -} - -main diff --git a/KAESnappy/test/kzip/runFunc.sh b/KAESnappy/test/kzip/runFunc.sh deleted file mode 100644 index 391d776..0000000 --- a/KAESnappy/test/kzip/runFunc.sh +++ /dev/null @@ -1,54 +0,0 @@ -export LD_LIBRARY_PATH=/usr/local/kaelz4/lib/:/usr/local/kaezstd/lib/:/usr/local/kaezip/lib/:$LD_LIBRARY_PATH -export KAE_LZ4_WINTYPE=8 -export KAE_LZ4_COMP_TYPE=8 - - -sh build.sh kaelz4 - -Algthm=("kaelz4" "kaelz4_frame" "kaelz4async_block" "kaelz4async_frame") -Datasets=("calgary" "itemdata" "dickens" "mozilla" "mr" "nci" "ooffice" "osdb" "reymont" "samba" "sao" "webster" "xml" "x-ray") -Datasets=("calgary" "itemdata" "ooffice" "osdb" "samba" "webster" "xml" "x-ray") -BlockSize=("0" "4" "8" "16" "60" "64" "68" "128" "512" "1024" "2090" "10244") - -current_time=$(date +"%Y-%m-%d_%H-%M-%S") -LogFile=kaelz4-function.log.$current_time -testFilePath=../../../scripts/compressTestDataset - -diffFile() { - local testFile=$1 - local testFileOrigin=$2 - if [[ ! -f "$testFile" ]]; then - echo "Error: 压缩异常!未成功压缩文件" - exit 1 - fi - if [[ ! -f "$testFileOrigin" ]]; then - echo "Error: 解压异常!未成功解压文件" - exit 1 - fi - diffRes=$(diff $testFile $testFileOrigin) - if [[ -n "$diffRes" ]] ; then - echo "Error: 解压后数据与原始数据比对不通过！！" - else - echo "Success: 测试通过 解压数据校验通过" - fi -} - -for da in "${Datasets[@]}"; do - for alg in "${Algthm[@]}"; do - for bs in "${BlockSize[@]}"; do - echo "Executing: $da $alg $bs kb chunk testing" - testFile="$testFilePath/$da" - testFileComped="$testFile.compressed" - testFileOrigin="$testFile.origin" - rm -rf $testFileComped - rm -rf $testFileOrigin - rm -rf $testFileComped.meta - rm -rf $testFileOrigin.meta - ./kzip -A $alg -m 2 -f $testFile -o $testFileComped -n 2 -s $bs -i 256 >> $LogFile # 压缩测试 - ./kzip -d -A $alg -m 1 -f $testFileComped -o $testFileOrigin -n 2 -s $bs -i 256 >> $LogFile # 压缩测试 - diffFile $testFile $testFileOrigin - done - done -done - -echo "功能测试结束" \ No newline at end of file diff --git a/KAESnappy/test/kzip/runLogfunc.sh b/KAESnappy/test/kzip/runLogfunc.sh deleted file mode 100644 index 81e576f..0000000 --- a/KAESnappy/test/kzip/runLogfunc.sh +++ /dev/null @@ -1,56 +0,0 @@ -export LD_LIBRARY_PATH=/usr/local/kaelz4/lib/:/usr/local/kaezstd/lib/:/usr/local/kaezip/lib/:$LD_LIBRARY_PATH -export KAE_LZ4_WINTYPE=8 -export KAE_LZ4_COMP_TYPE=8 -export KAELZ4_CONF_ENV=/var/log/ - -sh build.sh kaelz4 - -Algthm=("kaelz4async_block" "kaelz4async_frame") -Datasets=("calgary" "itemdata" "dickens" "mozilla" "mr" "nci" "ooffice" "osdb" "reymont" "samba" "sao" "webster" "xml" "x-ray") -Datasets=("calgary" "itemdata" "ooffice" "osdb" "samba" "webster" "xml" "x-ray") -BlockSize=("4" "8" "16" "60" "64" "68" "128" "512" "1024" "2090" "10244") - -current_time=$(date +"%Y-%m-%d_%H-%M-%S") -LogFile=kaelz4-logfunction.log.$current_time -testFilePath=../../../scripts/compressTestDataset -search_content="do polling" # debug等级日志信息里，异步压缩回收硬件压缩结果标志 -FILE_PATH="/var/log/kaelz4.log" # 日志文件路径 - -echo "日志测试开始" - -rm -f /var/log/kaelz4.cnf -cp ./kaelz4.cnf /var/log/ - -for da in "${Datasets[@]}"; do - for alg in "${Algthm[@]}"; do - for bs in "${BlockSize[@]}"; do - echo "Executing: $da $alg $bs testing" - testFile="$testFilePath/$da" - testFileComped="$testFile.compressed" - testFileOrigin="$testFile.origin" - rm -rf $testFileComped - rm -rf $testFileOrigin - rm -rf $testFileComped.meta - rm -rf $testFileOrigin.meta - echo " " > $FILE_PATH - - ./kzip -A $alg -m 1 -f $testFile -o $testFileComped -n 2 -s $bs -i 256 >> $LogFile # 压缩测试 - - if [ ! -f "$FILE_PATH" ]; then - echo "压缩日志文件 $FILE_PATH 不存在！" - exit 1 - fi - - if grep -q "$search_content" "$FILE_PATH"; then - sleep 0.1 - else - echo "日志文件中不包含指定内容：$search_content" - exit 1 - fi - wait - done - done -done - -rm -f /var/log/kaelz4.cnf -echo "日志功能正常，日志测试结束" diff --git a/KAESnappy/test/kzip/runPerf.sh b/KAESnappy/test/kzip/runPerf.sh deleted file mode 100644 index 831d843..0000000 --- a/KAESnappy/test/kzip/runPerf.sh +++ /dev/null @@ -1,101 +0,0 @@ -export LD_LIBRARY_PATH=/usr/local/kaelz4/lib/:/usr/local/kaezstd/lib/:/usr/local/kaezip/lib/:$LD_LIBRARY_PATH -export KAE_LZ4_WINTYPE=8 -export KAE_LZ4_COMP_TYPE=8 - -# 使用 getopts 解析命令行参数 -while getopts "m:l:n:w:f:o:v:A:h:g:s:c:i:t:p:" opt; do - case $opt in - A) # 要测试的算法 - Alg="$OPTARG" - ;; - m) # 支持的进程数量 - multiProcess="$OPTARG" - ;; - s) # 文件分片处理. 默认0kb - fileChunk="$OPTARG" - ;; - n) # 循环压缩次数处理。 默认100次 - loppTimes="$OPTARG" - ;; - i) - inflightNum="$OPTARG" - ;; - t) - threadsNum="$OPTARG" - ;; - p) - testFile="$OPTARG" - ;; - *) - echo "Usage: all params m:l:n:w:f:o:v:A:h:g:s:c:" - exit 1 - ;; - esac -done - -Alg=${Alg:=kaezip} -multiProcess=${multiProcess:=2} -fileChunk=${fileChunk:=0} -loppTimes=${loppTimes:=1} -inflightNum=${inflightNum:=64} -threadsNum=${threadsNum:=1} -testFile=${testFile:="../../../scripts/compressTestDataset/calgary"} - -buildParams="kaelz4" -sh build.sh $buildParams - -numa_nodes=$(lscpu | grep -i "numa node(s)" | awk '{print $NF}') -threads_per_core=$(lscpu | grep -i "Thread(s) per core" | awk '{print $NF}') -cpu_range=$(lscpu | grep -i "NUMA node0 CPU(s)" | awk '{print $NF}') -cpu_count_per_numa=40 -if [[ $cpu_range =~ ([0-9]+)-([0-9]+) ]]; then - start_cpu=${BASH_REMATCH[1]} - end_cpu=${BASH_REMATCH[2]} - cpu_count_per_numa=$((end_cpu - start_cpu + 1)) -fi -cpu1_range=$(lscpu | grep -i "NUMA node1 CPU(s)" | awk '{print $NF}') -if [[ $cpu1_range =~ ([0-9]+)-([0-9]+) ]]; then - cpu1_start_cpu=${BASH_REMATCH[1]} - cpu1_end_cpu=${BASH_REMATCH[2]} -fi -useEngineNumsForThisTest=2 # 本次测试使用的加速器的数量 -# 本次测试使用的CPU配置。 机器中numa总量-单个numa的cpu核心数量-超线程的单核线程倍数-本次测试使用的加速器数量 -cpuConfigStr="$numa_nodes-$cpu_count_per_numa-$threads_per_core-$useEngineNumsForThisTest" -# 获取使用前两个Numa的话，CPU的绑核范围 -bindCpu0AndCpu1="$start_cpu-$cpu1_end_cpu" -if [[ $useEngineNumsForThisTest == 1 ]]; then - bindCpu0AndCpu1="$start_cpu-$end_cpu" -fi - - -testFileComped="$testFile.compressed" -testFileOrigin="$testFile.origin" -rm -rf $testFileComped -rm -rf $testFileOrigin -rm -rf $testFileComped.meta -rm -rf $testFileOrigin.meta - -echo "taskset -c $bindCpu0AndCpu1 ./kzip -A $Alg -m $multiProcess -f $testFile -o $testFileComped -c $cpuConfigStr -n $loppTimes -s $fileChunk -i $inflightNum -t $threadsNum" -echo "./kzip -d -A $Alg -m $multiProcess -f $testFileComped -o $testFileOrigin -c $cpuConfigStr -n $loppTimes -s $fileChunk -i $inflightNum" - -date -# gdb --args -taskset -c $bindCpu0AndCpu1 ./kzip -A $Alg -m $multiProcess -f $testFile -o $testFileComped -c $cpuConfigStr -n $loppTimes -s $fileChunk -i $inflightNum -t $threadsNum -date - -./kzip -d -A $Alg -m $multiProcess -f $testFileComped -o $testFileOrigin -c $cpuConfigStr -n $loppTimes -s $fileChunk -i $inflightNum -date -if [[ ! -f "$testFile" ]]; then - echo "Error: 压缩异常!未成功压缩文件" - exit 1 -fi -if [[ ! -f "$testFileOrigin" ]]; then - echo "Error: 解压异常!未成功解压文件" - exit 1 -fi -diffRes=$(diff $testFile $testFileOrigin) -if [[ -n "$diffRes" ]] ; then - echo "Error: 解压后数据与原始数据比对不通过！！" -else - echo "Success: 解压数据校验通过" -fi diff --git a/KAESnappy/test/kzip/scene_test_functions/entry.c b/KAESnappy/test/kzip/scene_test_functions/entry.c deleted file mode 100644 index 63b81f1..0000000 --- a/KAESnappy/test/kzip/scene_test_functions/entry.c +++ /dev/null @@ -1,32 +0,0 @@ -#include - -// Declare test functions -void test_async_frame_with_perferences_0(); -void test_async_frame_with_perferences_1(); -void test_async_frame_with_perferences_2(); -void test_async_frame_with_perferences_3(); -void test_async_frame_with_perferences_4(); - -typedef struct { - const char *test_name; - void (*test_func)(void); -} test_case_t; - -test_case_t test_cases[] = { - {"异步frame压缩 基本没有参数测试", test_async_frame_with_perferences_0}, - {"异步frame压缩带 contentChecksum 测试", test_async_frame_with_perferences_1}, - {"异步frame压缩带 blockChecksum 测试", test_async_frame_with_perferences_2}, - {"异步frame压缩带 contentSize 测试", test_async_frame_with_perferences_3}, - {"异步frame压缩带 blockChecksum 和 contentChecksum 和 contentSize 测试", test_async_frame_with_perferences_4}, -}; - -#define NUM_TESTS (sizeof(test_cases) / sizeof(test_case_t)) - -int scene_tests_run() -{ - for (int i = 0; i < NUM_TESTS; i++) { - printf("Running: %s\n", test_cases[i].test_name); - test_cases[i].test_func(); - } - return 0; -} \ No newline at end of file diff --git a/KAESnappy/test/kzip/scene_test_functions/entry.h b/KAESnappy/test/kzip/scene_test_functions/entry.h deleted file mode 100644 index 6c44703..0000000 --- a/KAESnappy/test/kzip/scene_test_functions/entry.h +++ /dev/null @@ -1,7 +0,0 @@ - -#ifndef _SCENE_TEST_FUNCTIONS_ -#define _SCENE_TEST_FUNCTIONS_ - -int scene_tests_run(); - -#endif diff --git a/KAESnappy/test/kzip/scene_test_functions/test_async_frame.c b/KAESnappy/test/kzip/scene_test_functions/test_async_frame.c deleted file mode 100644 index 162d022..0000000 --- a/KAESnappy/test/kzip/scene_test_functions/test_async_frame.c +++ /dev/null @@ -1,201 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -int g_has_done = 0; // 异步回调是否完成。需要初始化为0。 -int g_test_file = 0; // - -struct my_custom_data { - void *src; - void *dst; - void *src_decompd; - size_t src_len; - size_t dst_len; - size_t src_decompd_len; -}; - -// 随机生成256KB的数据 -static void generate_random_data(void *data, size_t size) { - unsigned char *bytes = (unsigned char *)data; - for (size_t i = 0; i < size; i++) { - bytes[i] = rand() % 256; // 随机生成字节 - } -} -static size_t read_inputFile(const char* fileName, void** input) -{ - FILE* sourceFile = fopen(fileName, "r"); - if (sourceFile == NULL) { - fprintf(stderr, "%s not exist!\n", fileName); - return 0; - } - int fd = fileno(sourceFile); - struct stat fs; - (void)fstat(fd, &fs); - - size_t input_size = fs.st_size; - *input = malloc(input_size); - if (*input == NULL) { - return 0; - } - (void)fread(*input, 1, input_size, sourceFile); - fclose(sourceFile); - - return input_size; -} -// static size_t write_outputFile_2(const char* outFileName, void* output, size_t output_size) -// { -// FILE* outputFile = fopen(outFileName, "w"); -// if (!outputFile) { -// fprintf(stderr, "%s create failed!\n", outFileName); -// return 0; -// } -// size_t count = fwrite(output, 1, output_size, outputFile); -// fclose(outputFile); -// return count; -// } - -void compression_callback(struct kaelz4_result *result) { - if (result->status != 0) { - printf("Compression failed with status: %d\n", result->status); - return; - } - - // 在回调中获取压缩后的数据 - struct my_custom_data *my_data = (struct my_custom_data *)result->user_data; - void *compressed_data = my_data->dst; - size_t compressed_size = result->dst_len; - - my_data->dst_len = compressed_size; - - // 使用LZ4解压缩数据 - size_t tmp_src_len = result->src_size * 10; - // 为解压数据分配内存 - void *dst_buffer = malloc(tmp_src_len); - if (!dst_buffer) { - printf("Memory allocation failed for decompressed data.\n"); - return; - } - - LZ4F_decompressionContext_t dctx; - LZ4F_createDecompressionContext(&dctx, 100); - int ret = LZ4F_decompress(dctx, dst_buffer, &tmp_src_len, - compressed_data, &compressed_size, NULL); - if (ret < 0) { - printf("Decompression failed with error code: %d\n", ret); - free(dst_buffer); - return; - } - my_data->src_decompd = dst_buffer; - my_data->src_decompd_len = tmp_src_len; - - if (my_data->src_decompd_len != my_data->src_len) { - printf("Test Error: 解压后与原始长度不一样. result->src_size=%ld 原始长度=%ld 压缩后解压长度=%ld \n", - result->src_size, - my_data->src_len, - my_data->src_decompd_len); - } - - // 比较解压后的数据和原始数据 - if (memcmp(my_data->src_decompd, my_data->src, result->src_size) == 0) { - printf("Test Success.\n"); - } else { - printf("Test Error:Decompressed data does not match the original data.\n"); - } - - // 释放解压后的数据 - free(dst_buffer); - g_has_done = 1; -} - -static int test_async_frame_with_perferences(int contentChecksumFlag, int blockChecksumFlag, int contentSizeFlag) -{ - g_has_done = 0; - size_t src_len = 258 * 1024; // 256KB - void *inbuf = malloc(src_len); - if (!inbuf) { - printf("Memory allocation failed for input data.\n"); - return -1; - } - // 生成随机数据 - generate_random_data(inbuf, src_len); - - if (g_test_file) { - src_len = read_inputFile("../../../scripts/compressTestDataset/x-ray", &inbuf); - } - - - // 为压缩数据分配内存 - size_t compressed_size = LZ4F_compressBound(src_len, NULL); - void *compressed_data = malloc(compressed_size); - if (!compressed_data) { - printf("Memory allocation failed for compressed data.\n"); - free(inbuf); - return -1; - } - - // 初始化LZ4F压缩的参数 - LZ4F_preferences_t preferences = {0}; - preferences.frameInfo.blockSizeID = LZ4F_max64KB; // 设定块大小 - if (contentChecksumFlag) { - preferences.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; - } - if (blockChecksumFlag) { - preferences.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; - } - if (contentSizeFlag) { - preferences.frameInfo.contentSize = src_len; - } - - // 异步压缩 - struct kaelz4_result result = {0}; - struct my_custom_data mydata = {0}; - mydata.src = inbuf; - mydata.src_len = src_len; - mydata.dst = compressed_data; - result.user_data = &mydata; - result.src_size = src_len; - result.dst_len = compressed_size; - LZ4_async_compress_init(); - int compression_status = LZ4F_compressFrame_async(inbuf, compressed_data, - compression_callback, &result, &preferences); - - if (compression_status != 0) { - printf("Compression failed with error code: %d\n", compression_status); - free(inbuf); - free(compressed_data); - return -1; - } - - while (g_has_done != 1) { - usleep(100); - } - LZ4_teardown_async_compress(); - - return compression_status; -} -int test_async_frame_with_perferences_0() -{ - return test_async_frame_with_perferences(0, 0, 0); -} - -int test_async_frame_with_perferences_1() -{ - return test_async_frame_with_perferences(1, 0, 0); -} -int test_async_frame_with_perferences_2() -{ - return test_async_frame_with_perferences(0, 1, 0); -} -int test_async_frame_with_perferences_3() -{ - return test_async_frame_with_perferences(0, 0, 1); -} -int test_async_frame_with_perferences_4() -{ - return test_async_frame_with_perferences(1, 1, 1); -} diff --git a/KAESnappy/test/kzip/util.h b/KAESnappy/test/kzip/util.h deleted file mode 100644 index 3192ddc..0000000 --- a/KAESnappy/test/kzip/util.h +++ /dev/null @@ -1,697 +0,0 @@ -/* - util.h - utility functions - Copyright (C) 2016-2020, Przemyslaw Skibinski, Yann Collet - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ - -#ifndef UTIL_H_MODULE -#define UTIL_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/*-**************************************** -* Dependencies -******************************************/ -#include "platform.h" /* PLATFORM_POSIX_VERSION */ -#include /* size_t, ptrdiff_t */ -#include /* malloc */ -#include /* strlen, strncpy */ -#include /* fprintf, fileno */ -#include -#include /* stat, utime */ -#include /* stat */ -#if defined(_WIN32) -# include /* utime */ -# include /* _chmod */ -#else -# include /* chown, stat */ -# if PLATFORM_POSIX_VERSION < 200809L -# include /* utime */ -# else -# include /* AT_FDCWD */ -# include /* for utimensat */ -# endif -#endif -#include /* time */ -#include /* INT_MAX */ -#include - - - -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif - - -/* ************************************************************ -* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW -***************************************************************/ -#if defined(_MSC_VER) && (_MSC_VER >= 1400) -# define UTIL_fseek _fseeki64 -#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define UTIL_fseek fseeko -#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) -# define UTIL_fseek fseeko64 -#else -# define UTIL_fseek fseek -#endif - - -/*-**************************************** -* Sleep functions: Windows - Posix - others -******************************************/ -#if defined(_WIN32) -# include -# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) -# define UTIL_sleep(s) Sleep(1000*s) -# define UTIL_sleepMilli(milli) Sleep(milli) -#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ -# include -# include /* setpriority */ -# include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ -# if defined(PRIO_PROCESS) -# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20) -# else -# define SET_REALTIME_PRIORITY /* disabled */ -# endif -# define UTIL_sleep(s) sleep(s) -# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ -# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } -# else -# define UTIL_sleepMilli(milli) /* disabled */ -# endif -#else -# define SET_REALTIME_PRIORITY /* disabled */ -# define UTIL_sleep(s) /* disabled */ -# define UTIL_sleepMilli(milli) /* disabled */ -#endif - - -/*-**************************************** -* stat() functions -******************************************/ -#if defined(_MSC_VER) -# define UTIL_TYPE_stat __stat64 -# define UTIL_stat _stat64 -# define UTIL_fstat _fstat64 -# define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG) -#elif defined(__MINGW32__) && defined (__MSVCRT__) -# define UTIL_TYPE_stat _stati64 -# define UTIL_stat _stati64 -# define UTIL_fstat _fstati64 -# define UTIL_STAT_MODE_ISREG(st_mode) ((st_mode) & S_IFREG) -#else -# define UTIL_TYPE_stat stat -# define UTIL_stat stat -# define UTIL_fstat fstat -# define UTIL_STAT_MODE_ISREG(st_mode) (S_ISREG(st_mode)) -#endif - - -/*-**************************************** -* fileno() function -******************************************/ -#if defined(_MSC_VER) -# define UTIL_fileno _fileno -#else -# define UTIL_fileno fileno -#endif - -/* ************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) - - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ -#endif -#if defined(__GNUC__) -# define UTIL_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define UTIL_STATIC static inline -#elif defined(_MSC_VER) -# define UTIL_STATIC static __inline -#else -# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - - -/*-**************************************** -* Allocation functions -******************************************/ -/* - * A modified version of realloc(). - * If UTIL_realloc() fails the original block is freed. -*/ -UTIL_STATIC void* UTIL_realloc(void* ptr, size_t size) -{ - void* const newptr = realloc(ptr, size); - if (newptr) return newptr; - free(ptr); - return NULL; -} - - -/*-**************************************** -* String functions -******************************************/ -/* - * A modified version of realloc(). - * If UTIL_realloc() fails the original block is freed. -*/ -UTIL_STATIC int UTIL_sameString(const char* a, const char* b) -{ - assert(a!=NULL && b!=NULL); /* unsupported scenario */ - if (a==NULL) return 0; - if (b==NULL) return 0; - return !strcmp(a,b); -} - - -/*-**************************************** -* Time functions -******************************************/ -#if defined(_WIN32) /* Windows */ - - typedef LARGE_INTEGER UTIL_time_t; - UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) - { - static LARGE_INTEGER ticksPerSecond; - static int init = 0; - if (!init) { - if (!QueryPerformanceFrequency(&ticksPerSecond)) - fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n"); - init = 1; - } - return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; - } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) - { - static LARGE_INTEGER ticksPerSecond; - static int init = 0; - if (!init) { - if (!QueryPerformanceFrequency(&ticksPerSecond)) - fprintf(stderr, "ERROR: QueryPerformanceFrequency() failure\n"); - init = 1; - } - return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; - } - -#elif defined(__APPLE__) && defined(__MACH__) - - #include - typedef U64 UTIL_time_t; - UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) - { - static mach_timebase_info_data_t rate; - static int init = 0; - if (!init) { - mach_timebase_info(&rate); - init = 1; - } - return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom)) / 1000ULL; - } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) - { - static mach_timebase_info_data_t rate; - static int init = 0; - if (!init) { - mach_timebase_info(&rate); - init = 1; - } - return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom); - } - -#elif (PLATFORM_POSIX_VERSION >= 200112L) && (defined __UCLIBC__ || (defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) || __GLIBC__ > 2) ) ) - - #include - typedef struct timespec UTIL_time_t; - UTIL_STATIC UTIL_time_t UTIL_getTime(void) - { - UTIL_time_t now; - if (clock_gettime(CLOCK_MONOTONIC, &now)) - fprintf(stderr, "ERROR: Failed to get time\n"); /* we could also exit() */ - return now; - } - UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end) - { - UTIL_time_t diff; - if (end.tv_nsec < begin.tv_nsec) { - diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec; - diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec; - } else { - diff.tv_sec = end.tv_sec - begin.tv_sec; - diff.tv_nsec = end.tv_nsec - begin.tv_nsec; - } - return diff; - } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end) - { - UTIL_time_t const diff = UTIL_getSpanTime(begin, end); - U64 micro = 0; - micro += 1000000ULL * diff.tv_sec; - micro += diff.tv_nsec / 1000ULL; - return micro; - } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end) - { - UTIL_time_t const diff = UTIL_getSpanTime(begin, end); - U64 nano = 0; - nano += 1000000000ULL * diff.tv_sec; - nano += diff.tv_nsec; - return nano; - } - -#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ - - typedef clock_t UTIL_time_t; - UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } -#endif - - -/* returns time span in microseconds */ -UTIL_STATIC U64 UTIL_clockSpanMicro(UTIL_time_t clockStart) -{ - UTIL_time_t const clockEnd = UTIL_getTime(); - return UTIL_getSpanTimeMicro(clockStart, clockEnd); -} - -/* returns time span in nanoseconds */ -UTIL_STATIC U64 UTIL_clockSpanNano(UTIL_time_t clockStart) -{ - UTIL_time_t const clockEnd = UTIL_getTime(); - return UTIL_getSpanTimeNano(clockStart, clockEnd); -} - -UTIL_STATIC void UTIL_waitForNextTick(void) -{ - UTIL_time_t const clockStart = UTIL_getTime(); - UTIL_time_t clockEnd; - do { - clockEnd = UTIL_getTime(); - } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0); -} - - - -/*-**************************************** -* File functions -******************************************/ -#if defined(_MSC_VER) - #define chmod _chmod - typedef struct __stat64 stat_t; -#else - typedef struct stat stat_t; -#endif - - -UTIL_STATIC int UTIL_isRegFile(const char* infilename); -UTIL_STATIC int UTIL_isRegFD(int fd); - - -UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) -{ - int res = 0; - - if (!UTIL_isRegFile(filename)) - return -1; - - { -#if defined(_WIN32) || (PLATFORM_POSIX_VERSION < 200809L) - struct utimbuf timebuf; - timebuf.actime = time(NULL); - timebuf.modtime = statbuf->st_mtime; - res += utime(filename, &timebuf); /* set access and modification times */ -#else - struct timespec timebuf[2]; - memset(timebuf, 0, sizeof(timebuf)); - timebuf[0].tv_nsec = UTIME_NOW; - timebuf[1].tv_sec = statbuf->st_mtime; - res += utimensat(AT_FDCWD, filename, timebuf, 0); /* set access and modification times */ -#endif - } - -#if !defined(_WIN32) - res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ -#endif - - res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */ - - errno = 0; - return -res; /* number of errors is returned */ -} - - -UTIL_STATIC int UTIL_getFDStat(int fd, stat_t *statbuf) -{ - int r; -#if defined(_MSC_VER) - r = _fstat64(fd, statbuf); - if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ -#else - r = fstat(fd, statbuf); - if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ -#endif - return 1; -} - -UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) -{ - int r; -#if defined(_MSC_VER) - r = _stat64(infilename, statbuf); - if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ -#else - r = stat(infilename, statbuf); - if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ -#endif - return 1; -} - - -UTIL_STATIC int UTIL_isRegFD(int fd) -{ - stat_t statbuf; -#ifdef _WIN32 - /* Windows runtime library always open file descriptors 0, 1 and 2 in text mode, therefore we can't use them for binary I/O */ - if(fd < 3) return 0; -#endif - return UTIL_getFDStat(fd, &statbuf); /* Only need to know whether it is a regular file */ -} - - -UTIL_STATIC int UTIL_isRegFile(const char* infilename) -{ - stat_t statbuf; - return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ -} - - -UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) -{ - int r; - stat_t statbuf; -#if defined(_MSC_VER) - r = _stat64(infilename, &statbuf); - if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; -#else - r = stat(infilename, &statbuf); - if (!r && S_ISDIR(statbuf.st_mode)) return 1; -#endif - return 0; -} - - -UTIL_STATIC U64 UTIL_getOpenFileSize(FILE* file) -{ - int r; - int fd; - struct UTIL_TYPE_stat statbuf; - - fd = UTIL_fileno(file); - if (fd < 0) { - perror("fileno"); - exit(1); - } - r = UTIL_fstat(fd, &statbuf); - if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - -UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) -{ - int r; - struct UTIL_TYPE_stat statbuf; - - r = UTIL_stat(infilename, &statbuf); - if (r || !UTIL_STAT_MODE_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - -UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) -{ - U64 total = 0; - unsigned n; - for (n=0; n= *bufEnd) { - ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; - *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); - *bufEnd = *bufStart + newListSize; - if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } - } - if (*bufStart + *pos + pathLength < *bufEnd) { - strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos)); - *pos += pathLength + 1; - nbFiles++; - } - } - free(path); - } while (FindNextFileA(hFile, &cFile)); - - FindClose(hFile); - assert(nbFiles < INT_MAX); - return (int)nbFiles; -} - -#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ -# define UTIL_HAS_CREATEFILELIST -# include /* opendir, readdir */ -# include /* strerror, memcpy */ - -UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd) -{ - DIR* dir; - struct dirent * entry; - size_t dirLength; - int nbFiles = 0; - - if (!(dir = opendir(dirName))) { - fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); - return 0; - } - - dirLength = strlen(dirName); - errno = 0; - while ((entry = readdir(dir)) != NULL) { - char* path; - size_t fnameLength, pathLength; - if (strcmp (entry->d_name, "..") == 0 || - strcmp (entry->d_name, ".") == 0) continue; - fnameLength = strlen(entry->d_name); - path = (char*)malloc(dirLength + fnameLength + 2); - if (!path) { closedir(dir); return 0; } - memcpy(path, dirName, dirLength); - path[dirLength] = '/'; - memcpy(path+dirLength+1, entry->d_name, fnameLength); - pathLength = dirLength+1+fnameLength; - path[pathLength] = 0; - - if (UTIL_isDirectory(path)) { - nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd); /* Recursively call "UTIL_prepareFileList" with the new path. */ - if (*bufStart == NULL) { free(path); closedir(dir); return 0; } - } else { - if (*bufStart + *pos + pathLength >= *bufEnd) { - size_t const newListSize = (size_t)(*bufEnd - *bufStart) + LIST_SIZE_INCREASE; - *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); - *bufEnd = *bufStart + newListSize; - if (*bufStart == NULL) { free(path); closedir(dir); return 0; } - } - if (*bufStart + *pos + pathLength < *bufEnd) { - strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos)); - *pos += pathLength + 1; - nbFiles++; - } - } - free(path); - errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ - } - - if (errno != 0) { - fprintf(stderr, "readdir(%s) error: %s\n", dirName, strerror(errno)); - free(*bufStart); - *bufStart = NULL; - } - closedir(dir); - return nbFiles; -} - -#else - -UTIL_STATIC int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd) -{ - (void)bufStart; (void)bufEnd; (void)pos; - fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); - return 0; -} - -#endif /* #ifdef _WIN32 */ - -/* - * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, - * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb). - * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) - * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. - */ -UTIL_STATIC const char** -UTIL_createFileList(const char** inputNames, unsigned inputNamesNb, - char** allocatedBuffer, unsigned* allocatedNamesNb) -{ - size_t pos; - unsigned i, nbFiles; - char* buf = (char*)malloc(LIST_SIZE_INCREASE); - size_t bufSize = LIST_SIZE_INCREASE; - const char** fileTable; - - if (!buf) return NULL; - - for (i=0, pos=0, nbFiles=0; i= bufSize) { - while (pos + len >= bufSize) bufSize += LIST_SIZE_INCREASE; - buf = (char*)UTIL_realloc(buf, bufSize); - if (!buf) return NULL; - } - assert(pos + len < bufSize); - memcpy(buf + pos, inputNames[i], len); - pos += len; - nbFiles++; - } else { - char* bufend = buf + bufSize; - nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend); - if (buf == NULL) return NULL; - assert(bufend > buf); - bufSize = (size_t)(bufend - buf); - } } - - if (nbFiles == 0) { free(buf); return NULL; } - - fileTable = (const char**)malloc(((size_t)nbFiles+1) * sizeof(const char*)); - if (!fileTable) { free(buf); return NULL; } - - for (i=0, pos=0; i bufSize) { - free(buf); - free((void*)fileTable); - return NULL; - } /* can this happen ? */ - - *allocatedBuffer = buf; - *allocatedNamesNb = nbFiles; - - return fileTable; -} - - -UTIL_STATIC void -UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) -{ - free(allocatedBuffer); - free((void*)filenameTable); -} - - -#if defined (__cplusplus) -} -#endif - -#endif /* UTIL_H_MODULE */ diff --git a/KAESnappy/test/perftest/Readme.md b/KAESnappy/test/perftest/Readme.md deleted file mode 100644 index 7d9739d..0000000 --- a/KAESnappy/test/perftest/Readme.md +++ /dev/null @@ -1,26 +0,0 @@ -## 本目录用于KAELz4单P带宽能力测试和基于lzbench工具的性能测试 - -### 1. 单p带宽测试 -step1: 执行下列命令 -``` -export LD_LIBRARY_PATH=/usr/local/kaelz4/lib/:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH -``` - -step2: 在本目录执行make，即可得到kaelz4_perf可执行文件，根据提示输入参数可测试单P下KAELz4带宽上限 - -### 2. lzbench工具性能测试 - -step1: 从[Gitee仓](https://gitee.com/kunpeng_compute/lzbench)获取Kunpeng-lzbench测试工具源码，进入源码路径执行`make`，编译得到二进制工具，根据指定测试命令即可测试KAELz4性能 - -step2: 测试block模式性能 -``` -numactl -C 1 ./lzbench -relz4 -b128 -i1 -j -m1024 /数据集目录 -``` - -step3: 测试frame模式性能 -``` -numactl -C 1 ./lzbench -relz4frame -b128 -i1 -j -m1024 /数据集目录 -``` - -注：执行上述性能测试前，请确保kaelz4已安装完成 \ No newline at end of file diff --git a/KAESnappy/test/perftest/kaelz4_perf.c b/KAESnappy/test/perftest/kaelz4_perf.c deleted file mode 100644 index a67f469..0000000 --- a/KAESnappy/test/perftest/kaelz4_perf.c +++ /dev/null @@ -1,362 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include // presumes lz4 library is installed -#define __USE_GNU -#include -#include - -uint8_t *g_inbuf = NULL; -int g_threadnum = 15; -int core_seq = 2; // 绑核开始的 cpuid。 -int g_show_perf_print = 0; // 是否展示性能数据，默认0不展示。 1：仅展示最后一次时延和压缩比例。 2：展示每次的单次时延 -int g_use_cpu_number = 1; // 使用的加速器数量。使得绑核值会均匀分到几个加速器上 - -enum CompressFunc { - LZ4_BLOCKCOMPRESS, - LZ4_STREAMINGCOMPRESS, -}; - -struct ThreadArgs { - int blocksize; - int streamLen; - int cLevel; - int loopTimes; - int core_id; -}; - -// 获取指定大小的随机数据流 -uint8_t *CompressInputGet(size_t inputSize) -{ - uint8_t *inbuf = (uint8_t *)malloc(inputSize * sizeof(uint8_t)); - if (inbuf == NULL) { - printf("%s malloc failed\n", __func__); - return NULL; - } - - memset(inbuf, 0, inputSize); - srand((unsigned int)time(NULL)); - int i = 0; - for (i = 0; i < inputSize; i++) { - inbuf[i] = (uint8_t)rand() % 254 + 1; - } - - return inbuf; -} -static int read_inputFile(const char *fileName, void **input) -{ - FILE *sourceFile = fopen(fileName, "r"); - if (sourceFile == NULL) { - fprintf(stderr, "%s not exist!\n", fileName); - return 0; - } - int fd = fileno(sourceFile); - struct stat fs; - (void)fstat(fd, &fs); - - int input_size = fs.st_size; - *input = malloc(input_size); - if (*input == NULL) { - return 0; - } - (void)fread(*input, 1, input_size, sourceFile); - fclose(sourceFile); - - return input_size; -} - -// 块压缩模式：数据切块随后调用LZ4压缩接口 -static void DoBlockCompressPerf_next(int streamLen, int cLevel, int bsize) -{ - int inpOffset = 0; - int totalout = 0; - int totalIn = streamLen; - uint64_t timeonce; - - for (;;) { - int inpBytes = 0; - char tmp = (char)g_inbuf[inpOffset]; - char *src = &tmp; - if (streamLen <= 0) { - break; - } - if (streamLen >= bsize) { - inpOffset += bsize; - inpBytes = bsize; - } else { - inpOffset += streamLen; - inpBytes = streamLen; - } - streamLen -= inpBytes; - - char *const dst = (char *)malloc(LZ4_COMPRESSBOUND(inpBytes)); - struct timeval startOneTime, stopOneTime; - gettimeofday(&startOneTime, NULL); - const int cmpBytes = LZ4_compress_fast(src, dst, inpBytes, LZ4_COMPRESSBOUND(inpBytes), cLevel); - gettimeofday(&stopOneTime, NULL); - timeonce = (stopOneTime.tv_sec - startOneTime.tv_sec) * 1000000 + stopOneTime.tv_usec - startOneTime.tv_usec; - if (g_show_perf_print == 2) { - printf("单次 时延: %.4f milliseconds\n", timeonce / 1000.0); - } - - totalout += cmpBytes; - - free(dst); - - if (cmpBytes <= 0) { - break; - printf("LZ4 compress error\n"); - } - } - - if (g_show_perf_print == 1) { - float compressRate = (float)totalout / (float)totalIn; - // 单次时延和压缩比数据计算 - printf("the last compression delay is %.2f milliseconds. compress rate = %.3f \n ", - timeonce / 1000.0, - compressRate); - } -} - -void *thread_function(void *arg) -{ - struct ThreadArgs *args = (struct ThreadArgs *)arg; - int streamLen = args->streamLen; - int cLevel = args->cLevel; - int loopTimes = args->loopTimes; - int core_id = args->core_id; - int bsize = args->blocksize; - // 绑核操作 - cpu_set_t cpuSet; - CPU_ZERO(&cpuSet); // 清空cpuSet - // 将线程绑定到第0个CPU内核 - CPU_SET(core_id, &cpuSet); - - // 设置CPU亲和性 - if (pthread_setaffinity_np(pthread_self(), sizeof(cpuSet), &cpuSet) == -1) { - fprintf(stderr, "Failed to set CPU affinity\n"); - return NULL; - } - - for (int i = 0; i < loopTimes; ++i) { - DoBlockCompressPerf_next(streamLen, cLevel, bsize); - } - - return NULL; -} - -void DoCompressPerf(int multi, int streamLenP, int cLevel, int loopTimes, int bsize, const char *in_filename) -{ - pid_t pidChild = 0; - struct timeval start, stop; - int core_id = 0; - - int streamLen = streamLenP; - if (in_filename != NULL) { - void *inbuf = NULL; - streamLen = read_inputFile(in_filename, &inbuf); - g_inbuf = inbuf; - } else { - g_inbuf = CompressInputGet(streamLenP); - } - if (g_inbuf == NULL) { - return; - } - - for (int i = 0; i < multi; i++) { - pidChild = fork(); - if (pidChild == 0) { - if (g_use_cpu_number > 1) { - // 子进程开始绑核的cpuid。 - // 针对一组numa80个核的机器，连续并均匀分配到N个不同加速器。连续分配测速会小一点 - // core_id = (int)(i / g_use_cpu_number) + core_seq + (80 * (i % g_use_cpu_number)) ; - - // 开始绑核的cpuid：间隔不连续绑核。均匀分片到前N个CPU上。 - core_id = i + core_seq + (80 * (i % g_use_cpu_number)); - } else { - core_id = i + core_seq; // 单个numa连续绑核 - } - if (g_use_cpu_number > 1) { - printf("bind core: %d.\n", core_id); - } - break; - } else if (pidChild < 0) { - printf("%s fork failed\n", __func__); - } - } - - if (pidChild > 0) { - gettimeofday(&start, NULL); - } - - if (pidChild == 0) { - pthread_t threads[100]; - struct ThreadArgs args = {bsize, streamLen, cLevel, loopTimes, core_id}; - - for (int i = 0; i < g_threadnum; i++) { - pthread_create(&threads[i], NULL, thread_function, &args); - } - - for (int i = 0; i < g_threadnum; i++) { - pthread_join(threads[i], NULL); - } - } - - if (pidChild > 0) { - int ret = -1; - while (1) { - ret = wait(NULL); - if (ret == -1) { - if (errno == EINTR) { - continue; - } - free(g_inbuf); - break; - } - } - } - - if (pidChild > 0 || multi == 0) { - if (multi == 0) { - multi = 1; - } - - gettimeofday(&stop, NULL); - uint64_t time1 = (stop.tv_sec - start.tv_sec) * 1000000 + stop.tv_usec - start.tv_usec; - float speed1 = 1000000.0 / time1 * loopTimes * multi * g_threadnum * streamLen / (1 << 30); - printf("kaelz4 %s perf result:\n", "compress"); - printf(" time used: %lu us, speed = %.3f GB/s\n", time1, speed1); - } -} - -static void Usage(void) -{ - printf("usage: \n"); - printf(" -m: multi process \n"); - printf(" -n: loop times\n"); - printf(" -l: stream length(KB)\n"); - printf(" -c: compress level\n"); - printf(" -b: block size(KB)\n"); - printf(" -s: core sequence\n"); - printf(" -C: bind cpu numbers\n"); - printf(" -f: use this file for input data\n"); - printf(" -t: thread number\n"); - printf(" example: ./kaelz4_perf -c 1 -l 64000 -m 10 -b 64\n"); -} - -int main(int argc, char **argv) -{ - int o = 0; - const char *optstring = "c:l:hb:m:n:s:t:f:C:"; - int multi = 10; - int loopTimes = 1; - int streamLen = 64000; - int cLevel = 1; // 压缩等级 - int bsize = 64; // 切块大小 - enum CompressFunc cFunction = LZ4_BLOCKCOMPRESS; // 压缩模式 - char input_filename[128] = {0}; - while ((o = getopt(argc, argv, optstring)) != -1) { - if (optstring == NULL) { - continue; - } - switch (o) { - case 'C': - g_use_cpu_number = atoi(optarg); - break; - case 't': - g_threadnum = atoi(optarg); - if (g_threadnum < 0) { - printf("Error: the number of thread must be larger than 0\n"); - exit(1); - } - break; - case 's': - core_seq = atoi(optarg); - if (core_seq < 0 || core_seq > 319) { - printf("Error: the sequence of core must be larger than 0 and smaller than 320\n"); - exit(1); - } - break; - case 'm': - multi = atoi(optarg); - if (multi < 0) { - printf("Error: the number of process must be larger than 0\n"); - exit(1); - } - break; - case 'n': - loopTimes = atoi(optarg); - if (loopTimes <= 0) { - printf("Error: loop times must be larger than 0\n"); - exit(1); - } - break; - case 'l': - streamLen = atoi(optarg); - if (streamLen <= 0) { - printf("Error: stream length must be larger than 0\n"); - exit(1); - } - break; - case 'c': - cLevel = atoi(optarg); - if (cLevel < 0) { - printf("Error: compress function is out of range\n"); - exit(1); - } - break; - case 'b': - bsize = atoi(optarg); - break; - case 'f': - strcpy(input_filename, optarg); - break; - case 'h': - Usage(); - return 0; - default: - printf("Error: Incorrect parameters\n"); - Usage(); - return 0; - } - } - - if (argc <= 1) { - Usage(); - printf("\ndefault input parameter used\n"); - } - printf("kaelz4 perf parameter: multi process %d, stream length: %d(KB), block size: %d(KB), compress level: %d, " - "compress function: %d, loop times: %d, g_threadnum: %d, core sequence start: %d, use %d cpu. \n", - multi, - streamLen, - bsize, - cLevel, - cFunction, - loopTimes, - g_threadnum, - core_seq, - g_use_cpu_number); - - streamLen = 1024 * streamLen; - bsize = 1024 * bsize; - const char *in_filename = input_filename[0] == 0 ? NULL : input_filename; - switch (cFunction) { - case LZ4_BLOCKCOMPRESS: - DoCompressPerf(multi, streamLen, cLevel, loopTimes, bsize, in_filename); - break; - default: - printf("Error: no such compress funciton\n"); - break; - } - - return 0; -} diff --git a/KAESnappy/test/perftest/makefile b/KAESnappy/test/perftest/makefile deleted file mode 100644 index 5056260..0000000 --- a/KAESnappy/test/perftest/makefile +++ /dev/null @@ -1,16 +0,0 @@ - -# Include Libs. -LIBDIR := -L/usr/local/kaelz4/lib -llz4 -lkaelz4 -# The flags -CFLAGS := -Wall -LDFLAGS := $(LIBDIR) - -all: kaelz4_perf -objects = kaelz4_perf.c - -kaelz4_perf : $(objects) - gcc $(CFLAGS) $(LDFLAGS) -o kaelz4_perf $(objects) - -.PHONY : clean -clean : - -rm -f kaelz4_perf -- Gitee From 614f694d6fef77f934cdb131f47d72bb41ce1227 Mon Sep 17 00:00:00 2001 From: wrh <812507909@qq.com> Date: Thu, 11 Sep 2025 16:40:25 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix=20:=20KAESNAPPY/src/utils/=E5=8F=8Akaes?= =?UTF-8?q?nappy.h=EF=BC=8C=E5=8E=9Fkaelz4=E5=87=BD=E6=95=B0=E5=90=8D?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- KAESnappy/include/kaesnappy.h | 54 ++++++------- KAESnappy/src/kaesnappy_adapter.c | 58 +++++++------- KAESnappy/src/kaesnappy_adapter.h | 28 +++---- KAESnappy/src/utils/kaesnappy_common.c | 22 +++--- KAESnappy/src/utils/kaesnappy_common.h | 16 ++-- KAESnappy/src/utils/kaesnappy_log.c | 88 ++++++++++----------- KAESnappy/src/utils/kaesnappy_log.h | 60 +++++++-------- KAESnappy/src/v1/kaesnappy_comp.c | 102 ++++++++++++------------- KAESnappy/src/v1/kaesnappy_comp.h | 12 +-- KAESnappy/src/v1/kaesnappy_init.c | 10 +-- KAESnappy/src/v1/kaesnappy_init.h | 6 +- KAESnappy/src/v2/kaesnappy_compress.c | 12 +-- KAESnappy/src/v2/kaesnappy_config.c | 22 +++--- KAESnappy/src/v2/kaesnappy_config.h | 6 +- 14 files changed, 248 insertions(+), 248 deletions(-) diff --git a/KAESnappy/include/kaesnappy.h b/KAESnappy/include/kaesnappy.h index fe51282..a8dc0ef 100644 --- a/KAESnappy/include/kaesnappy.h +++ b/KAESnappy/include/kaesnappy.h @@ -1,6 +1,6 @@ /* * Copyright (c) Huawei Technologies Co., Ltd. 2021-2024. All rights reserved. - * Description: contain kaelz4 api defines + * Description: contain kaesnappy api defines * Author: DSA * Create: 2024-7-6 */ @@ -29,10 +29,10 @@ typedef struct seqDef_s { /* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */ typedef enum { - LZ4_llt_none = 0, /* no longLengthType */ - LZ4_llt_literalLength = 1, /* represents a long literal */ - LZ4_llt_matchLength = 2 /* represents a long match */ -} LZ4_longLengthType_e; + SNAPPY_llt_none = 0, /* no longLengthType */ + SNAPPY_llt_literalLength = 1, /* represents a long literal */ + SNAPPY_llt_matchLength = 2 /* represents a long match */ +} SNAPPY_longLengthType_e; typedef struct { seqDef* sequencesStart; @@ -44,7 +44,7 @@ typedef struct { BYTE* ofCode; size_t maxNbSeq; size_t maxNbLit; - LZ4_longLengthType_e longLengthType; + SNAPPY_longLengthType_e longLengthType; U32 longLengthPos; } seqStore_t; @@ -56,20 +56,20 @@ typedef struct { int compressionLevel; int kaeLevel; int seqnum; -} LZ4_CCtx; +} SNAPPY_CCtx; #define TUPLE_STATUS_COMPRESS 2 #define TUPLE_STATUS_RLEBLOCK 1 #define TUPLE_STATUS_NOCOMPRESS 0 -#define KAE_LZ4_SUCC 0 -#define KAE_LZ4_INVAL_PARA 1 -#define KAE_LZ4_INIT_FAIL 2 -#define KAE_LZ4_COMP_FAIL 3 -#define KAE_LZ4_RELEASE_FAIL 4 -#define KAE_LZ4_ALLOC_FAIL 5 -#define KAE_LZ4_SET_FAIL 6 -#define KAE_LZ4_HW_TIMEOUT_FAIL 7 +#define KAE_SNAPPY_SUCC 0 +#define KAE_SNAPPY_INVAL_PARA 1 +#define KAE_SNAPPY_INIT_FAIL 2 +#define KAE_SNAPPY_COMP_FAIL 3 +#define KAE_SNAPPY_RELEASE_FAIL 4 +#define KAE_SNAPPY_ALLOC_FAIL 5 +#define KAE_SNAPPY_SET_FAIL 6 +#define KAE_SNAPPY_HW_TIMEOUT_FAIL 7 #define VERSION_STRUCT_LEN 100 typedef struct { @@ -77,16 +77,16 @@ typedef struct { char productVersion[VERSION_STRUCT_LEN]; char componentName[VERSION_STRUCT_LEN]; char componentVersion[VERSION_STRUCT_LEN]; -} KAELz4Version; +} KAESnappyVersion; -extern int kaelz4_get_version(KAELz4Version* ver); -extern int kaelz4_init(LZ4_CCtx* zc); -extern void kaelz4_reset(LZ4_CCtx* zc); -extern void kaelz4_release(LZ4_CCtx* zc); -extern void kaelz4_setstatus(LZ4_CCtx* zc, unsigned int status); -extern int kaelz4_compress(LZ4_CCtx* zc, const void* src, size_t srcSize); +extern int kaesnappy_get_version(KAESnappyVersion* ver); +extern int kaesnappy_init(SNAPPY_CCtx* zc); +extern void kaesnappy_reset(SNAPPY_CCtx* zc); +extern void kaesnappy_release(SNAPPY_CCtx* zc); +extern void kaesnappy_setstatus(SNAPPY_CCtx* zc, unsigned int status); +extern int kaesnappy_compress(SNAPPY_CCtx* zc, const void* src, size_t srcSize); -struct kaelz4_result { +struct kaesnappy_result { int status; unsigned int flag; void *user_data; @@ -96,11 +96,11 @@ struct kaelz4_result { uint32_t *obuf_crc; }; -typedef void (*lz4_async_callback)(struct kaelz4_result *result); +typedef void (*snappy_async_callback)(struct kaesnappy_result *result); typedef int (*sw_compress_fn)(const char* src, char* dst, int srcSize, int dstCapacity); typedef size_t (*sw_compress_frame_fn)(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const void* preferences_ptr); -int KAELZ4_compress_async(const void *src, void *dst, lz4_async_callback callback, struct kaelz4_result *result); -void KAELZ4_teardown_async_compress(void); -int KAELZ4_async_compress_init(sw_compress_fn sw_compress, sw_compress_frame_fn sw_compress_frame); +int KAESNAPPY_compress_async(const void *src, void *dst, snappy_async_callback callback, struct kaesnappy_result *result); +void KAESNAPPY_teardown_async_compress(void); +int KAESNAPPY_async_compress_init(sw_compress_fn sw_compress, sw_compress_frame_fn sw_compress_frame); #endif \ No newline at end of file diff --git a/KAESnappy/src/kaesnappy_adapter.c b/KAESnappy/src/kaesnappy_adapter.c index 056a994..8b03ba8 100644 --- a/KAESnappy/src/kaesnappy_adapter.c +++ b/KAESnappy/src/kaesnappy_adapter.c @@ -27,7 +27,7 @@ static void uadk_get_accel_platform(void) return; } // init log - kaelz4_debug_init_log(); + kaesnappy_debug_init_log(); // check sva struct uacce_dev* dev = wd_get_accel_dev("lz77_zstd"); if (dev) { @@ -50,7 +50,7 @@ end: US_INFO("kaelz4 v%d inited!\n", g_platform); } -int kaelz4_init(LZ4_CCtx* zc) +int kaesnappy_init(SNAPPY_CCtx* zc) { uadk_get_accel_platform(); @@ -68,11 +68,11 @@ int kaelz4_init(LZ4_CCtx* zc) default: break; } - US_INFO("kaelz4_init return code is %d\n", ret); + US_INFO("kaesnappy_init return code is %d\n", ret); return ret; } -void kaelz4_reset(LZ4_CCtx* zc) +void kaesnappy_reset(SNAPPY_CCtx* zc) { uadk_get_accel_platform(); @@ -88,10 +88,10 @@ void kaelz4_reset(LZ4_CCtx* zc) default: break; } - US_INFO("kaelz4_reset"); + US_INFO("kaesnappy_reset"); } -void kaelz4_release(LZ4_CCtx* zc) +void kaesnappy_release(SNAPPY_CCtx* zc) { uadk_get_accel_platform(); @@ -111,7 +111,7 @@ void kaelz4_release(LZ4_CCtx* zc) US_INFO("kaelz4_released"); } -void kaelz4_setstatus(LZ4_CCtx* zc, unsigned int status) +void kaesnappy_setstatus(SNAPPY_CCtx* zc, unsigned int status) { uadk_get_accel_platform(); @@ -131,7 +131,7 @@ void kaelz4_setstatus(LZ4_CCtx* zc, unsigned int status) US_INFO("kaelz4_set blk_type %d\n", status); } -int kaelz4_compress(LZ4_CCtx* zc, const void* src, size_t srcSize) +int kaesnappy_compress(SNAPPY_CCtx* zc, const void* src, size_t srcSize) { uadk_get_accel_platform(); @@ -149,7 +149,7 @@ int kaelz4_compress(LZ4_CCtx* zc, const void* src, size_t srcSize) default: break; } - US_INFO("kaelz4_compress return code is %d\n", ret); + US_INFO("kaesnappy_compress return code is %d\n", ret); return ret; } @@ -267,7 +267,7 @@ static void *compress_thread_func(void *arg) while (1) { // 等待任务 - while (task_queue->pi == task_queue->ci && ret == KAE_LZ4_PROCESS_IDLE) { + while (task_queue->pi == task_queue->ci && ret == KAE_SNAPPY_PROCESS_IDLE) { if (enter_idle == 0) { get_time_out_spec(&timeout, &polling_timeout); enter_idle = 1; @@ -326,7 +326,7 @@ static int kaelz4_task_queue_init(lz4_task_queue *task_queue, int index) { task_queue->tasks = malloc(KAELZ4_TASK_QUEUE_DEPTH * sizeof(lz4_async_task_t)); if (task_queue->tasks == NULL) { - return KAE_LZ4_ALLOC_FAIL; + return KAE_SNAPPY_ALLOC_FAIL; } task_queue->pi = 0; task_queue->ci = 0; @@ -338,9 +338,9 @@ static int kaelz4_task_queue_init(lz4_task_queue *task_queue, int index) US_ERR("Error: Failed to create compression worker thread"); free(task_queue->tasks); task_queue->tasks = NULL; - return KAE_LZ4_INIT_FAIL; + return KAE_SNAPPY_INIT_FAIL; } - return KAE_LZ4_SUCC; + return KAE_SNAPPY_SUCC; } static void kaelz4_task_queue_free(lz4_task_queue *task_queue) @@ -364,7 +364,7 @@ static int kaelz4_task_queues_init(int task_queue_num) } if (g_task_queues.num > MAX_TASK_NUM) { - return KAE_LZ4_INIT_FAIL; + return KAE_SNAPPY_INIT_FAIL; } for (i = 0; i < g_task_queues.num; i++) { @@ -372,16 +372,16 @@ static int kaelz4_task_queues_init(int task_queue_num) goto task_queue_free; } - return KAE_LZ4_SUCC; + return KAE_SNAPPY_SUCC; task_queue_free: while (i--) { kaelz4_task_queue_free(&g_task_queues.task_queue[i]); } - return KAE_LZ4_INIT_FAIL; + return KAE_SNAPPY_INIT_FAIL; } -int KAELZ4_async_compress_init(sw_compress_fn sw_compress, sw_compress_frame_fn sw_compress_frame) +int KAESNAPPY_async_compress_init(sw_compress_fn sw_compress, sw_compress_frame_fn sw_compress_frame) { int ret = 0; pthread_mutex_lock(&g_task_queue_init_mutex); @@ -400,7 +400,7 @@ int KAELZ4_async_compress_init(sw_compress_fn sw_compress, sw_compress_frame_fn return ret; } -void KAELZ4_teardown_async_compress(void) +void KAESNAPPY_teardown_async_compress(void) { pthread_mutex_lock(&g_task_queue_init_mutex); if (g_task_queues.init == 0) { @@ -424,7 +424,7 @@ static inline int kaelz4_enqueue(lz4_task_queue *task_queue, lz4_async_task_t *t while ((task_queue->pi + 1) % KAELZ4_TASK_QUEUE_DEPTH == task_queue->ci) { pthread_mutex_unlock(task_queue->mutex); if (cnt > ENQUEUE_TIME_OUT_US) { - return KAE_LZ4_ALLOC_FAIL; + return KAE_SNAPPY_ALLOC_FAIL; } cnt++; @@ -433,7 +433,7 @@ static inline int kaelz4_enqueue(lz4_task_queue *task_queue, lz4_async_task_t *t } if (task_queue->tasks == NULL) { - return KAE_LZ4_ALLOC_FAIL; + return KAE_SNAPPY_ALLOC_FAIL; } US_DEBUG("receive a task, enqueue"); @@ -464,13 +464,13 @@ static unsigned int kaelz4_get_queue_id(void) return index; } static int kaelz4_async_do_comp(const void *src, void *dst, - lz4_async_callback callback, struct kaelz4_result *result, - enum kae_lz4_async_data_format data_format, const int* preferences_ptr) + snappy_async_callback callback, struct kaesnappy_result *result, + enum kae_snappy_async_data_format data_format, const int* preferences_ptr) { // 初始化队列容量 if (unlikely(g_task_queues.init == 0)) { - if (KAELZ4_async_compress_init(NULL, NULL) != 0) { - return KAE_LZ4_INIT_FAIL; + if (KAESNAPPY_async_compress_init(NULL, NULL) != 0) { + return KAE_SNAPPY_INIT_FAIL; } } @@ -489,17 +489,17 @@ static int kaelz4_async_do_comp(const void *src, void *dst, return kaelz4_enqueue(&g_task_queues.task_queue[index], &task); } -int KAELZ4_compress_async(const void *src, void *dst, lz4_async_callback callback, - struct kaelz4_result *result) +int KAESNAPPY_compress_async(const void *src, void *dst, snappy_async_callback callback, + struct kaesnappy_result *result) { if (unlikely(src == NULL || dst == NULL || callback == NULL || result == NULL || result->src_size == 0)) { - return KAE_LZ4_INVAL_PARA; + return KAE_SNAPPY_INVAL_PARA; } if (result->src_size <= SMALL_BLOCK_SIZE) { - return kaelz4_async_do_comp(src, dst, callback, result, KAELZ4_ASYNC_SMALL_BLOCK, NULL); + return kaelz4_async_do_comp(src, dst, callback, result, KAESNAPPY_ASYNC_SMALL_BLOCK, NULL); } - return kaelz4_async_do_comp(src, dst, callback, result, KAELZ4_ASYNC_BLOCK, NULL); + return kaelz4_async_do_comp(src, dst, callback, result, KAESNAPPY_ASYNC_BLOCK, NULL); } diff --git a/KAESnappy/src/kaesnappy_adapter.h b/KAESnappy/src/kaesnappy_adapter.h index 61617fb..1f8032d 100644 --- a/KAESnappy/src/kaesnappy_adapter.h +++ b/KAESnappy/src/kaesnappy_adapter.h @@ -28,9 +28,9 @@ enum { typedef struct { const void *src; void *dst; - lz4_async_callback callback; - struct kaelz4_result *result; - enum kae_lz4_async_data_format data_format; + snappy_async_callback callback; + struct kaesnappy_result *result; + enum kae_snappy_async_data_format data_format; int preferences; } lz4_async_task_t; @@ -54,20 +54,20 @@ typedef struct { volatile int init; } lz4_task_queues; -int kaelz4_init_v1(LZ4_CCtx* zc); -void kaelz4_reset_v1(LZ4_CCtx* zc); -void kaelz4_release_v1(LZ4_CCtx* zc); -void kaelz4_setstatus_v1(LZ4_CCtx* zc, unsigned int status); -int kaelz4_compress_v1(LZ4_CCtx* zc, const void* src, size_t srcSize); +int kaelz4_init_v1(SNAPPY_CCtx* zc); +void kaelz4_reset_v1(SNAPPY_CCtx* zc); +void kaelz4_release_v1(SNAPPY_CCtx* zc); +void kaelz4_setstatus_v1(SNAPPY_CCtx* zc, unsigned int status); +int kaelz4_compress_v1(SNAPPY_CCtx* zc, const void* src, size_t srcSize); void kaelz4_compress_async(const void *src, void *dst, - lz4_async_callback callback, struct kaelz4_result *result, - enum kae_lz4_async_data_format data_format, const int *ptr); + snappy_async_callback callback, struct kaesnappy_result *result, + enum kae_snappy_async_data_format data_format, const int *ptr); int kaelz4_async_compress_polling(int budget); -int kaelz4_init_v2(LZ4_CCtx* zc); -void kaelz4_release_v2(LZ4_CCtx* zc); -void kaelz4_setstatus_v2(LZ4_CCtx* zc, unsigned int status); -int kaelz4_compress_v2(LZ4_CCtx* zc, const void* src, size_t srcSize); +int kaelz4_init_v2(SNAPPY_CCtx* zc); +void kaelz4_release_v2(SNAPPY_CCtx* zc); +void kaelz4_setstatus_v2(SNAPPY_CCtx* zc, unsigned int status); +int kaelz4_compress_v2(SNAPPY_CCtx* zc, const void* src, size_t srcSize); int wd_get_available_dev_num(const char* alogrithm); int kaelz4_async_is_thread_do_comp_full(); diff --git a/KAESnappy/src/utils/kaesnappy_common.c b/KAESnappy/src/utils/kaesnappy_common.c index bae5172..ba66576 100644 --- a/KAESnappy/src/utils/kaesnappy_common.c +++ b/KAESnappy/src/utils/kaesnappy_common.c @@ -18,7 +18,7 @@ typedef enum ARCH_TYPE { CPU_HISILICOM_V4, /* for the future */ CPU_UNKNOW, } ARCH_TYPE; -static int g_kaelz4Initialized = 0; +static int g_kaesnappyInitialized = 0; static inline void versionCpy(char str1[], const char str2[]) { @@ -30,19 +30,19 @@ static inline void versionCpy(char str1[], const char str2[]) str1[i] = '\0'; } -int kaelz4_get_version(KAELz4Version* ver) +int kaesnappy_get_version(KAESnappyVersion* ver) { if (ver == NULL) { - return KAE_LZ4_INVAL_PARA; + return KAE_SNAPPY_INVAL_PARA; } versionCpy(ver->productName, "Kunpeng Boostkit"); versionCpy(ver->productVersion, "23.0.RC2"); - versionCpy(ver->componentName, "KAELz4"); + versionCpy(ver->componentName, "KAESnappy"); versionCpy(ver->componentVersion, "2.0.4"); - return KAE_LZ4_SUCC; + return KAE_SNAPPY_SUCC; } -static ARCH_TYPE KaeLz4Detect(void) +static ARCH_TYPE KaeSnappyDetect(void) { unsigned long long cpuId; __asm__ volatile("mrs %0, MIDR_EL1":"=r"(cpuId)); @@ -61,15 +61,15 @@ static ARCH_TYPE KaeLz4Detect(void) return CPU_UNKNOW; } -HIDDEN_API void CONSTRUCTOR KaeLz4Init(void) +HIDDEN_API void CONSTRUCTOR KaeSnappyInit(void) { - if (g_kaelz4Initialized != 0) { + if (g_kaesnappyInitialized != 0) { return; } - if (KaeLz4Detect() != CPU_HISILICOM_V2) { - fprintf(stderr, "KAElz4 only support in V2, please check CPU ID.\n"); + if (KaeSnappyDetect() != CPU_HISILICOM_V2) { + fprintf(stderr, "KAEsnappy only support in V2, please check CPU ID.\n"); abort(); } - g_kaelz4Initialized = 1; + g_kaesnappyInitialized = 1; } diff --git a/KAESnappy/src/utils/kaesnappy_common.h b/KAESnappy/src/utils/kaesnappy_common.h index 1b00e25..213f520 100644 --- a/KAESnappy/src/utils/kaesnappy_common.h +++ b/KAESnappy/src/utils/kaesnappy_common.h @@ -8,16 +8,16 @@ #ifndef KAESNAPPY_COMMON_H #define KAESNAPPY_COMMON_H -#define CONF_KAELZ4 +#define CONF_KAESNAPPY -#define KAE_LZ4_PROCESS_IDLE 0 -#define KAE_LZ4_PROCESS_HW_BUSY -1 +#define KAE_SNAPPY_PROCESS_IDLE 0 +#define KAE_SNAPPY_PROCESS_HW_BUSY -1 -enum kae_lz4_async_data_format { - KAELZ4_ASYNC_SMALL_BLOCK = 0, - KAELZ4_ASYNC_BLOCK, - KAELZ4_ASYNC_FRAME, - KAELZ4_ASYNC_BUTT, +enum kae_snappy_async_data_format { + KAESNAPPY_ASYNC_SMALL_BLOCK = 0, + KAESNAPPY_ASYNC_BLOCK, + KAESNAPPY_ASYNC_FRAME, + KAESNAPPY_ASYNC_BUTT, }; #include "kaesnappy.h" diff --git a/KAESnappy/src/utils/kaesnappy_log.c b/KAESnappy/src/utils/kaesnappy_log.c index 50ec6a9..a562144 100644 --- a/KAESnappy/src/utils/kaesnappy_log.c +++ b/KAESnappy/src/utils/kaesnappy_log.c @@ -13,20 +13,20 @@ #include #include "kaesnappy_log.h" -#define KAE_CONFIG_FILE_NAME "/kaelz4.cnf" +#define KAE_CONFIG_FILE_NAME "/kaesnappy.cnf" #define MAX_LEVEL_LEN 10 #define MAX_CONFIG_LEN 512 -static const char *g_kaelz4_conf_env = "KAELZ4_CONF_ENV"; +static const char *g_kaesnappy_conf_env = "KAESNAPPY_CONF_ENV"; -FILE *g_kaelz4_debug_log_file = (FILE *)NULL; -pthread_mutex_t g_kaelz4_debug_file_mutex = PTHREAD_MUTEX_INITIALIZER; -int g_kaelz4_debug_file_ref_count = 0; -int g_kaelz4_log_init_times = 0; -int g_kaelz4_log_level = 0; -__thread int g_kaelz4_threadid = 0; +FILE *g_kaesnappy_debug_log_file = (FILE *)NULL; +pthread_mutex_t g_kaesnappy_debug_file_mutex = PTHREAD_MUTEX_INITIALIZER; +int g_kaesnappy_debug_file_ref_count = 0; +int g_kaesnappy_log_init_times = 0; +int g_kaesnappy_log_level = 0; +__thread int g_kaesnappy_threadid = 0; -const char *g_kaelz4_log_level_string[] = { +const char *g_kaesnappy_log_level_string[] = { "none", "error", "warning", @@ -34,7 +34,7 @@ const char *g_kaelz4_log_level_string[] = { "debug", }; -int kaelz4_drv_findsection(FILE *stream, const char *v_pszSection) +int kaesnappy_drv_findsection(FILE *stream, const char *v_pszSection) { char line[256]; // array length:256 char *pos = NULL; @@ -61,7 +61,7 @@ int kaelz4_drv_findsection(FILE *stream, const char *v_pszSection) return -1; } -void kaelz4_drv_get_value(char *pos, char *v_pszValue) +void kaesnappy_drv_get_value(char *pos, char *v_pszValue) { while (*pos != '\0') { if (*pos == ' ') { @@ -78,7 +78,7 @@ void kaelz4_drv_get_value(char *pos, char *v_pszValue) } } -int kaelz4_drv_find_item(FILE *stream, const char *v_pszItem, char *v_pszValue) +int kaesnappy_drv_find_item(FILE *stream, const char *v_pszItem, char *v_pszValue) { char line[256]; // array length:256 char *pos = NULL; @@ -92,7 +92,7 @@ int kaelz4_drv_find_item(FILE *stream, const char *v_pszItem, char *v_pszValue) pos = strstr(line, "="); if (pos != NULL) { pos++; - kaelz4_drv_get_value(pos, v_pszValue); + kaesnappy_drv_get_value(pos, v_pszValue); return 0; } } @@ -105,7 +105,7 @@ int kaelz4_drv_find_item(FILE *stream, const char *v_pszItem, char *v_pszValue) return -1; } -int kaelz4_drv_get_item(const char *config_file, const char *v_pszSection, +int kaesnappy_drv_get_item(const char *config_file, const char *v_pszSection, const char *v_pszItem, char *v_pszValue) { FILE *stream; @@ -116,8 +116,8 @@ int kaelz4_drv_get_item(const char *config_file, const char *v_pszSection, return -1; } - if (kaelz4_drv_findsection(stream, v_pszSection) == 0) { - retvalue = kaelz4_drv_find_item(stream, v_pszItem, v_pszValue); + if (kaesnappy_drv_findsection(stream, v_pszSection) == 0) { + retvalue = kaesnappy_drv_find_item(stream, v_pszItem, v_pszValue); } fclose(stream); @@ -132,7 +132,7 @@ static char *kae_getenv(const char *name) static void kae_set_conf_debuglevel() { - char *conf_path = kae_getenv(g_kaelz4_conf_env); + char *conf_path = kae_getenv(g_kaesnappy_conf_env); unsigned int i = 0; const char *filename = KAE_CONFIG_FILE_NAME; char *file_path = (char *)NULL; @@ -149,14 +149,14 @@ static void kae_set_conf_debuglevel() memset(file_path, 0, sizeof(conf_path) + sizeof(filename) + 1); strcat(file_path, conf_path); strcat(file_path, filename); - int ret = kaelz4_drv_get_item(file_path, "LogSection", "debug_level", debuglev); + int ret = kaesnappy_drv_get_item(file_path, "LogSection", "debug_level", debuglev); if (ret != 0) { goto err; } - for (i = 0; i < sizeof(g_kaelz4_log_level_string) / sizeof(g_kaelz4_log_level_string[0]); i++) { - if (strncmp(g_kaelz4_log_level_string[i], debuglev, strlen(debuglev) - 1) == 0) { - g_kaelz4_log_level = i; + for (i = 0; i < sizeof(g_kaesnappy_log_level_string) / sizeof(g_kaesnappy_log_level_string[0]); i++) { + if (strncmp(g_kaesnappy_log_level_string[i], debuglev, strlen(debuglev) - 1) == 0) { + g_kaesnappy_log_level = i; free(file_path); free(debuglev); return; @@ -164,7 +164,7 @@ static void kae_set_conf_debuglevel() } err: - g_kaelz4_log_level = KAE_NONE; + g_kaesnappy_log_level = KAE_NONE; if (debuglev != NULL) { free(debuglev); debuglev = (char *)NULL; @@ -177,39 +177,39 @@ err: return; } -void kaelz4_debug_init_log() +void kaesnappy_debug_init_log() { - pthread_mutex_lock(&g_kaelz4_debug_file_mutex); - g_kaelz4_threadid = gettid(); + pthread_mutex_lock(&g_kaesnappy_debug_file_mutex); + g_kaesnappy_threadid = gettid(); kae_set_conf_debuglevel(); - if (!g_kaelz4_debug_file_ref_count && g_kaelz4_log_level != KAE_NONE) { - g_kaelz4_debug_log_file = fopen(KAELZ4_DEBUG_FILE_PATH, "a+"); - if (g_kaelz4_debug_log_file == NULL) { - g_kaelz4_debug_log_file = stderr; - fprintf(stderr, "unable to open %s, %s\n", KAELZ4_DEBUG_FILE_PATH, strerror(errno)); + if (!g_kaesnappy_debug_file_ref_count && g_kaesnappy_log_level != KAE_NONE) { + g_kaesnappy_debug_log_file = fopen(KAESNAPPY_DEBUG_FILE_PATH, "a+"); + if (g_kaesnappy_debug_log_file == NULL) { + g_kaesnappy_debug_log_file = stderr; + fprintf(stderr, "unable to open %s, %s\n", KAESNAPPY_DEBUG_FILE_PATH, strerror(errno)); } else { - g_kaelz4_debug_file_ref_count++; + g_kaesnappy_debug_file_ref_count++; } } - g_kaelz4_log_init_times++; - pthread_mutex_unlock(&g_kaelz4_debug_file_mutex); + g_kaesnappy_log_init_times++; + pthread_mutex_unlock(&g_kaesnappy_debug_file_mutex); } -void kaelz4_debug_close_log() +void kaesnappy_debug_close_log() { - pthread_mutex_lock(&g_kaelz4_debug_file_mutex); - g_kaelz4_log_init_times--; - if (g_kaelz4_debug_file_ref_count && (g_kaelz4_log_init_times == 0)) { - if (g_kaelz4_debug_log_file != NULL) { - fclose(g_kaelz4_debug_log_file); - g_kaelz4_debug_file_ref_count--; - g_kaelz4_debug_log_file = stderr; + pthread_mutex_lock(&g_kaesnappy_debug_file_mutex); + g_kaesnappy_log_init_times--; + if (g_kaesnappy_debug_file_ref_count && (g_kaesnappy_log_init_times == 0)) { + if (g_kaesnappy_debug_log_file != NULL) { + fclose(g_kaesnappy_debug_log_file); + g_kaesnappy_debug_file_ref_count--; + g_kaesnappy_debug_log_file = stderr; } } - pthread_mutex_unlock(&g_kaelz4_debug_file_mutex); + pthread_mutex_unlock(&g_kaesnappy_debug_file_mutex); } -void kaelz4_save_log(FILE *src) +void kaesnappy_save_log(FILE *src) { int size = 0; char buf[1024] = {0}; // buf length:1024 @@ -218,7 +218,7 @@ void kaelz4_save_log(FILE *src) return; } - FILE *dst = fopen(KAELZ4_DEBUG_FILE_PATH_OLD, "w"); + FILE *dst = fopen(KAESNAPPY_DEBUG_FILE_PATH_OLD, "w"); if (dst == NULL) { return; } diff --git a/KAESnappy/src/utils/kaesnappy_log.h b/KAESnappy/src/utils/kaesnappy_log.h index c665aec..3645f74 100644 --- a/KAESnappy/src/utils/kaesnappy_log.h +++ b/KAESnappy/src/utils/kaesnappy_log.h @@ -13,15 +13,15 @@ #include #include -#define KAELZ4_DEBUG_FILE_PATH "/var/log/kaelz4.log" -#define KAELZ4_DEBUG_FILE_PATH_OLD "/var/log/kaelz4.log.old" +#define KAESNAPPY_DEBUG_FILE_PATH "/var/log/kaesnappy.log" +#define KAESNAPPY_DEBUG_FILE_PATH_OLD "/var/log/kaesnappy.log.old" #define KAE_LOG_MAX_SIZE 209715200 -extern FILE *g_kaelz4_debug_log_file; -extern pthread_mutex_t g_kaelz4_debug_file_mutex; -extern const char *g_kaelz4_log_level_string[]; -extern int g_kaelz4_log_level; -extern __thread int g_kaelz4_threadid; +extern FILE *g_kaesnappy_debug_log_file; +extern pthread_mutex_t g_kaesnappy_debug_file_mutex; +extern const char *g_kaesnappy_log_level_string[]; +extern int g_kaesnappy_log_level; +extern __thread int g_kaesnappy_threadid; enum KAE_LOG_LEVEL { KAE_NONE = 0, @@ -33,42 +33,42 @@ enum KAE_LOG_LEVEL { void ENGINE_LOG_LIMIT(int level, int times, int limit, const char *fmt, ...); -#define KAELZ4_CRYPTO(LEVEL, fmt, args...) \ +#define KAESNAPPY_CRYPTO(LEVEL, fmt, args...) \ do { \ - if (LEVEL > g_kaelz4_log_level) { \ + if (LEVEL > g_kaesnappy_log_level) { \ break; \ } \ struct tm *log_tm_p = NULL; \ time_t timep = time((time_t *)NULL); \ log_tm_p = localtime(&timep); \ - flock(g_kaelz4_debug_log_file->_fileno, LOCK_EX); \ - pthread_mutex_lock(&g_kaelz4_debug_file_mutex); \ - fseek(g_kaelz4_debug_log_file, 0, SEEK_END); \ + flock(g_kaesnappy_debug_log_file->_fileno, LOCK_EX); \ + pthread_mutex_lock(&g_kaesnappy_debug_file_mutex); \ + fseek(g_kaesnappy_debug_log_file, 0, SEEK_END); \ if (log_tm_p != NULL) { \ - fprintf(g_kaelz4_debug_log_file, "[%4d-%02d-%02d %02d:%02d:%02d][%d][%s][%s:%d:%s()] " fmt "\n", \ + fprintf(g_kaesnappy_debug_log_file, "[%4d-%02d-%02d %02d:%02d:%02d][%d][%s][%s:%d:%s()] " fmt "\n", \ (1900 + log_tm_p->tm_year), (1 + log_tm_p->tm_mon), log_tm_p->tm_mday, \ - log_tm_p->tm_hour, log_tm_p->tm_min, log_tm_p->tm_sec, g_kaelz4_threadid, \ - g_kaelz4_log_level_string[LEVEL], __FILE__, __LINE__, __func__, ##args); \ + log_tm_p->tm_hour, log_tm_p->tm_min, log_tm_p->tm_sec, g_kaesnappy_threadid, \ + g_kaesnappy_log_level_string[LEVEL], __FILE__, __LINE__, __func__, ##args); \ } else { \ - fprintf(g_kaelz4_debug_log_file, "[%d][%s][%s:%d:%s()] " fmt "\n", g_kaelz4_threadid, \ - g_kaelz4_log_level_string[LEVEL], __FILE__, __LINE__, __func__, ##args); \ + fprintf(g_kaesnappy_debug_log_file, "[%d][%s][%s:%d:%s()] " fmt "\n", g_kaesnappy_threadid, \ + g_kaesnappy_log_level_string[LEVEL], __FILE__, __LINE__, __func__, ##args); \ } \ - if (ftell(g_kaelz4_debug_log_file) > KAE_LOG_MAX_SIZE) { \ - kaelz4_save_log(g_kaelz4_debug_log_file); \ - if(ftruncate(g_kaelz4_debug_log_file->_fileno, 0)); \ - fseek(g_kaelz4_debug_log_file, 0, SEEK_SET); \ + if (ftell(g_kaesnappy_debug_log_file) > KAE_LOG_MAX_SIZE) { \ + kaesnappy_save_log(g_kaesnappy_debug_log_file); \ + if(ftruncate(g_kaesnappy_debug_log_file->_fileno, 0)); \ + fseek(g_kaesnappy_debug_log_file, 0, SEEK_SET); \ } \ - pthread_mutex_unlock(&g_kaelz4_debug_file_mutex); \ - flock(g_kaelz4_debug_log_file->_fileno, LOCK_UN); \ + pthread_mutex_unlock(&g_kaesnappy_debug_file_mutex); \ + flock(g_kaesnappy_debug_log_file->_fileno, LOCK_UN); \ } while (0) -#define US_ERR(fmt, args...) KAELZ4_CRYPTO(KAE_ERROR, fmt, ##args) -#define US_WARN(fmt, args...) KAELZ4_CRYPTO(KAE_WARNING, fmt, ##args) -#define US_INFO(fmt, args...) KAELZ4_CRYPTO(KAE_INFO, fmt, ##args) -#define US_DEBUG(fmt, args...) KAELZ4_CRYPTO(KAE_DEBUG, fmt, ##args) +#define US_ERR(fmt, args...) KAESNAPPY_CRYPTO(KAE_ERROR, fmt, ##args) +#define US_WARN(fmt, args...) KAESNAPPY_CRYPTO(KAE_WARNING, fmt, ##args) +#define US_INFO(fmt, args...) KAESNAPPY_CRYPTO(KAE_INFO, fmt, ##args) +#define US_DEBUG(fmt, args...) KAESNAPPY_CRYPTO(KAE_DEBUG, fmt, ##args) -void kaelz4_debug_init_log(); -void kaelz4_debug_close_log(); -void kaelz4_save_log(FILE *src); +void kaesnappy_debug_init_log(); +void kaesnapy_debug_close_log(); +void kaesnappy_save_log(FILE *src); #endif diff --git a/KAESnappy/src/v1/kaesnappy_comp.c b/KAESnappy/src/v1/kaesnappy_comp.c index 98175b3..94fc16e 100644 --- a/KAESnappy/src/v1/kaesnappy_comp.c +++ b/KAESnappy/src/v1/kaesnappy_comp.c @@ -13,7 +13,7 @@ __thread struct kaelz4_async_ctrl g_async_ctrl = {0}; -void kaelz4_setstatus_v1(LZ4_CCtx* zc, unsigned int status) +void kaelz4_setstatus_v1(SNAPPY_CCtx* zc, unsigned int status) { kaelz4_ctx_t* kaelz4_ctx = (kaelz4_ctx_t*)zc->kaeConfig; if (kaelz4_ctx) { @@ -22,11 +22,11 @@ void kaelz4_setstatus_v1(LZ4_CCtx* zc, unsigned int status) } } -static int kaelz4_data_parsing(LZ4_CCtx* zc, kaelz4_ctx_t* config) +static int kaelz4_data_parsing(SNAPPY_CCtx* zc, kaelz4_ctx_t* config) { if (!config->lz4_data.literals_start || !config->lz4_data.sequences_start) { US_ERR("lz4 literals or sequences start is NULL!\n"); - return KAE_LZ4_INVAL_PARA; + return KAE_SNAPPY_INVAL_PARA; } zc->seqStore.litStart = config->lz4_data.literals_start; @@ -36,15 +36,15 @@ static int kaelz4_data_parsing(LZ4_CCtx* zc, kaelz4_ctx_t* config) zc->seqStore.sequencesStart = config->lz4_data.sequences_start; zc->seqStore.sequences = zc->seqStore.sequencesStart; zc->seqStore.sequences += config->lz4_data.seq_num; - return KAE_LZ4_SUCC; + return KAE_SNAPPY_SUCC; } -int kaelz4_compress_v1(LZ4_CCtx* zc, const void* src, size_t srcSize) +int kaelz4_compress_v1(SNAPPY_CCtx* zc, const void* src, size_t srcSize) { kaelz4_ctx_t* kaelz4_ctx = (kaelz4_ctx_t*)zc->kaeConfig; if (kaelz4_ctx == NULL || src == NULL || srcSize == 0) { US_ERR("compress parameter invalid\n"); - return KAE_LZ4_INVAL_PARA; + return KAE_SNAPPY_INVAL_PARA; } US_INFO("kaelz4 compress srcSize : %lu", srcSize); @@ -264,18 +264,18 @@ static uint32_t KAELZ4CRC32(uint32_t crc, const char *data, uint64_t len) static void kaelz4_compress_async_callback(struct kaelz4_compress_ctx *compress_ctx, int status) { - struct kaelz4_result *result = compress_ctx->result; + struct kaesnappy_result *result = compress_ctx->result; result->status = status; result->dst_len = compress_ctx->dst_len; - if (result->ibuf_crc != NULL && status == KAE_LZ4_SUCC) { + if (result->ibuf_crc != NULL && status == KAE_SNAPPY_SUCC) { *result->ibuf_crc = KAELZ4CRC32(*result->ibuf_crc, compress_ctx->src, compress_ctx->srcSize); } - if (result->obuf_crc != NULL && status == KAE_LZ4_SUCC) { + if (result->obuf_crc != NULL && status == KAE_SNAPPY_SUCC) { *result->obuf_crc = KAELZ4CRC32(*result->obuf_crc, compress_ctx->dst, compress_ctx->dst_len); } - if (unlikely(status != KAE_LZ4_SUCC)) { + if (unlikely(status != KAE_SNAPPY_SUCC)) { US_ERR("kae async compress fail! ret = %d\n", status); } @@ -623,13 +623,13 @@ static int kaelz4_triples_rebuild_64Kblock(struct kaelz4_async_req *req, const v static void kaelz4_async_compress_cb(int status, void *param) { struct kaelz4_async_req* req = param; - LZ4_CCtx* zc = &req->zc; + SNAPPY_CCtx* zc = &req->zc; kaelz4_ctx_t* kaelz4_ctx = (kaelz4_ctx_t*)zc->kaeConfig; struct wcrypto_comp_op_data *op_data = &kaelz4_ctx->op_data; if (status != 0) { US_ERR("kaelz4_async_compress_cb status %d !\n", status); - req->compress_ctx->status = KAE_LZ4_COMP_FAIL; + req->compress_ctx->status = KAE_SNAPPY_COMP_FAIL; req->done = 1; return; } @@ -646,20 +646,20 @@ static void kaelz4_async_compress_cb(int status, void *param) kaelz4_get_output_data(kaelz4_ctx); int ret = kaelz4_data_parsing(zc, kaelz4_ctx); - if (ret != KAE_LZ4_SUCC) { - req->compress_ctx->status = KAE_LZ4_COMP_FAIL; + if (ret != KAE_SNAPPY_SUCC) { + req->compress_ctx->status = KAE_SNAPPY_COMP_FAIL; req->done = 1; return; } req->done = 1; } -static int kaelz4_compress_async_impl(LZ4_CCtx* zc, const void* src, size_t srcSize, void *usr_data) +static int kaelz4_compress_async_impl(SNAPPY_CCtx* zc, const void* src, size_t srcSize, void *usr_data) { kaelz4_ctx_t* kaelz4_ctx = (kaelz4_ctx_t*)zc->kaeConfig; if (kaelz4_ctx == NULL || src == NULL || srcSize == 0) { US_ERR("compress parameter invalid\n"); - return KAE_LZ4_INVAL_PARA; + return KAE_SNAPPY_INVAL_PARA; } US_INFO("kaelz4 compress srcSize : %lu", srcSize); @@ -705,7 +705,7 @@ static void kaelz4_do_compress_polling(struct kaelz4_async_req *req) if (unlikely(ret < 0)) { US_ERR("poll fail! ret = %d\n", ret); kaelz4_find_and_free_kz_ctx(kaelz4_ctx); - req->compress_ctx->status = KAE_LZ4_COMP_FAIL; + req->compress_ctx->status = KAE_SNAPPY_COMP_FAIL; req->done = 1; } return; @@ -725,11 +725,11 @@ void kaelz4_async_init(volatile int *stop, sw_compress_fn sw_compress, sw_compre static int kaelz4_async_sw_compress(struct kaelz4_compress_ctx *compress_ctx) { int ret = -1; - compress_ctx->status = KAE_LZ4_SUCC; - if (compress_ctx->data_format == KAELZ4_ASYNC_FRAME && g_async_ctrl.sw_compress_frame != NULL) { + compress_ctx->status = KAE_SNAPPY_SUCC; + if (compress_ctx->data_format == KAESNAPPY_ASYNC_FRAME && g_async_ctrl.sw_compress_frame != NULL) { ret = g_async_ctrl.sw_compress_frame(compress_ctx->dst, compress_ctx->dstCapacity, compress_ctx->src, compress_ctx->srcSize, &compress_ctx->preferences); - } else if (compress_ctx->data_format <= KAELZ4_ASYNC_BLOCK && g_async_ctrl.sw_compress != NULL) { + } else if (compress_ctx->data_format <= KAESNAPPY_ASYNC_BLOCK && g_async_ctrl.sw_compress != NULL) { ret = g_async_ctrl.sw_compress(compress_ctx->src, compress_ctx->dst, compress_ctx->srcSize, compress_ctx->dstCapacity); } @@ -750,31 +750,31 @@ int kaelz4_async_compress_polling(int budget) while (req && cnt < budget) { kaelz4_do_compress_polling(req); if (!req->done) { - return KAE_LZ4_PROCESS_HW_BUSY; + return KAE_SNAPPY_PROCESS_HW_BUSY; } int ret = -1; - if (likely(compress_ctx->status == KAE_LZ4_SUCC)) { + if (likely(compress_ctx->status == KAE_SNAPPY_SUCC)) { ret = compress_ctx->kaelz4_post_process_handle(req, req->src, compress_ctx->dst + compress_ctx->dst_len); if (ret < 0) { US_ERR("kaelz4_post_process_handle err. ret=%d\n", ret); } } - if (unlikely(ret < 0 && req->idx == 0 && req->last != 0 && req->compress_ctx->status != KAE_LZ4_HW_TIMEOUT_FAIL)) { + if (unlikely(ret < 0 && req->idx == 0 && req->last != 0 && req->compress_ctx->status != KAE_SNAPPY_HW_TIMEOUT_FAIL)) { US_WARN("KAELz4 async compress switch to soft"); // 异常切软算处理 ret = kaelz4_async_sw_compress(compress_ctx); } - if (ret >= 0 && compress_ctx->status == KAE_LZ4_SUCC) { + if (ret >= 0 && compress_ctx->status == KAE_SNAPPY_SUCC) { compress_ctx->dst_len += ret; - compress_ctx->status = KAE_LZ4_SUCC; + compress_ctx->status = KAE_SNAPPY_SUCC; } else { compress_ctx->dst_len = 0; - if (compress_ctx->status == KAE_LZ4_SUCC) { - compress_ctx->status = KAE_LZ4_COMP_FAIL; + if (compress_ctx->status == KAE_SNAPPY_SUCC) { + compress_ctx->status = KAE_SNAPPY_COMP_FAIL; } US_ERR("kae post process fail! req index %d src size 0x%lx dst size 0x%lx last %d ret = %d status %d\n", @@ -807,7 +807,7 @@ int kaelz4_async_compress_polling(int budget) static struct timespec polling_timeout_10us = { 0, 10000 }; // 10us超时 -static void kaelz4_ctx_body_init(LZ4_CCtx *ctx_body) +static void kaelz4_ctx_body_init(SNAPPY_CCtx *ctx_body) { ctx_body->kaeInited = 0; ctx_body->kaeFrameMode = 1; // 相当于每个都强刷 @@ -826,14 +826,14 @@ static void kaelz4_ctx_body_init(LZ4_CCtx *ctx_body) ctx_body->seqnum = 0; } -static int kaelz4_async_init_ctx(LZ4_CCtx *ctx_body) +static int kaelz4_async_init_ctx(SNAPPY_CCtx *ctx_body) { int enter_polling = 0; kaelz4_ctx_body_init(ctx_body); if (unlikely(g_async_ctrl.kz_ctx[g_async_ctrl.ctx_index] == NULL)) { - while (kaelz4_init(ctx_body) != KAE_LZ4_SUCC) { // 本质来说，这个初始化函数就初始化了其中的kaeConfig，其他是没有的，所以在外面要赋值 + while (kaesnappy_init(ctx_body) != KAE_SNAPPY_SUCC) { // 本质来说，这个初始化函数就初始化了其中的kaeConfig，其他是没有的，所以在外面要赋值 struct timespec timeout; if (enter_polling == 0) { get_time_out_spec(&timeout, &polling_timeout_10us); @@ -842,7 +842,7 @@ static int kaelz4_async_init_ctx(LZ4_CCtx *ctx_body) // 如果发生超时则提前退出，到polling阶段再处理切软算 if (unlikely(*g_async_ctrl.stop_flag != 0 || check_time_out(&timeout))) { - return KAE_LZ4_INIT_FAIL; + return KAE_SNAPPY_INIT_FAIL; } (void)kaelz4_async_compress_polling(1); @@ -858,12 +858,12 @@ static int kaelz4_async_init_ctx(LZ4_CCtx *ctx_body) (void)kaelz4_async_compress_polling(1); // 此分支不需要超时判断，kaelz4_async_compress_polling本身具有超时机制，如果硬件超时，会主动释放资源 if (unlikely(*g_async_ctrl.stop_flag != 0)) { - return KAE_LZ4_INIT_FAIL; + return KAE_SNAPPY_INIT_FAIL; } if (g_async_ctrl.kz_ctx[g_async_ctrl.ctx_index] == NULL) { // polling 过程可能发生超时，kz资源可能已经释放 - return KAE_LZ4_INIT_FAIL; + return KAE_SNAPPY_INIT_FAIL; } } kaelz4_init_ctx(g_async_ctrl.kz_ctx[g_async_ctrl.ctx_index]); @@ -873,7 +873,7 @@ static int kaelz4_async_init_ctx(LZ4_CCtx *ctx_body) g_async_ctrl.ctx_index = (g_async_ctrl.ctx_index + 1) % MAX_NUM_IN_COMP; g_async_ctrl.cur_num_in_comp++; ctx_body->kaeInited = 1; - return KAE_LZ4_SUCC; + return KAE_SNAPPY_SUCC; } void kaelz4_ctx_clear(void) @@ -891,13 +891,13 @@ static int kaelz4_send_async_compress(struct kaelz4_async_req *req) int ret; // 1.kae上下文初始化函数调用 ret = kaelz4_async_init_ctx(&req->zc); - if (unlikely(ret != KAE_LZ4_SUCC)) { + if (unlikely(ret != KAE_SNAPPY_SUCC)) { US_ERR("Get kae hw ctx failed!\n"); return ret; } size_t compress_size = req->src_size - MFLIMIT; ret = kaelz4_compress_async_impl(&req->zc, req->src, compress_size, (void *)req); - if (unlikely(ret != KAE_LZ4_SUCC)) { + if (unlikely(ret != KAE_SNAPPY_SUCC)) { kaelz4_find_and_free_kz_ctx((kaelz4_ctx_t *)req->zc.kaeConfig); g_async_ctrl.ctx_index = (g_async_ctrl.ctx_index + MAX_NUM_IN_COMP - 1) % MAX_NUM_IN_COMP; g_async_ctrl.cur_num_in_comp--; @@ -929,13 +929,13 @@ static int kaelz4_async_compress_process(void *arg) struct kaelz4_async_req *req = (struct kaelz4_async_req *)kae_malloc(sizeof(struct kaelz4_async_req)); if (unlikely(req == NULL)) { US_ERR("Alloc kaelz4_async_req failed!\n"); - compress_ctx->status = KAE_LZ4_ALLOC_FAIL; + compress_ctx->status = KAE_SNAPPY_ALLOC_FAIL; if (compress_ctx->req_list) { tail->last = 1; // 有已经下发的req命令，这里不能返回失败，要统一走回收流程 - return KAE_LZ4_SUCC; + return KAE_SNAPPY_SUCC; } else { - return KAE_LZ4_ALLOC_FAIL; + return KAE_SNAPPY_ALLOC_FAIL; } } req->src = src; @@ -959,7 +959,7 @@ static int kaelz4_async_compress_process(void *arg) remainingLength = 0; } - int ret = KAE_LZ4_SUCC; + int ret = KAE_SNAPPY_SUCC; if (!req->special_flag) { ret = kaelz4_send_async_compress(req); } else { @@ -974,8 +974,8 @@ static int kaelz4_async_compress_process(void *arg) } idx++; tail = req; - if (ret != KAE_LZ4_SUCC) { - req->compress_ctx->status = KAE_LZ4_COMP_FAIL; + if (ret != KAE_SNAPPY_SUCC) { + req->compress_ctx->status = KAE_SNAPPY_COMP_FAIL; req->special_flag = 1; req->done = 1; } @@ -983,7 +983,7 @@ static int kaelz4_async_compress_process(void *arg) src += req->src_size; } - return KAE_LZ4_SUCC; + return KAE_SNAPPY_SUCC; } static void kaelz4_flush_compress(void) @@ -1010,14 +1010,14 @@ void kaelz4_async_deinit(void) kaelz4_free_all_qps(); } -const kaelz4_post_process_handle_t g_post_process_handle[KAELZ4_ASYNC_BUTT] = { - [KAELZ4_ASYNC_SMALL_BLOCK] = kaelz4_triples_rebuild, - [KAELZ4_ASYNC_BLOCK] = kaelz4_triples_rebuild_64Kblock, +const kaelz4_post_process_handle_t g_post_process_handle[KAESNAPPY_ASYNC_BUTT] = { + [KAESNAPPY_ASYNC_SMALL_BLOCK] = kaelz4_triples_rebuild, + [KAESNAPPY_ASYNC_BLOCK] = kaelz4_triples_rebuild_64Kblock, }; void kaelz4_compress_async(const void *src, void *dst, - lz4_async_callback callback, struct kaelz4_result *result, - enum kae_lz4_async_data_format data_format, const int *ptr) + snappy_async_callback callback, struct kaesnappy_result *result, + enum kae_snappy_async_data_format data_format, const int *ptr) { struct kaelz4_compress_ctx *compress_ctx = (struct kaelz4_compress_ctx *)kae_malloc(sizeof(struct kaelz4_compress_ctx)); if (unlikely(compress_ctx == NULL)) { @@ -1037,7 +1037,7 @@ void kaelz4_compress_async(const void *src, void *dst, compress_ctx->kaelz4_post_process_handle = g_post_process_handle[data_format]; compress_ctx->dst_len = 0; compress_ctx->next = NULL; - compress_ctx->status = KAE_LZ4_SUCC; + compress_ctx->status = KAE_SNAPPY_SUCC; compress_ctx->req_list = NULL; compress_ctx->prev_last_lit_ptr = NULL; compress_ctx->prev_last_lit_len = 0; @@ -1049,7 +1049,7 @@ void kaelz4_compress_async(const void *src, void *dst, } g_async_ctrl.tail = compress_ctx; - if (unlikely(kaelz4_async_compress_process(compress_ctx) != KAE_LZ4_SUCC)) { + if (unlikely(kaelz4_async_compress_process(compress_ctx) != KAE_SNAPPY_SUCC)) { goto free_compress_ctx; } @@ -1063,7 +1063,7 @@ free_compress_ctx: } err_callback: - result->status = KAE_LZ4_ALLOC_FAIL; + result->status = KAE_SNAPPY_ALLOC_FAIL; result->dst_len = 0; callback(result); return; diff --git a/KAESnappy/src/v1/kaesnappy_comp.h b/KAESnappy/src/v1/kaesnappy_comp.h index 4836f53..8194510 100644 --- a/KAESnappy/src/v1/kaesnappy_comp.h +++ b/KAESnappy/src/v1/kaesnappy_comp.h @@ -57,9 +57,9 @@ struct kaelz4_compress_ctx { size_t prev_last_lit_len; unsigned int recv_cnt; int status; - lz4_async_callback callback; - struct kaelz4_result *result; - enum kae_lz4_async_data_format data_format; + snappy_async_callback callback; + struct kaesnappy_result *result; + enum kae_snappy_async_data_format data_format; int preferences; kaelz4_post_process_handle_t kaelz4_post_process_handle; struct kaelz4_async_req *req_list; @@ -67,7 +67,7 @@ struct kaelz4_compress_ctx { }; struct kaelz4_async_req { - LZ4_CCtx zc; + SNAPPY_CCtx zc; const void* src; size_t src_size; U32 idx; @@ -90,8 +90,8 @@ struct kaelz4_async_ctrl { volatile int *stop_flag; }; -void kaelz4_setstatus_v1(LZ4_CCtx* zc, unsigned int status); -int kaelz4_compress_v1(LZ4_CCtx* zc, const void* src, size_t srcSize); +void kaelz4_setstatus_v1(SNAPPY_CCtx* zc, unsigned int status); +int kaelz4_compress_v1(SNAPPY_CCtx* zc, const void* src, size_t srcSize); // part1.frame模式的header & footer描述 #define KAELZ4_MAGIC_NUMBER 0x184D2204U diff --git a/KAESnappy/src/v1/kaesnappy_init.c b/KAESnappy/src/v1/kaesnappy_init.c index 2147636..d0c1a11 100644 --- a/KAESnappy/src/v1/kaesnappy_init.c +++ b/KAESnappy/src/v1/kaesnappy_init.c @@ -9,20 +9,20 @@ #include "kaesnappy_init.h" #include "kaesnappy_log.h" -int kaelz4_init_v1(LZ4_CCtx* zc) +int kaelz4_init_v1(SNAPPY_CCtx* zc) { kaelz4_ctx_t* kaelz4_ctx = kaelz4_get_ctx(WCRYPTO_LZ77_ONLY, WCRYPTO_DEFLATE); if (!kaelz4_ctx) { US_ERR("kaelz4 failed to get kaezip ctx!"); - return KAE_LZ4_INIT_FAIL; + return KAE_SNAPPY_INIT_FAIL; } zc->kaeConfig = (uintptr_t)kaelz4_ctx; US_INFO("kaelz4 deflate init success, kaelz4_ctx %p!", kaelz4_ctx); - return KAE_LZ4_SUCC; + return KAE_SNAPPY_SUCC; } -void kaelz4_reset_v1(LZ4_CCtx* zc) +void kaelz4_reset_v1(SNAPPY_CCtx* zc) { kaelz4_ctx_t* kaelz4_ctx = (kaelz4_ctx_t*)zc->kaeConfig; if (kaelz4_ctx) { @@ -32,7 +32,7 @@ void kaelz4_reset_v1(LZ4_CCtx* zc) } } -void kaelz4_release_v1(LZ4_CCtx* zc) +void kaelz4_release_v1(SNAPPY_CCtx* zc) { kaelz4_ctx_t* kaelz4_ctx = (kaelz4_ctx_t*)zc->kaeConfig; if (kaelz4_ctx) { diff --git a/KAESnappy/src/v1/kaesnappy_init.h b/KAESnappy/src/v1/kaesnappy_init.h index 7df8c73..66142b9 100644 --- a/KAESnappy/src/v1/kaesnappy_init.h +++ b/KAESnappy/src/v1/kaesnappy_init.h @@ -11,8 +11,8 @@ #include "kaesnappy_common.h" -int kaelz4_init_v1(LZ4_CCtx* zc); -void kaelz4_reset_v1(LZ4_CCtx* zc); -void kaelz4_release_v1(LZ4_CCtx* zc); +int kaelz4_init_v1(SNAPPY_CCtx* zc); +void kaelz4_reset_v1(SNAPPY_CCtx* zc); +void kaelz4_release_v1(SNAPPY_CCtx* zc); #endif \ No newline at end of file diff --git a/KAESnappy/src/v2/kaesnappy_compress.c b/KAESnappy/src/v2/kaesnappy_compress.c index ff14ce5..c7713e7 100644 --- a/KAESnappy/src/v2/kaesnappy_compress.c +++ b/KAESnappy/src/v2/kaesnappy_compress.c @@ -19,18 +19,18 @@ #include "kaesnappy_config.h" #include "kaesnappy_log.h" -void kaelz4_setstatus_v2(LZ4_CCtx* zc, unsigned int status) +void kaelz4_setstatus_v2(SNAPPY_CCtx* zc, unsigned int status) { KaeLz4Config *config; config = kaelz4_get_config(zc); config->tuple.bstatus = status; } -static int kaelz4_data_parsing(LZ4_CCtx* zc, KaeLz4Config* config) +static int kaelz4_data_parsing(SNAPPY_CCtx* zc, KaeLz4Config* config) { if (config->tuple.litStart == NULL || config->tuple.sequencesStart == NULL) { US_ERR("config parameter invalid\n"); - return KAE_LZ4_INVAL_PARA; + return KAE_SNAPPY_INVAL_PARA; } memcpy(zc->seqStore.litStart, config->tuple.litStart, config->tuple.litlen); @@ -40,7 +40,7 @@ static int kaelz4_data_parsing(LZ4_CCtx* zc, KaeLz4Config* config) config->tuple.seqnum*sizeof(seqDef)); zc->seqStore.sequences += config->tuple.seqnum; - if (config->tuple.longLengthType != LZ4_llt_none) { + if (config->tuple.longLengthType != SNAPPY_llt_none) { zc->seqStore.longLengthType = config->tuple.longLengthType; zc->seqStore.longLengthPos = config->tuple.longLengthPos; } @@ -48,7 +48,7 @@ static int kaelz4_data_parsing(LZ4_CCtx* zc, KaeLz4Config* config) return 0; } -int kaelz4_compress_v2(LZ4_CCtx* zc, const void* src, size_t srcSize) +int kaelz4_compress_v2(SNAPPY_CCtx* zc, const void* src, size_t srcSize) { KaeLz4Config *config = NULL; int ret; @@ -56,7 +56,7 @@ int kaelz4_compress_v2(LZ4_CCtx* zc, const void* src, size_t srcSize) US_INFO("KAE lz4 compress, srcSize is %lu", srcSize); if (zc == NULL || src == NULL || srcSize == 0) { US_ERR("compress parameter invalid\n"); - return KAE_LZ4_INVAL_PARA; + return KAE_SNAPPY_INVAL_PARA; } config = kaelz4_get_config(zc); diff --git a/KAESnappy/src/v2/kaesnappy_config.c b/KAESnappy/src/v2/kaesnappy_config.c index 5d52d47..3faa939 100644 --- a/KAESnappy/src/v2/kaesnappy_config.c +++ b/KAESnappy/src/v2/kaesnappy_config.c @@ -46,7 +46,7 @@ static inline int kaelz4_unlock() return pthread_mutex_unlock(&kz_lz4_mutex); } -inline KaeLz4Config* kaelz4_get_config(LZ4_CCtx* zc) +inline KaeLz4Config* kaelz4_get_config(SNAPPY_CCtx* zc) { KaeLz4Config* config = (KaeLz4Config*)(zc->kaeConfig); @@ -57,7 +57,7 @@ inline KaeLz4Config* kaelz4_get_config(LZ4_CCtx* zc) } } -inline void kaelz4_set_config(LZ4_CCtx* zc, KaeLz4Config* config) +inline void kaelz4_set_config(SNAPPY_CCtx* zc, KaeLz4Config* config) { if (zc != NULL) { zc->kaeConfig = (uintptr_t)config; @@ -153,7 +153,7 @@ static int kaelz4_create_session(KaeLz4Config *config, int lz4_level) config->sess = wd_comp_alloc_sess(&(config->setup)); if (!(config->sess)) { US_ERR("failed to alloc comp sess!\n"); - return KAE_LZ4_ALLOC_FAIL; + return KAE_SNAPPY_ALLOC_FAIL; } config->req.dst = malloc(REQ_DSTBUFF_LEN); config->req.dst_len = REQ_DSTBUFF_LEN; @@ -184,7 +184,7 @@ static int kaelz4_alg_init2(void) ctx_set_num = calloc(KAELZ4_CTX_SET_NUM, sizeof(*ctx_set_num)); if (!ctx_set_num) { WD_ERR("failed to alloc ctx_set_size!\n"); - return KAE_LZ4_ALLOC_FAIL; + return KAE_SNAPPY_ALLOC_FAIL; } cparams.op_type_num = KAELZ4_CTX_SET_NUM; @@ -192,7 +192,7 @@ static int kaelz4_alg_init2(void) cparams.bmp = numa_allocate_nodemask(); if (!cparams.bmp) { WD_ERR("failed to create nodemask!\n"); - ret = KAE_LZ4_INIT_FAIL; + ret = KAE_SNAPPY_INIT_FAIL; goto out_freectx; } @@ -201,7 +201,7 @@ static int kaelz4_alg_init2(void) struct uacce_dev *dev = wd_get_accel_dev("lz77_zstd");//获取支持某种算法的最亲和的设备 if (dev == NULL) { - ret = KAE_LZ4_INIT_FAIL; + ret = KAE_SNAPPY_INIT_FAIL; goto out_freebmp; } numa_bitmask_setbit(cparams.bmp, dev->numa_id); @@ -213,7 +213,7 @@ static int kaelz4_alg_init2(void) ret = wd_comp_init2_("lz77_zstd", 0, 1, &cparams); if (ret && ret != -WD_EEXIST) { WD_ERR("failed to init wd_comp_init2_ ret is :%d!\n", ret); - ret = KAE_LZ4_INIT_FAIL; + ret = KAE_SNAPPY_INIT_FAIL; goto out_freedev; } atexit(lz4_uadk_uninit); // 注册退出处理函数 @@ -229,7 +229,7 @@ out_freectx: return ret; } -int kaelz4_init_v2(LZ4_CCtx* zc) +int kaelz4_init_v2(SNAPPY_CCtx* zc) { int ret; KaeLz4Config *config = NULL; @@ -238,7 +238,7 @@ int kaelz4_init_v2(LZ4_CCtx* zc) config = (KaeLz4Config*)malloc(sizeof(KaeLz4Config)); if (config == NULL) { US_ERR("failed to alloc config!\n"); - return KAE_LZ4_INIT_FAIL; + return KAE_SNAPPY_INIT_FAIL; } memset(config, 0, sizeof(KaeLz4Config)); kaelz4_options_init(config); @@ -265,10 +265,10 @@ int kaelz4_init_v2(LZ4_CCtx* zc) free_config: free(config); kaelz4_unlock(); - return KAE_LZ4_INIT_FAIL; + return KAE_SNAPPY_INIT_FAIL; } -void kaelz4_release_v2(LZ4_CCtx* zc) +void kaelz4_release_v2(SNAPPY_CCtx* zc) { KaeLz4Config *config = NULL; if (zc == NULL) { diff --git a/KAESnappy/src/v2/kaesnappy_config.h b/KAESnappy/src/v2/kaesnappy_config.h index 99996e8..d90d503 100644 --- a/KAESnappy/src/v2/kaesnappy_config.h +++ b/KAESnappy/src/v2/kaesnappy_config.h @@ -22,7 +22,7 @@ typedef struct Comp4Tuple_S { seqDef* sequencesStart; /* sequences address start */ unsigned int litlen; /* literal lens */ unsigned int seqnum; /* sequences lens */ - LZ4_longLengthType_e longLengthType; /* litlen overflow flag */ + SNAPPY_longLengthType_e longLengthType; /* litlen overflow flag */ unsigned int longLengthPos; /* litlen overflow position */ char* additional_p; /* addition data ptr */ /* @@ -64,7 +64,7 @@ typedef struct KaeLz4Config_S { #define REQ_WINDOW_SIZE 2 #define REQ_COMPRESS_LEVEL 8 -KaeLz4Config* kaelz4_get_config(LZ4_CCtx* zc); -void kaelz4_set_config(LZ4_CCtx* zc, KaeLz4Config* config); +KaeLz4Config* kaelz4_get_config(SNAPPY_CCtx* zc); +void kaelz4_set_config(SNAPPY_CCtx* zc, KaeLz4Config* config); #endif -- Gitee