diff --git a/kae_zip_engine/Makefile b/kae_zip_engine/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..56b3128c4d31ff4bbbd666a64d4f1c5080df9c7b --- /dev/null +++ b/kae_zip_engine/Makefile @@ -0,0 +1,103 @@ +# +# Date: 2020/2/10 + +# Description: +# compile for KAEzip +# +# Usage: +# $ make compile and link the program. +# $ make rebuild rebuild the program. The same as make clean && make all. +# $ make clean clean the objective, dependent and executable files. +# $ make install copy to the system directory. +# $ make uninstall clean the executable file from the system directory. +#============================================================================== +WORK_PATH := . +ENGINE_INSTALL_PATH := /usr/local/kaezip + +CC=gcc + +LIBNAME := libkaezip.so +VERSION = 1.3.11 +TARGET = ${LIBNAME}.${VERSION} +SOFTLINK = libkaezip.so + +ifndef SILENCE + SILENCE = @ +endif + +# Src +SRCDIRS := ${WORK_PATH}/ +SRCDIRS += ${WORK_PATH}/src + +SRCEXTS := .c # C program + +# Include +INCDIR += -I $(WORK_PATH)/ +INCDIR += -I $(WORK_PATH)/include +INCDIR += -I $(WORK_PATH)/open_source/zlib-1.2.11 +#INCDIR += -I $(WORK_PATH)/open_source/zlib-1.2.7 +INCDIR += -I /usr/local/include/uadk/ +#INCDIR += -I /usr/include/warpdrive/ + +# Include Libs. +LIBDIR += -L/usr/local/lib +LIBDIR += -L/usr/lib64 +LIBDIR += -L$(WORK_PATH)/open_source/zlib-1.2.11 +#LIBDIR += -L$(WORK_PATH)/open_source/zlib-1.2.7 +LIBS := -lz -lwd -pthread +LIBS += -lc_nonshared + +# The flags +CFLAGS := -Wall -Werror -fstack-protector-all -fPIC -D_GNU_SOURCE -shared +LDFLAGS := $(LIBDIR) +LDFLAGS += $(LIBS) +LDFLAGS += -Wl,-z,relro,-z,now,-z,noexecstack #safe link option + +# The command used to delete file. +RM = rm -f +LN = ln -sf + +SOURCES = $(foreach d,$(SRCDIRS),$(wildcard $(addprefix $(d)/*,$(SRCEXTS)))) + +OBJS = $(foreach x,$(SRCEXTS), \ + $(patsubst %$(x), %.o, $(filter %$(x),$(SOURCES)))) + +.PHONY : all objs clean cleanall rebuild + +all : $(TARGET) + +# Rules for creating the dependency files (.d). +%.d : %.c + $(CC) -MM -MD $(CFLAGS) $< + +# Rules for producing the objects. +objs : $(OBJS) +%.o : %.c + @echo compiling $(notdir $<) + $(SILENCE) $(CC) -c $(CFLAGS) $(INCDIR) $(LDFLAGS) -D$(KAE) -o $@ $< + +$(TARGET): $(OBJS) + @echo Linking $@ + $(SILENCE) $(CC) $(CFLAGS) $(INCDIR) -D$(KAE) -o ./$(TARGET) $(OBJS) $(LDFLAGS) +rebuild: clean all + +clean : + @-$(RM) *.d *.a *.so *.symbol $(TARGET) + @-$(RM) + @find ${WORK_PATH} -name '*.o' -exec $(RM) {} \; + @echo all clean + +install : + mkdir -p $(ENGINE_INSTALL_PATH)/include + mkdir -p $(ENGINE_INSTALL_PATH)/lib + install -m 755 $(TARGET) $(ENGINE_INSTALL_PATH)/lib + $(LN) $(ENGINE_INSTALL_PATH)/lib/$(TARGET) $(ENGINE_INSTALL_PATH)/lib/$(SOFTLINK) + $(LN) $(ENGINE_INSTALL_PATH)/lib/$(TARGET) $(ENGINE_INSTALL_PATH)/lib/$(SOFTLINK).0 + install -m 755 $(WORK_PATH)/include/kaezip.h $(ENGINE_INSTALL_PATH)/include +uninstall : + $(RM) $(ENGINE_INSTALL_PATH)/lib/$(SOFTLINK) + $(RM) $(ENGINE_INSTALL_PATH)/lib/$(SOFTLINK).0 + $(RM) $(ENGINE_INSTALL_PATH)/lib/$(TARGET) + $(RM) $(ENGINE_INSTALL_PATH)/include/kaezip.h + $(RM) /var/log/kaezip.log + $(RM) /var/log/kaezip.log.old \ No newline at end of file diff --git a/kae_zip_engine/README.md b/kae_zip_engine/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f17bca03387aff7a8fe26629d3e3ed5ddff16ab --- /dev/null +++ b/kae_zip_engine/README.md @@ -0,0 +1,121 @@ +# Kunpeng Zlib Acceleration Engine + +- [Introduction](#introduction) +- [License](#license) +- [Requirements](#requirements) +- [Installation Instructions](#installation-instructions) +- [Test Performace](#test-performace) +- [Contribution Guidelines](#contribution-Guidelines) +- [Vulnerability Management](#Vulnerability-Management) +- [Quality Requirements](#Quality-Requirements) +- [Secure Design](#Secure-Design) +- [More Information](#more-information) +- [Copyright](#copyright) + +## Introduction + +Kunpeng Zlib Acceleration Engine is used to build performance competitiveness of common software libraries on the Kunpeng platform. + +As a new function in the HiSilicon Kunpeng 920 processors, Kunpeng Zlib Acceleration Engine provides a hardware-enabled foundation for compression. It significantly increases the performance across cloud, big data, and storage applications and platforms. By using Kunpeng Zlib Acceleration Engine, you can: + +- Have higher-performance compression and decompression +- Maximize CPU utilization + +The compression and decompression algorithm supported by Kunpeng Acceleration Engine are: gzip/zlib + +## License + +It is licensed under the zlib License(https://www.zlib.net/zlib_license.html ). For more information, see the LICENSE file. + +## Requirements + +- CPU: Kunpeng 920 +- Support Operating System: + - CentOS 7.6 4.14.0-115.el7a.0.1.aarch64 version + - SuSE 15.1 4.12.14-195-default arch64 version + - NeoKylin 7.6 4.14.0-115.5.1.el7a.06.aarch64 version + - EulerOS 2.8 4.19.36-vhulk1907.1.0.h410.eulerosv2r8.aarch64 version + - BCLinux-R7-U6-Server-aarch64 version + - Kylin 4.0.2 (juniper) 4.15.0-70-generic version + - Kylin release 4.0.2 (SP2) 4.19.36-vhulk1907.1.0.h403.ky4.aarch64 version + - UniKylin Linux release 3(Core) 4.18.0-80.ky3.kb21.hw.aarch64 version + - Ubuntu 18.04.1 LTS 4.15.0-29-generic version + +## Installation Instructions + +Clone the Github repository containing the Kunpeng Accelerator Engine Driver. + +``` +git clone https://github.com/kunpengcompute/KAEzip +``` + +Download the release version of Kunpeng Accelerator Engine Driver from: + + + +Firstly, build and install the accelerator driver. +Note: To build the Kunpeng Accelerator Engine Driver, install the `kernel-devel` package first. + +``` +tar -zxf Kunpeng_KAE_driver.tar.gz +cd kae_driver +make +make install +modprobe uacce +modprobe hisi_qm +modprobe hisi_zip +``` + +Secondly, install the accelerator library: + +``` +cd warpdrive +sh autogen.sh +./configure +make +make install +``` + +Then Install KAEzip library: + +Download [zlib-1.2.11.tar.gz](https://www.zlib.net/zlib-1.2.11.tar.gz) at KAEzip/open_source. + +``` +cd KAEzip +sh setup.sh install +``` + +for more install guid and user guid, get information at: + + +## Test Performace + +``` +cd test +make +export LD_LIBRARY_PATH=/usr/local/kaezip/lib +./kaezip_perf -m 8 -l 1024 -n 1000 +./kaezip_perf -m 8 -l 1024 -n 1000 -d +``` +## Contribution Guidelines + +If you want to contribute to KAEzip, please use GitHub [issues](https://github.com/kunpengcompute/KAEzip/issues/new) for tracking requests and bugs. + +## Vulnerability Management +Please refer to https://github.com/kunpengcompute/Kunpeng#security + +## Quality Requirements +Please refer to [Secure Coding Specifications](https://github.com/kunpengcompute/Kunpeng/blob/master/security/SecureCoding.md). + +## Secure Design +Please refer to [Secure Design](https://github.com/kunpengcompute/Kunpeng/blob/master/security/SecureDesign.md). + +## More Information + +For further assistance and more QAs, contact Huawei Support at: + + + +## Copyright + +Copyright © 2018 Huawei Corporation. All rights reserved. diff --git a/kae_zip_engine/include/kaezip.h b/kae_zip_engine/include/kaezip.h new file mode 100644 index 0000000000000000000000000000000000000000..53cc84878d23576ceb99a17caaf743b8f0df2ea8 --- /dev/null +++ b/kae_zip_engine/include/kaezip.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip.h + * + * This file provides the interface for zlib; + * + *****************************************************************************/ + +#ifndef KAEZIP_H +#define KAEZIP_H +#include "zlib.h" + +extern int kz_get_devices(void); +extern int kz_deflate(z_streamp strm, int flush); +extern int kz_inflate(z_streamp strm, int flush); +extern int kz_inflateEnd(z_streamp strm); +extern int kz_deflateInit2_(z_streamp strm, int level, int metho, int windowBit, int memLevel, int strategy, + const char *version, int stream_size); +extern int kz_inflateInit2_(z_streamp strm, int windowBits, const char *version, int stream_size); +extern int kz_deflateEnd(z_streamp strm); +extern int kz_inflateReset(z_streamp strm); +extern int kz_deflateReset(z_streamp strm); +extern int kz_getAutoInflateAlgType(z_streamp strm); +extern int kz_do_inflateInit(z_streamp strm, int alg_comp_type); + +extern int lz_deflateEnd(z_streamp strm); +extern int lz_deflateInit2_(z_streamp strm, int level, int metho, int windowBit, int memLevel, int strategy, + const char *version, int stream_size); +extern int lz_deflateReset(z_streamp strm); + +extern int lz_inflateEnd(z_streamp strm); +extern int lz_inflateInit2_(z_streamp strm, int windowBits, const char *version, int stream_size); +extern int lz_inflateReset(z_streamp strm); + +extern int getInflateStateWrap(z_streamp strm); +extern unsigned long getInflateKaezipCtx(z_streamp strm); +extern void setInflateKaezipCtx(z_streamp strm, unsigned long kaezip_ctx); +extern unsigned long getDeflateKaezipCtx(z_streamp strm); +extern void setDeflateKaezipCtx(z_streamp strm, unsigned long kaezip_ctx); +#endif + diff --git a/kae_zip_engine/open_source/.gitkeep b/kae_zip_engine/open_source/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/kae_zip_engine/patch/kaezip_for_zlib-1.2.11.patch b/kae_zip_engine/patch/kaezip_for_zlib-1.2.11.patch new file mode 100644 index 0000000000000000000000000000000000000000..955da20d4105d9afd8d2e255220a94404823f272 --- /dev/null +++ b/kae_zip_engine/patch/kaezip_for_zlib-1.2.11.patch @@ -0,0 +1,479 @@ +diff -Naru zlib-1.2.11/compress.c ./zlib-1.2.11_new/compress.c +--- zlib-1.2.11/compress.c 2017-01-01 15:37:10.000000000 +0800 ++++ ./zlib-1.2.11_new/compress.c 2020-09-17 11:38:29.404655410 +0800 +@@ -81,6 +81,10 @@ + uLong ZEXPORT compressBound (sourceLen) + uLong sourceLen; + { ++#ifdef CONF_KAEZIP ++ return sourceLen + (sourceLen >> 3) + 13; ++#else + return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + +- (sourceLen >> 25) + 13; ++ (sourceLen >> 25) + 13; ++#endif + } +diff -Naru zlib-1.2.11/deflate.c ./zlib-1.2.11_new/deflate.c +--- zlib-1.2.11/deflate.c 2017-01-16 01:29:40.000000000 +0800 ++++ ./zlib-1.2.11_new/deflate.c 2020-09-17 11:41:29.454655410 +0800 +@@ -50,6 +50,7 @@ + /* @(#) $Id$ */ + + #include "deflate.h" ++#include "kaezip.h" + + const char deflate_copyright[] = + " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; +@@ -236,8 +237,9 @@ + /* To do: ignore strm->next_in if we use it as window */ + } + ++ + /* ========================================================================= */ +-int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, ++int ZEXPORT lz_deflateInit2_(strm, level, method, windowBits, memLevel, strategy, + version, stream_size) + z_streamp strm; + int level; +@@ -344,7 +346,29 @@ + s->strategy = strategy; + s->method = (Byte)method; + +- return deflateReset(strm); ++ return lz_deflateReset(strm); ++} ++ ++int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, ++ version, stream_size) ++ z_streamp strm; ++ int level; ++ int method; ++ int windowBits; ++ int memLevel; ++ int strategy; ++ const char *version; ++ int stream_size; ++{ ++#ifdef CONF_KAEZIP ++ if (kz_get_devices()) { ++ return kz_deflateInit2_(strm, level, method, windowBits, memLevel, strategy, ++ version, stream_size); ++ } ++#endif ++ ++ return lz_deflateInit2_(strm, level, method, windowBits, memLevel, strategy, ++ version, stream_size); + } + + /* ========================================================================= +@@ -502,7 +526,7 @@ + } + + /* ========================================================================= */ +-int ZEXPORT deflateReset (strm) ++int ZEXPORT lz_deflateReset (strm) + z_streamp strm; + { + int ret; +@@ -513,6 +537,18 @@ + return ret; + } + ++int ZEXPORT deflateReset(strm) ++z_streamp strm; ++{ ++#ifdef CONF_KAEZIP ++ if (kz_get_devices()) { ++ return kz_deflateReset(strm); ++ } ++#endif ++ ++ return lz_deflateReset(strm); ++} ++ + /* ========================================================================= */ + int ZEXPORT deflateSetHeader (strm, head) + z_streamp strm; +@@ -653,6 +689,17 @@ + z_streamp strm; + uLong sourceLen; + { ++#ifdef CONF_KAEZIP ++ return sourceLen + (sourceLen >> 3) + 13; ++#else ++ return lz_deflateBound(strm, sourceLen); ++#endif ++} ++ ++uLong ZEXPORT lz_deflateBound(strm, sourceLen) ++ z_streamp strm; ++ uLong sourceLen; ++{ + deflate_state *s; + uLong complen, wraplen; + +@@ -760,7 +807,8 @@ + } while (0) + + /* ========================================================================= */ +-int ZEXPORT deflate (strm, flush) ++ ++int ZEXPORT lz_deflate (strm, flush) + z_streamp strm; + int flush; + { +@@ -1072,10 +1120,30 @@ + return s->pending != 0 ? Z_OK : Z_STREAM_END; + } + ++int ZEXPORT deflate(strm, flush) ++z_streamp strm; ++int flush; ++{ ++#ifdef CONF_KAEZIP ++ if (kz_get_devices()) { ++ unsigned long kaezip_ctx = getDeflateKaezipCtx(strm); ++ if (kaezip_ctx != 0 && flush != Z_PARTIAL_FLUSH && flush != Z_TREES) { ++ return kz_deflate(strm, flush); ++ } else { ++ return lz_deflate(strm, flush); ++ } ++ } ++#endif ++ ++ return lz_deflate(strm, flush); ++} ++ + /* ========================================================================= */ +-int ZEXPORT deflateEnd (strm) ++ ++int ZEXPORT lz_deflateEnd (strm) + z_streamp strm; + { ++ + int status; + + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; +@@ -1091,9 +1159,22 @@ + ZFREE(strm, strm->state); + strm->state = Z_NULL; + ++ + return status == BUSY_STATE ? Z_DATA_ERROR : Z_OK; + } + ++int ZEXPORT deflateEnd (strm) ++ z_streamp strm; ++{ ++#ifdef CONF_KAEZIP ++ if (kz_get_devices()) { ++ return kz_deflateEnd(strm); ++ } ++#endif ++ ++ return lz_deflateEnd(strm); ++} ++ + /* ========================================================================= + * Copy the source state to the destination state. + * To simplify the source, this is not supported for 16-bit MSDOS (which +@@ -2161,3 +2242,36 @@ + FLUSH_BLOCK(s, 0); + return block_done; + } ++ ++unsigned long getDeflateKaezipCtx(z_streamp strm) ++{ ++ deflate_state *state; ++ ++ if (strm == Z_NULL) { ++ return (unsigned long)0; ++ } ++ state = (deflate_state *)strm->state; ++ if (state == Z_NULL) { ++ return (unsigned long)0; ++ } ++ state = (deflate_state *)strm->state; ++ ++ return state->kaezip_ctx; ++} ++ ++void setDeflateKaezipCtx(z_streamp strm, unsigned long kaezip_ctx) ++{ ++ deflate_state *state; ++ ++ if (strm == Z_NULL) { ++ return; ++ } ++ state = (deflate_state *)strm->state; ++ if (state == Z_NULL) { ++ return; ++ } ++ ++ state->kaezip_ctx = kaezip_ctx; ++ return; ++} ++ +diff -Naru zlib-1.2.11/deflate.h ./zlib-1.2.11_new/deflate.h +--- zlib-1.2.11/deflate.h 2017-01-01 15:37:10.000000000 +0800 ++++ ./zlib-1.2.11_new/deflate.h 2020-09-17 11:41:29.494655410 +0800 +@@ -272,7 +272,7 @@ + * longest match routines access bytes past the input. This is then + * updated to the new high water mark. + */ +- ++ ulg kaezip_ctx; /* kunpeng kaezip context */ + } FAR deflate_state; + + /* Output a byte on the stream. +diff -Naru zlib-1.2.11/inflate.c ./zlib-1.2.11_new/inflate.c +--- zlib-1.2.11/inflate.c 2017-01-01 15:37:10.000000000 +0800 ++++ ./zlib-1.2.11_new/inflate.c 2020-09-17 11:41:29.434655410 +0800 +@@ -84,6 +84,7 @@ + #include "inftrees.h" + #include "inflate.h" + #include "inffast.h" ++#include "kaezip.h" + + #ifdef MAKEFIXED + # ifndef BUILDFIXED +@@ -141,7 +142,7 @@ + return Z_OK; + } + +-int ZEXPORT inflateReset(strm) ++int ZEXPORT lz_inflateReset(strm) + z_streamp strm; + { + struct inflate_state FAR *state; +@@ -154,6 +155,18 @@ + return inflateResetKeep(strm); + } + ++int ZEXPORT inflateReset(strm) ++z_streamp strm; ++{ ++#ifdef CONF_KAEZIP ++ if (kz_get_devices()) { ++ return kz_inflateReset(strm); ++ } ++#endif ++ ++ return lz_inflateReset(strm); ++} ++ + int ZEXPORT inflateReset2(strm, windowBits) + z_streamp strm; + int windowBits; +@@ -189,10 +202,10 @@ + /* update state and reset the rest of it */ + state->wrap = wrap; + state->wbits = (unsigned)windowBits; +- return inflateReset(strm); ++ return lz_inflateReset(strm); + } + +-int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) ++int ZEXPORT lz_inflateInit2_(strm, windowBits, version, stream_size) + z_streamp strm; + int windowBits; + const char *version; +@@ -236,6 +249,21 @@ + return ret; + } + ++int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) ++z_streamp strm; ++int windowBits; ++const char *version; ++int stream_size; ++{ ++#ifdef CONF_KAEZIP ++ if (kz_get_devices()) { ++ return kz_inflateInit2_(strm, windowBits, version, stream_size); ++ } ++#endif ++ ++ return lz_inflateInit2_(strm, windowBits, version, stream_size); ++} ++ + int ZEXPORT inflateInit_(strm, version, stream_size) + z_streamp strm; + const char *version; +@@ -619,7 +647,8 @@ + will return Z_BUF_ERROR if it has not reached the end of the stream. + */ + +-int ZEXPORT inflate(strm, flush) ++ ++int ZEXPORT lz_inflate(strm, flush) + z_streamp strm; + int flush; + { +@@ -1273,8 +1302,30 @@ + ret = Z_BUF_ERROR; + return ret; + } ++int ZEXPORT inflate(strm, flush) ++z_streamp strm; ++int flush; ++{ ++#ifdef CONF_KAEZIP ++ int alg_type; ++ unsigned long kaezip_ctx; ++ if (kz_get_devices()) { ++ alg_type = kz_getAutoInflateAlgType(strm); ++ (void)kz_do_inflateInit(strm, alg_type); ++ kaezip_ctx = getInflateKaezipCtx(strm); ++ if (kaezip_ctx != 0 && flush != Z_PARTIAL_FLUSH ++ && flush != Z_TREES) { ++ return kz_inflate(strm, flush); ++ } else { ++ return lz_inflate(strm, flush); ++ } ++ } ++#endif + +-int ZEXPORT inflateEnd(strm) ++ return lz_inflate(strm, flush); ++} ++ ++int ZEXPORT lz_inflateEnd(strm) + z_streamp strm; + { + struct inflate_state FAR *state; +@@ -1285,9 +1336,22 @@ + ZFREE(strm, strm->state); + strm->state = Z_NULL; + Tracev((stderr, "inflate: end\n")); ++ + return Z_OK; + } + ++int ZEXPORT inflateEnd(strm) ++z_streamp strm; ++{ ++#ifdef CONF_KAEZIP ++ if (kz_get_devices()) { ++ return kz_inflateEnd(strm); ++ } ++#endif ++ ++ return lz_inflateEnd(strm); ++} ++ + int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) + z_streamp strm; + Bytef *dictionary; +@@ -1397,6 +1461,7 @@ + return next; + } + ++ + int ZEXPORT inflateSync(strm) + z_streamp strm; + { +@@ -1434,7 +1499,7 @@ + /* return no joy or set up to restart inflate() on a new block */ + if (state->have != 4) return Z_DATA_ERROR; + in = strm->total_in; out = strm->total_out; +- inflateReset(strm); ++ lz_inflateReset(strm); + strm->total_in = in; strm->total_out = out; + state->mode = TYPE; + return Z_OK; +@@ -1559,3 +1624,53 @@ + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); + } ++ ++int getInflateStateWrap(z_streamp strm) ++{ ++ struct inflate_state FAR *state; ++ ++ if (strm == Z_NULL) { ++ return 0; ++ } ++ state = (struct inflate_state FAR *)strm->state; ++ ++ if (state == Z_NULL) { ++ return 0; ++ } ++ ++ return state->wrap; ++} ++ ++unsigned long getInflateKaezipCtx(z_streamp strm) ++{ ++ struct inflate_state FAR *state; ++ ++ if (strm == Z_NULL) { ++ return (unsigned long)0; ++ } ++ ++ state = (struct inflate_state FAR *)strm->state; ++ if (state == Z_NULL) { ++ return (unsigned long)0; ++ } ++ ++ return state->kaezip_ctx; ++} ++ ++void setInflateKaezipCtx(z_streamp strm, unsigned long kaezip_ctx) ++{ ++ struct inflate_state FAR *state; ++ ++ if (strm == Z_NULL) { ++ return; ++ } ++ state = (struct inflate_state FAR *)strm->state; ++ ++ if (state == Z_NULL) { ++ return; ++ } ++ ++ state->kaezip_ctx = kaezip_ctx; ++ return; ++} ++ +diff -Naru zlib-1.2.11/inflate.h ./zlib-1.2.11_new/inflate.h +--- zlib-1.2.11/inflate.h 2017-01-01 15:37:10.000000000 +0800 ++++ ./zlib-1.2.11_new/inflate.h 2020-09-17 11:41:29.474655410 +0800 +@@ -122,4 +122,5 @@ + int sane; /* if false, allow invalid distance too far */ + int back; /* bits back of last unprocessed length/lit */ + unsigned was; /* initial length of match */ ++ unsigned long kaezip_ctx; /* kunpeng kaezip context */ + }; +diff -Naru zlib-1.2.11/Makefile.in ./zlib-1.2.11_new/Makefile.in +--- zlib-1.2.11/Makefile.in 2017-01-16 01:29:40.000000000 +0800 ++++ ./zlib-1.2.11_new/Makefile.in 2020-09-17 11:38:29.414655410 +0800 +@@ -26,7 +26,15 @@ + + SFLAGS=-O + LDFLAGS= ++ ++KAEZIP_CFLAGS=-I/usr/local/kaezip/include -I. -DCONF_KAEZIP ++KAEZIP_LDFLAGS=-L/usr/lib64 -L/usr/local/lib -L/usr/local/kaezip/lib -Wl,-rpath,/usr/local/kaezip/lib:/usr/local/lib -lkaezip -lwd ++CFLAGS+=$(KAEZIP_CFLAGS) ++SFLAGS+=$(KAEZIP_CFLAGS) ++LDFLAGS+=$(KAEZIP_LDFLAGS) + TEST_LDFLAGS=-L. libz.a ++ ++TEST_LDFLAGS+=$(KAEZIP_LDFLAGS) + LDSHARED=$(CC) + CPP=$(CC) -E + +@@ -292,10 +300,10 @@ + $(CC) $(CFLAGS) -o $@ minigzip.o $(TEST_LDFLAGS) + + examplesh$(EXE): example.o $(SHAREDLIBV) +- $(CC) $(CFLAGS) -o $@ example.o -L. $(SHAREDLIBV) ++ $(CC) $(CFLAGS) -o $@ example.o -L. $(SHAREDLIBV) $(KAEZIP_LDFLAGS) + + minigzipsh$(EXE): minigzip.o $(SHAREDLIBV) +- $(CC) $(CFLAGS) -o $@ minigzip.o -L. $(SHAREDLIBV) ++ $(CC) $(CFLAGS) -o $@ minigzip.o -L. $(SHAREDLIBV) $(KAEZIP_LDFLAGS) + + example64$(EXE): example64.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ example64.o $(TEST_LDFLAGS) diff --git a/kae_zip_engine/setup.sh b/kae_zip_engine/setup.sh new file mode 100644 index 0000000000000000000000000000000000000000..dbee5ba5166bc1df68e3491c219172e8b7d3fb22 --- /dev/null +++ b/kae_zip_engine/setup.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# Copyright © Huawei Technologies Co., Ltd. 2010-2019. All rights reserved. +# @rem Copyright © Huawei Technologies Co., Ltd. 2010-2019. All rights reserved. +# @rem Description: build script +set -e +#set -x +SRC_PATH=$(pwd) +BUILDVERSION=$(ls "${SRC_PATH}"/open_source | grep libwd | awk '{print substr($0,7,5)}') + +function Install_warpdrive() +{ + local wd_src_path=$(ls /usr/local/lib | grep libwd.so.${BUILDVERSION}) + local wd_rpm_path=$(ls /usr/lib64 | grep libwd.so.${BUILDVERSION}) + if [ ! -n "${wd_src_path}" ] && [ ! -n "${wd_rpm_path}" ]; then + cd "${SRC_PATH}"/open_source + rm -rf warpdrive + tar -zxvf libwd-"${BUILDVERSION}".tar.gz + + cd warpdrive/ + sh autogen.sh + ./configure + make clean && make + make install + fi +} + +function Target_zlib() +{ + cd "${SRC_PATH}"/open_source + rm -rf zlib-1.2.11 + tar -zxvf zlib-1.2.11.tar.gz + cd "${SRC_PATH}"/open_source/zlib-1.2.11/ + #./configure + #make +} + +function Build_kaezip() +{ + #Install_warpdrive + Target_zlib + cd "${SRC_PATH}" + if [ "$1" = "KAE2" ];then + make clean && make KAE=KAE2 + else + make clean && make KAE=KAE + fi + make install + echo "install kaezip" + + cd - + patch -Np1 < ../../patch/kaezip_for_zlib-1.2.11.patch + ./configure --prefix=/usr/local/kaezip + make + echo "build zlib success" +} + +function Install_kaezip() +{ + if [ -d "${SRC_PATH}"/open_source/zlib-1.2.11/ ]; then + cd "${SRC_PATH}"/open_source/zlib-1.2.11/ + make install + fi + echo "install zlib success" +} + +function Uninstall_kaezip() +{ + local zlib_path= + if [ -d "${SRC_PATH}"/open_source/zlib-1.2.11/ ]; then + set +e + zlib_path=$(ls /usr/local/kaezip/lib | grep libz.so.1.2.11) + set -e + if [ -n "${zlib_path}" ]; then + cd "${SRC_PATH}"/open_source/zlib-1.2.11/ + make uninstall && make clean + rm -rf "${SRC_PATH}"/open_source/zlib-1.2.11 + fi + fi + +# local wd_src_path=$(ls /usr/local/lib | grep libwd.so.${BUILDVERSION}) +# local wd_rpm_path=$(ls /usr/lib64 | grep libwd.so.${BUILDVERSION}) +# if [ -n "${wd_src_path}" ] || [ -n "${wd_rpm_path}" ]; then +# if [ -d "${SRC_PATH}"/open_source/warpdrive ]; then +# cd "${SRC_PATH}"/open_source/warpdrive +# make uninstall && make clean +# rm -rf "${SRC_PATH}"/open_source/warpdrive +# fi +# fi + + local kaezip_path=$(ls /usr/local/kaezip/lib | grep libkaezip.so.${BUILDVERSION}) + if [ -n "${kaezip_path}" ]; then + if [ -d "${SRC_PATH}" ]; then + cd "${SRC_PATH}" + make uninstall && make clean + fi + fi + echo "uninstall success" +} + +function Operate() +{ + cd "${SRC_PATH}"/open_source + case "$1" in + build) + Build_kaezip "$2" + ;; + install) + Build_kaezip "$2" + Install_kaezip + ;; + uninstall) + Uninstall_kaezip + ;; + esac +} + +function main() +{ + Operate "$1" "$2" +} + +main "$@" +exit $? diff --git a/kae_zip_engine/src/kaezip_common.c b/kae_zip_engine/src/kaezip_common.c new file mode 100644 index 0000000000000000000000000000000000000000..741252eafb1d488d703e0ffae91601709256a6c4 --- /dev/null +++ b/kae_zip_engine/src/kaezip_common.c @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_common.c + * + * This file provides the common implemenation for ZIP engine dealing with wrapdrive + * + *****************************************************************************/ +#include +#include +#include "kaezip_common.h" +#include "kaezip_ctx.h" +#include "kaezip_log.h" +#include "uadk/v1/wd.h" +#include "uadk/v1/wd_comp.h" + +#define __swab32(x) \ + ((((x)&0x000000ff) << 24) | \ + (((x)&0x0000ff00) << 8) | \ + (((x)&0x00ff0000) >> 8) | \ + (((x)&0xff000000) >> 24)) +#define __cpu_to_be32(x) __swab32(x) + +static unsigned int inline __kaezip_checksum_reverse(unsigned int x) +{ + x = (((x & 0xaaaaaaaa) >> 1) | ((x & 0x55555555) << 1)); + x = (((x & 0xcccccccc) >> 2) | ((x & 0x33333333) << 2)); + x = (((x & 0xf0f0f0f0) >> 4) | ((x & 0x0f0f0f0f) << 4)); + x = (((x & 0xff00ff00) >> 8) | ((x & 0x00ff00ff) << 8)); + + return ((x >> 16) | (x << 16)); +} + +int kz_get_devices(void) +{ + DIR *uacce_dev = NULL; + struct dirent *device; + int found = 0; + static const char* zipdev = "hisi_zip"; + static int get_devices_flag = 0; + + if (get_devices_flag) { + return 1; + } + + kaezip_debug_init_log(); + uacce_dev = opendir("/sys/class/uacce"); + if (!uacce_dev) { + US_WARN("No /sys/class/uacce directory or it cannot be opened"); + return 0; + } + + while ((device = readdir(uacce_dev))) { + if(strstr(device->d_name, zipdev)) { + US_DEBUG("find device %s", device->d_name); + found = 1; + break; + } + } + closedir(uacce_dev); + + if (!found) { + US_WARN("No running hisi_zip devices found"); + return 0; + } + get_devices_flag = 1; + return 1; +} + +int kaezip_winbits2algtype(int windowbits) +{ + static const int ZLIB_MAX_WBITS = 15; + static const int ZLIB_MIN_WBITS = 8; + static const int GZIP_MAX_WBITS = 31; + static const int GZIP_MIN_WBITS = 24; + static const int DEFLATE_MAX_WBITS = -8; + static const int DEFLATE_MIN_WBITS = -15; + + int alg_type = WCRYPTO_NONE; + if ((windowbits >= ZLIB_MIN_WBITS) && (windowbits <= ZLIB_MAX_WBITS)) { + alg_type = WCRYPTO_ZLIB; + } else if ((windowbits >= GZIP_MIN_WBITS) && (windowbits <= GZIP_MAX_WBITS)) { + alg_type = WCRYPTO_GZIP; + } else if ((windowbits >= DEFLATE_MIN_WBITS) && (windowbits <= DEFLATE_MAX_WBITS)) { + alg_type = WCRYPTO_RAW_DEFLATE; + } else { + alg_type = WCRYPTO_NONE; + } + + return alg_type; +} + +const uint32_t kaezip_fmt_header_sz(int comp_alg_type) +{ + switch (comp_alg_type) { + case WCRYPTO_ZLIB: + return 2U; + case WCRYPTO_GZIP: + return 10U; + default: + US_WARN("not support alg comp type!"); + return 0U; + } +} + +const char* kaezip_get_fmt_header(int comp_alg_type) +{ + static const char zlib_head[] = {0x78, 0x9c}; + static const char gzip_head[] = {0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03}; + + switch (comp_alg_type) { + case WCRYPTO_ZLIB: + return zlib_head; + case WCRYPTO_GZIP: + return gzip_head; + default: + US_WARN("not support alg comp type!"); + return NULL; + } +} + +static void kaezip_append_fmt_tail(kaezip_ctx_t *kz_ctx) +{ +#define KAEZIP_APPEND_BLOCK(kz_ctx, offset, block, block_sz) \ + do { \ + memcpy(kz_ctx->end_block.buffer + offset, block, block_sz); \ + kz_ctx->end_block.data_len += block_sz; \ + } while (0) + + US_DEBUG("kaezip append fmt tail!"); + + kz_ctx->end_block.data_len = 0; + + int alg_type = kz_ctx->comp_alg_type; + uint32_t checksum = kz_ctx->op_data.checksum; + uint32_t isize = kz_ctx->op_data.isize; + + const char wd_deflate_end_block[] = {0x1, 0x0, 0x0, 0xff, 0xff}; + KAEZIP_APPEND_BLOCK(kz_ctx, 0, wd_deflate_end_block, sizeof(wd_deflate_end_block)); + + if (alg_type == WCRYPTO_ZLIB) { + checksum = (uint32_t)__cpu_to_be32(checksum); + KAEZIP_APPEND_BLOCK(kz_ctx, sizeof(wd_deflate_end_block), &checksum, sizeof(checksum)); + } + + if (alg_type == WCRYPTO_GZIP) { + checksum = ~checksum; + checksum = __kaezip_checksum_reverse(checksum); + KAEZIP_APPEND_BLOCK(kz_ctx, sizeof(wd_deflate_end_block), &checksum, sizeof(checksum)); + KAEZIP_APPEND_BLOCK(kz_ctx, sizeof(wd_deflate_end_block) + sizeof(checksum), &isize, sizeof(isize)); + } + + kz_ctx->end_block.b_set = 1; + kz_ctx->end_block.remain = kz_ctx->end_block.data_len; +} + +void kaezip_set_fmt_tail(kaezip_ctx_t *kaezip_ctx) +{ + if (kaezip_ctx->status == KAEZIP_COMP_END) { + return; + } + + if (kaezip_ctx->end_block.b_set == 0) { + kaezip_append_fmt_tail(kaezip_ctx); + } + + int data_begin = kaezip_ctx->end_block.data_len - kaezip_ctx->end_block.remain; + if (kaezip_ctx->end_block.remain <= kaezip_ctx->avail_out) { + kaezip_ctx->produced = kaezip_ctx->end_block.remain; + kaezip_ctx->end_block.remain = 0; + } else { + kaezip_ctx->produced = kaezip_ctx->avail_out; + kaezip_ctx->end_block.remain -= kaezip_ctx->produced; + } + + memcpy(kaezip_ctx->out, kaezip_ctx->end_block.buffer + data_begin, kaezip_ctx->produced); + kaezip_ctx->status = (kaezip_ctx->end_block.remain == 0 ? KAEZIP_COMP_END : KAEZIP_COMP_END_BUT_DATAREMAIN); + kaezip_ctx->end_block.b_set = (kaezip_ctx->end_block.remain == 0 ? 0 : 1); +} + +void kaezip_deflate_addcrc(kaezip_ctx_t *kz_ctx) +{ + if (kz_ctx->status != KAEZIP_COMP_CRC_UNCHECK) { + US_DEBUG("kaezip status wrong, not crc uncheck"); + return; + } + + kaezip_append_fmt_tail(kz_ctx); + + int data_begin = kz_ctx->end_block.data_len - kz_ctx->end_block.remain; + int end_produced = 0; + if (kz_ctx->end_block.remain <= kz_ctx->avail_out - kz_ctx->produced) { + end_produced = kz_ctx->end_block.remain; + kz_ctx->end_block.remain = 0; + } else { + end_produced = kz_ctx->avail_out - kz_ctx->produced; + kz_ctx->end_block.remain -= end_produced; + } + + memcpy(kz_ctx->out + kz_ctx->produced, kz_ctx->end_block.buffer + data_begin, end_produced); + kz_ctx->produced += end_produced; + kz_ctx->status = (kz_ctx->end_block.remain == 0 ? KAEZIP_COMP_END : KAEZIP_COMP_END_BUT_DATAREMAIN); +} + diff --git a/kae_zip_engine/src/kaezip_common.h b/kae_zip_engine/src/kaezip_common.h new file mode 100644 index 0000000000000000000000000000000000000000..414bac62038d7e8e814e5eb1e6e501b3970af0c9 --- /dev/null +++ b/kae_zip_engine/src/kaezip_common.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_common.h + * + * This file provides the common interface for ZIP engine dealing with wrapdrive + * + *****************************************************************************/ + +#ifndef KAEZIP_COMMON_H +#define KAEZIP_COMMON_H +#include "kaezip_ctx.h" + +#define KAEZIP_STREAM_CHUNK_IN ((COMP_BLOCK_SIZE) >> 3) // change the input size would change the performace +#define KAEZIP_STREAM_CHUNK_OUT (COMP_BLOCK_SIZE) + +#define WCRYPTO_NONE (-1) +#define WCRYPTO_RAW_DEFLATE 2 + +int kz_get_devices(void); +int kaezip_winbits2algtype(int windowBits); + +const uint32_t kaezip_fmt_header_sz(int comp_alg_type); +const char* kaezip_get_fmt_header(int comp_alg_type); +void kaezip_set_fmt_tail(kaezip_ctx_t *kz_ctx); +void kaezip_deflate_addcrc(kaezip_ctx_t *kz_ctx); +#endif + + diff --git a/kae_zip_engine/src/kaezip_conf.c b/kae_zip_engine/src/kaezip_conf.c new file mode 100644 index 0000000000000000000000000000000000000000..e92b3f7156b320e4a02d6dcadbafcdcf91cc2338 --- /dev/null +++ b/kae_zip_engine/src/kaezip_conf.c @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_cnf.h + * + * This file provides configure funtion; + * + *****************************************************************************/ + +#include "kaezip_conf.h" + +int kaezip_drv_findsection(FILE *stream, const char *v_pszSection) +{ + char line[256]; // array length:256 + char *pos = NULL; + size_t section_len = strlen(v_pszSection); + + while (!feof(stream)) { + if (fgets(line, sizeof(line), stream) == NULL) { + return -1; + } + + pos = line; + if (*(pos++) != '[') { + continue; + } + + if (memcmp(pos, v_pszSection, section_len) == 0) { + pos += section_len; + if (*pos == ']') { + return 0; + } + } + } + + return -1; +} + +void kaezip_drv_get_value(char *pos, char *v_pszValue) +{ + while (*pos != '\0') { + if (*pos == ' ') { + pos++; + continue; + } + + if (*pos == ';') { + *(v_pszValue++) = '\0'; + return; + } + + *(v_pszValue++) = *(pos++); + } +} + +int kaezip_drv_find_item(FILE *stream, const char *v_pszItem, char *v_pszValue) +{ + char line[256]; // array length:256 + char *pos = NULL; + + while (!feof(stream)) { + if (fgets(line, sizeof(line), stream) == NULL) { + return -1; + } + + if (strstr(line, v_pszItem) != NULL) { + pos = strstr(line, "="); + if (pos != NULL) { + pos++; + kaezip_drv_get_value(pos, v_pszValue); + return 0; + } + } + + if ('[' == line[0]) { + break; + } + } + + return -1; +} + +int kaezip_drv_get_item(const char *config_file, const char *v_pszSection, + const char *v_pszItem, char *v_pszValue) +{ + FILE *stream; + int retvalue = -1; + + stream = fopen(config_file, "r"); + if (stream == NULL) { + return -1; + } + + if (kaezip_drv_findsection(stream, v_pszSection) == 0) { + retvalue = kaezip_drv_find_item(stream, v_pszItem, v_pszValue); + } + + fclose(stream); + + return retvalue; +} diff --git a/kae_zip_engine/src/kaezip_conf.h b/kae_zip_engine/src/kaezip_conf.h new file mode 100644 index 0000000000000000000000000000000000000000..eafa4c579c419c75dd8e85c78797086b9e3090b2 --- /dev/null +++ b/kae_zip_engine/src/kaezip_conf.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_cnf.h + * + * This file provides configure funtion; + * + *****************************************************************************/ + +#ifndef KAEZIP_CONFIG_H +#define KAEZIP_CONFIG_H + +#include +#include +#include + +int kaezip_drv_get_item(const char *config_file, const char *v_pszSection, + const char *v_pszItem, char *v_pszValue); + +#endif // HISI_ACC_OPENSSL_CONFIG_H diff --git a/kae_zip_engine/src/kaezip_ctx.c b/kae_zip_engine/src/kaezip_ctx.c new file mode 100644 index 0000000000000000000000000000000000000000..789052bfd47620373dcd4491ab79a01bd42bec35 --- /dev/null +++ b/kae_zip_engine/src/kaezip_ctx.c @@ -0,0 +1,443 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_ctx.h + * + * This file provides kaezip ctx control and driver compress funtion; + * + *****************************************************************************/ + +#include "kaezip_ctx.h" +#include "kaezip_common.h" +#include "kaezip_utils.h" +#include "kaezip_log.h" + +static KAE_QUEUE_POOL_HEAD_S* g_kaezip_deflate_qp = NULL; +static KAE_QUEUE_POOL_HEAD_S* g_kaezip_inflate_qp = NULL; +static pthread_mutex_t g_kaezip_deflate_pool_init_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t g_kaezip_inflate_pool_init_mutex = PTHREAD_MUTEX_INITIALIZER; + +static KAE_QUEUE_POOL_HEAD_S* kaezip_get_qp(int algtype); +static kaezip_ctx_t* kaezip_new_ctx(KAE_QUEUE_DATA_NODE_S* q_node, int alg_comp_type, int comp_optype); +static int kaezip_create_wd_ctx(kaezip_ctx_t *kz_ctx, int alg_comp_type, int comp_optype); +static int kaezip_driver_do_comp_impl(kaezip_ctx_t *kz_ctx); +static void kaezip_set_input_data(kaezip_ctx_t *kz_ctx); +static void kaezip_get_output_data(kaezip_ctx_t *kz_ctx); + +void kaezip_free_ctx(void* kz_ctx) +{ + kaezip_ctx_t* kaezip_ctx = (kaezip_ctx_t *)kz_ctx; + if (kaezip_ctx == NULL) { + return; + } + + if (kaezip_ctx->op_data.in && kaezip_ctx->setup.br.usr) { + kaezip_ctx->setup.br.free(kaezip_ctx->setup.br.usr, (void *)kaezip_ctx->op_data.in); + kaezip_ctx->op_data.in = NULL; + } + + if (kaezip_ctx->op_data.out && kaezip_ctx->setup.br.usr) { + kaezip_ctx->setup.br.free(kaezip_ctx->setup.br.usr, (void *)kaezip_ctx->op_data.out); + kaezip_ctx->op_data.out = NULL; + } + + if (kaezip_ctx->wd_ctx != NULL) { + wcrypto_del_comp_ctx(kaezip_ctx->wd_ctx); + kaezip_ctx->wd_ctx = NULL; + } + + kae_free(kaezip_ctx); + + return; +} + +static kaezip_ctx_t* kaezip_new_ctx(KAE_QUEUE_DATA_NODE_S* q_node, int alg_comp_type, int comp_optype) +{ + kaezip_ctx_t *kz_ctx = NULL; + kz_ctx = (kaezip_ctx_t *)kae_malloc(sizeof(kaezip_ctx_t)); + if (unlikely(kz_ctx == NULL)) { + US_ERR("kaezip ctx malloc fail."); + return NULL; + } + memset(kz_ctx, 0, sizeof(kaezip_ctx_t)); + + kz_ctx->setup.br.alloc = kaezip_wd_alloc_blk; + kz_ctx->setup.br.free = kaezip_wd_free_blk; + kz_ctx->setup.br.iova_map = kaezip_dma_map; + kz_ctx->setup.br.iova_unmap = kaezip_dma_unmap; + kz_ctx->setup.br.usr = q_node->kae_queue_mem_pool; + + kz_ctx->op_data.in = kz_ctx->setup.br.alloc(kz_ctx->setup.br.usr, COMP_BLOCK_SIZE); + if (kz_ctx->op_data.in == NULL) { + US_ERR("alloc opdata in buf failed"); + goto err; + } + + kz_ctx->op_data.out = kz_ctx->setup.br.alloc(kz_ctx->setup.br.usr, COMP_BLOCK_SIZE); + if (kz_ctx->op_data.out == NULL) { + US_ERR("alloc opdata out buf failed"); + goto err; + } + + kz_ctx->q_node = q_node; + q_node->priv_ctx = kz_ctx; + + if (kaezip_create_wd_ctx(kz_ctx, alg_comp_type, comp_optype) == KAEZIP_FAILED) { + US_ERR("create wd ctx fail!"); + goto err; + } + + return kz_ctx; + +err: + kaezip_free_ctx(kz_ctx); + + return NULL; +} + +static int kaezip_create_wd_ctx(kaezip_ctx_t *kz_ctx, int alg_comp_type, int comp_optype) +{ + if (kz_ctx->wd_ctx != NULL) { + US_WARN("wd ctx is in used by other comp"); + return KAEZIP_FAILED; + } + + struct wd_queue *q = kz_ctx->q_node->kae_wd_queue; + + kz_ctx->setup.alg_type = (enum wcrypto_comp_alg_type)alg_comp_type; + kz_ctx->setup.op_type = (enum wcrypto_comp_optype)comp_optype; + kz_ctx->setup.stream_mode = (enum wcrypto_comp_state)WCRYPTO_COMP_STATEFUL; + + kz_ctx->wd_ctx = wcrypto_create_comp_ctx(q, &kz_ctx->setup); + if (kz_ctx->wd_ctx == NULL) { + US_ERR("wd create kae comp ctx fail!"); + return KAEZIP_FAILED; + } + + kz_ctx->comp_alg_type = alg_comp_type; + kz_ctx->comp_type = comp_optype; + + return KAEZIP_SUCCESS; +} + +kaezip_ctx_t* kaezip_get_ctx(int alg_comp_type, int comp_optype) +{ + KAE_QUEUE_DATA_NODE_S *q_node = NULL; + kaezip_ctx_t *kz_ctx = NULL; + + KAE_QUEUE_POOL_HEAD_S* qp = kaezip_get_qp(comp_optype); + if(unlikely(!qp)) { + US_ERR("failed to get hardware queue pool"); + return NULL; + } + + kaezip_queue_pool_check_and_release(qp, kaezip_free_ctx); + + q_node = kaezip_get_node_from_pool(qp, alg_comp_type, comp_optype); + if (q_node == NULL) { + US_ERR("failed to get hardware queue"); + return NULL; + } + + kz_ctx = (kaezip_ctx_t *)q_node->priv_ctx; + if (kz_ctx == NULL) { + kz_ctx = kaezip_new_ctx(q_node, alg_comp_type, comp_optype); + if (kz_ctx == NULL) { + US_ERR("kaezip new engine ctx fail!"); + (void)kaezip_put_node_to_pool(qp, q_node); + return NULL; + } + } + + kz_ctx->q_node = q_node; + kaezip_init_ctx(kz_ctx); + + return kz_ctx; +} + +void kaezip_init_ctx(kaezip_ctx_t* kz_ctx) +{ + if(unlikely(!kz_ctx)) { + US_ERR("kae zip ctx NULL!"); + return; + } + + kz_ctx->in = NULL; + kz_ctx->in_len = 0; + kz_ctx->out = NULL; + kz_ctx->avail_out = 0; + kz_ctx->consumed = 0; + kz_ctx->produced = 0; + kz_ctx->remain = 0; + + kz_ctx->header_pos = 0; + kz_ctx->flush = 0; + kz_ctx->status = 0; + + memset(&kz_ctx->end_block, 0, sizeof(struct wcrypto_end_block)); +} + +void kaezip_put_ctx(kaezip_ctx_t* kz_ctx) +{ + KAE_QUEUE_DATA_NODE_S* temp = NULL; + if (unlikely(kz_ctx == NULL)) { + US_ERR("kae zip ctx NULL!"); + return; + } + + if (kz_ctx->q_node != NULL) { + temp = kz_ctx->q_node; + kz_ctx->q_node = NULL; + (void)kaezip_put_node_to_pool(kaezip_get_qp(kz_ctx->comp_type), temp); + } + + kz_ctx = NULL; + + return; +} + +static int kaezip_driver_do_comp_impl(kaezip_ctx_t* kz_ctx) +{ + KAEZIP_RETURN_FAIL_IF(kz_ctx == NULL, "kaezip ctx is NULL.", KAEZIP_FAILED); + + struct wcrypto_comp_op_data *op_data = &kz_ctx->op_data; + + int ret = wcrypto_do_comp(kz_ctx->wd_ctx, op_data, NULL); + if (unlikely(ret < 0)) { + US_ERR("wd_do_comp fail!"); + return KAEZIP_FAILED; + } + + if (op_data->stream_pos == WCRYPTO_COMP_STREAM_NEW) { + op_data->stream_pos = WCRYPTO_COMP_STREAM_OLD; + } + + return KAEZIP_SUCCESS; +} + +int kaezip_driver_do_comp(kaezip_ctx_t *kaezip_ctx) +{ + KAEZIP_RETURN_FAIL_IF(kaezip_ctx == NULL, "kaezip ctx is NULL.", KAEZIP_FAILED); + + if (kaezip_ctx->remain != 0) { + return kaezip_get_remain_data(kaezip_ctx); + } + + if (kaezip_ctx->in_len == 0) { + US_DEBUG("kaezip do comp impl success, for input len zero, comp type : %s", + kaezip_ctx->comp_type == WCRYPTO_DEFLATE ? "deflate" : "inflate"); + return KAEZIP_SUCCESS; + } + + if (kaezip_ctx->in_len >= KAEZIP_STREAM_CHUNK_IN) { + kaezip_ctx->do_comp_len = KAEZIP_STREAM_CHUNK_IN; + } else { + kaezip_ctx->do_comp_len = kaezip_ctx->in_len; + } + + kaezip_set_input_data(kaezip_ctx); + int ret = kaezip_driver_do_comp_impl(kaezip_ctx); + if (ret != KAEZIP_SUCCESS) { + US_DEBUG("kaezip do comp impl success, comp type : %s", + kaezip_ctx->comp_type == WCRYPTO_DEFLATE ? "deflate" : "inflate"); + return ret; + } + kaezip_get_output_data(kaezip_ctx); + + return KAEZIP_SUCCESS; +} + +static void kaezip_set_input_data(kaezip_ctx_t *kz_ctx) +{ + kz_ctx->op_data.in_len = 0; + + memcpy((uint8_t *)kz_ctx->op_data.in, kz_ctx->in, kz_ctx->do_comp_len); + kz_ctx->op_data.in_len += kz_ctx->do_comp_len; + kz_ctx->op_data.avail_out = KAEZIP_STREAM_CHUNK_OUT; + kz_ctx->op_data.flush = kz_ctx->flush; + kz_ctx->op_data.alg_type = kz_ctx->comp_alg_type; + + if (kz_ctx->status == KAEZIP_COMP_INIT || kz_ctx->status == KAEZIP_DECOMP_INIT) { + kz_ctx->op_data.stream_pos = WCRYPTO_COMP_STREAM_NEW; + } +} + +static void kaezip_set_comp_status(kaezip_ctx_t *kz_ctx) +{ + if (kz_ctx->comp_type == WCRYPTO_INFLATE) { + switch (kz_ctx->op_data.status) { + case WCRYPTO_DECOMP_END: + kz_ctx->status = (kz_ctx->remain == 0 ? KAEZIP_DECOMP_END : KAEZIP_DECOMP_END_BUT_DATAREMAIN); + break; + case WCRYPTO_STATUS_NULL: + kz_ctx->status = KAEZIP_DECOMP_DOING; + break; + case WD_VERIFY_ERR: + kz_ctx->status = KAEZIP_DECOMP_VERIFY_ERR; + break; + default: + kz_ctx->status = KAEZIP_DECOMP_DOING; + break; + } + } else { + switch (kz_ctx->op_data.status) { + case WCRYPTO_STATUS_NULL: + if (kz_ctx->in_len > kz_ctx->consumed) { + kz_ctx->status = KAEZIP_COMP_DOING; + break; + } + + if (kz_ctx->flush != WCRYPTO_FINISH) { + kz_ctx->status = KAEZIP_COMP_CRC_UNCHECK; + break; + } + + if (kz_ctx->remain != 0) { + kz_ctx->status = KAEZIP_COMP_END_BUT_DATAREMAIN; + } else { + kz_ctx->status = KAEZIP_COMP_END; + } + break; + case WD_VERIFY_ERR: + kz_ctx->status = KAEZIP_COMP_VERIFY_ERR; + break; + default: + kz_ctx->status = KAEZIP_COMP_DOING; + break; + } + } +} + +static void kaezip_get_output_data(kaezip_ctx_t *kz_ctx) +{ + kz_ctx->consumed = kz_ctx->op_data.consumed; + + if (kz_ctx->avail_out < kz_ctx->op_data.produced) { + kz_ctx->produced = kz_ctx->avail_out; + kz_ctx->remain = kz_ctx->op_data.produced - kz_ctx->produced; + } else { + kz_ctx->produced = kz_ctx->op_data.produced; + } + + memcpy(kz_ctx->out, (uint8_t*)kz_ctx->op_data.out, kz_ctx->produced); + + kaezip_set_comp_status(kz_ctx); +} + +static void kaezip_state_machine_trans(kaezip_ctx_t *kz_ctx) +{ + if (kz_ctx->comp_type == WCRYPTO_INFLATE) { + switch (kz_ctx->status) { + case KAEZIP_DECOMP_INIT: // fall-through, trans to next state + kz_ctx->status = KAEZIP_DECOMP_DOING; + case KAEZIP_DECOMP_DOING: + break; + case KAEZIP_DECOMP_END_BUT_DATAREMAIN: // fall-through, trans to next state + kz_ctx->status = (kz_ctx->remain == 0 ? KAEZIP_DECOMP_END : KAEZIP_DECOMP_END_BUT_DATAREMAIN); + case KAEZIP_DECOMP_END: + break; + case KAEZIP_DECOMP_VERIFY_ERR: + US_ERR("kaezip inflate verify err"); + break; + default: + kz_ctx->status = KAEZIP_DECOMP_DOING; + break; + } + } else { + switch (kz_ctx->status) { + case KAEZIP_COMP_INIT: // fall-through, trans to next state + kz_ctx->status = KAEZIP_COMP_DOING; + case KAEZIP_COMP_DOING: // fall-through, trans to next state + kz_ctx->status = KAEZIP_COMP_CRC_UNCHECK; + case KAEZIP_COMP_CRC_UNCHECK: + if (kz_ctx->remain == 0 && kz_ctx->flush == WCRYPTO_FINISH && kz_ctx->in_len == 0) { + kaezip_deflate_addcrc(kz_ctx); + kz_ctx->status = (kz_ctx->end_block.remain == 0 ? KAEZIP_COMP_END : KAEZIP_COMP_END_BUT_DATAREMAIN); + } + break; + case KAEZIP_COMP_END_BUT_DATAREMAIN: // fall-through, trans to next state + kz_ctx->status = (kz_ctx->remain == 0 ? KAEZIP_COMP_END : KAEZIP_COMP_END_BUT_DATAREMAIN); + case KAEZIP_COMP_END: + break; + case KAEZIP_COMP_VERIFY_ERR: + US_ERR("kaezip deflate verify err"); + break; + default: + kz_ctx->status = KAEZIP_COMP_DOING; + break; + } + } +} + +int kaezip_get_remain_data(kaezip_ctx_t *kz_ctx) +{ + KAEZIP_RETURN_FAIL_IF(kz_ctx->op_data.produced < kz_ctx->remain, "wrong remain data", KAEZIP_FAILED); + int data_begin = kz_ctx->op_data.produced - kz_ctx->remain; + + if (kz_ctx->remain < kz_ctx->avail_out) { + kz_ctx->produced = kz_ctx->remain; + memcpy(kz_ctx->out, (uint8_t*)kz_ctx->op_data.out + data_begin, kz_ctx->produced); + kz_ctx->remain = 0; + } else { + kz_ctx->produced = kz_ctx->avail_out; + memcpy(kz_ctx->out, (uint8_t*)kz_ctx->op_data.out + data_begin, kz_ctx->produced); + kz_ctx->remain -= kz_ctx->produced; + } + + kaezip_state_machine_trans(kz_ctx); + + return KAEZIP_SUCCESS; +} + +static KAE_QUEUE_POOL_HEAD_S* kaezip_get_qp(int algtype) +{ + if ((algtype != WCRYPTO_DEFLATE) && (algtype != WCRYPTO_INFLATE) ) { + US_ERR("kaezip get q pool failed, not a support algtye %d!", algtype); + return NULL; + } + + if (algtype == WCRYPTO_DEFLATE) { + if (g_kaezip_deflate_qp) { + return g_kaezip_deflate_qp; + } + pthread_mutex_lock(&g_kaezip_deflate_pool_init_mutex); + if (g_kaezip_deflate_qp != NULL) { + pthread_mutex_unlock(&g_kaezip_deflate_pool_init_mutex); + return g_kaezip_deflate_qp; + } + kaezip_queue_pool_destroy(g_kaezip_deflate_qp, kaezip_free_ctx); + g_kaezip_deflate_qp = kaezip_init_queue_pool(algtype); + pthread_mutex_unlock(&g_kaezip_deflate_pool_init_mutex); + + return g_kaezip_deflate_qp == NULL ? NULL : g_kaezip_deflate_qp; + } else { + if (g_kaezip_inflate_qp) { + return g_kaezip_inflate_qp; + } + pthread_mutex_lock(&g_kaezip_inflate_pool_init_mutex); + if (g_kaezip_inflate_qp != NULL) { + pthread_mutex_unlock(&g_kaezip_inflate_pool_init_mutex); + return g_kaezip_inflate_qp; + } + kaezip_queue_pool_destroy(g_kaezip_inflate_qp, kaezip_free_ctx); + g_kaezip_inflate_qp = kaezip_init_queue_pool(algtype); + pthread_mutex_unlock(&g_kaezip_inflate_pool_init_mutex); + + return g_kaezip_inflate_qp == NULL ? NULL : g_kaezip_inflate_qp; + } + + return NULL; +} diff --git a/kae_zip_engine/src/kaezip_ctx.h b/kae_zip_engine/src/kaezip_ctx.h new file mode 100644 index 0000000000000000000000000000000000000000..9708fbc0e2d20638f2ff4d19fcc06f5d58cd8574 --- /dev/null +++ b/kae_zip_engine/src/kaezip_ctx.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_ctx.h + * + * This file provides kaezip ctx control and driver compress funtion; + * + *****************************************************************************/ + +#ifndef KAEZIP_CTX_H +#define KAEZIP_CTX_H +#include "wd_queue_memory.h" +#include "uadk/v1/wd_comp.h" + +enum kaezip_comp_status { + KAEZIP_COMP_INIT = 0, + KAEZIP_COMP_DOING, + KAEZIP_COMP_CRC_UNCHECK, + KAEZIP_COMP_END_BUT_DATAREMAIN, + KAEZIP_COMP_END, + KAEZIP_COMP_VERIFY_ERR, +}; + +enum kaezip_decomp_status { + KAEZIP_DECOMP_INIT = 0, + KAEZIP_DECOMP_DOING, + KAEZIP_DECOMP_END_BUT_DATAREMAIN, + KAEZIP_DECOMP_END, + KAEZIP_DECOMP_VERIFY_ERR, +}; + +struct wcrypto_end_block { + char buffer[32]; + unsigned int data_len; + unsigned int remain; + unsigned int b_set; +}; + +struct kaezip_ctx { + void *in; + unsigned int in_len; + void *out; + unsigned int avail_out; + unsigned int consumed; + unsigned int produced; + unsigned int remain; //data produced by warpdrive but haven't been take away for not enough avail out buf + + unsigned int header_pos; // the format header pos + int flush; // WCRYPTO_SYNC_FLUSH / WCRYPTO_FINISH + int comp_alg_type; // WCRYPTO_ZLIB / WCRYPTO_GZIP + int comp_type; // WCRYPTO_DEFLATE / WCRYPTO_INFLATE + unsigned int do_comp_len; // a compress proccess cost len + int status; // enum kaezip_comp_status + + struct wcrypto_end_block end_block; + KAE_QUEUE_DATA_NODE_S* q_node; + struct wcrypto_comp_ctx_setup setup; + struct wcrypto_comp_op_data op_data; + void* wd_ctx; +}; +typedef struct kaezip_ctx kaezip_ctx_t; + +kaezip_ctx_t* kaezip_get_ctx(int alg_comp_type, int comp_optype); +void kaezip_put_ctx(kaezip_ctx_t* kz_ctx); +void kaezip_init_ctx(kaezip_ctx_t* kz_ctx); +void kaezip_free_ctx(void* kz_ctx); + +int kaezip_get_remain_data(kaezip_ctx_t *kz_ctx); +int kaezip_driver_do_comp(kaezip_ctx_t *kaezip_ctx); + +#endif + diff --git a/kae_zip_engine/src/kaezip_deflate.c b/kae_zip_engine/src/kaezip_deflate.c new file mode 100644 index 0000000000000000000000000000000000000000..aa073c63d62dab3fb36addd5dc065de7104b88b3 --- /dev/null +++ b/kae_zip_engine/src/kaezip_deflate.c @@ -0,0 +1,229 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_deflate.h + * + * This file provides inflate function + * + *****************************************************************************/ + +#include +#include +#include +#include "zlib.h" +#include "kaezip_deflate.h" +#include "kaezip_ctx.h" +#include "kaezip_common.h" +#include "kaezip_utils.h" +#include "kaezip_log.h" + +#define KAEZIP_UPDATE_ZSTREAM_IN(zstrm, in_len) \ + do { \ + zstrm->next_in += in_len; \ + zstrm->avail_in -= in_len; \ + zstrm->total_in += in_len; \ + } while (0) + +#define KAEZIP_UPDATE_ZSTREAM_OUT(zstrm, out_len) \ + do { \ + zstrm->next_out += out_len; \ + zstrm->avail_out -= out_len; \ + zstrm->total_out += out_len; \ + } while (0) + +static int kaezip_do_deflate(z_streamp strm, int flush); +static void kaezip_deflate_set_fmt_header(z_streamp strm, int comp_alg_type); + +int kz_deflateInit2_(z_streamp strm, int level, + int method, int windowBits, + int memLevel, int strategy, + const char *version, + int stream_size) +{ + int ret = lz_deflateInit2_(strm, level, method, windowBits, memLevel, strategy, version, stream_size); + if (ret != Z_OK) { + US_ERR("zlib deflate init failed windowbits %d!", windowBits); + return Z_ERRNO; + } + + int alg_comp_type = kaezip_winbits2algtype(windowBits); + if (alg_comp_type != WCRYPTO_ZLIB && alg_comp_type != WCRYPTO_GZIP) { + US_WARN("unsupport windowbits %d!", windowBits); + setDeflateKaezipCtx(strm, 0); + return Z_OK; + } + + kaezip_ctx_t* kaezip_ctx = kaezip_get_ctx(alg_comp_type, WCRYPTO_DEFLATE); + if (kaezip_ctx == NULL) { + US_ERR("failed to get kaezip ctx, windowbits %d!", windowBits); + setDeflateKaezipCtx(strm, 0); + return Z_OK; + } + + kaezip_ctx->status = KAEZIP_COMP_INIT; + setDeflateKaezipCtx(strm, (uLong)kaezip_ctx); + + US_DEBUG("kae zip deflate init success, kaezip_ctx %p, kaezip_ctx->comp_alg_type %s!", + kaezip_ctx, kaezip_ctx->comp_alg_type == WCRYPTO_ZLIB ? "zlib" : "gzip"); + return Z_OK; +} + +static int kz_deflate_check_strm_avail(z_streamp strm, int flush) +{ + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getDeflateKaezipCtx(strm); + KAEZIP_RETURN_FAIL_IF(kaezip_ctx == NULL, "kaezip ctx is NULL.", 0); + + //z stream finished and outbuf == 0, but there is still some data that needs to be taken + if (strm->avail_out == 0 && flush == Z_FINISH && kaezip_ctx->remain != 0) { + US_WARN("kz deflate warning, no enough output buff, kaezip_ctx->remain %d", kaezip_ctx->remain); + return 0; + } + + //z stream not finished but inbuf == 0 and no data remained, so we consider that it has reached the end of data + if (strm->avail_in == 0 && flush != Z_FINISH && kaezip_ctx->remain == 0) { + US_WARN("kz deflate warning, no more input buff, kaezip_ctx->remain %d", kaezip_ctx->remain); + return 0; + } + + return 1; +} + +int kz_deflate(z_streamp strm, int flush) +{ + int ret = -1; + KAEZIP_RETURN_FAIL_IF(strm == NULL, "strm is NULL.", Z_ERRNO); + + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getDeflateKaezipCtx(strm); + KAEZIP_RETURN_FAIL_IF(kaezip_ctx == NULL, "kaezip ctx is NULL.", Z_ERRNO); + + if (!kz_deflate_check_strm_avail(strm, flush)) { + return Z_BUF_ERROR; + } + + //wcrypto deflate need to add output format header + const uint32_t fmt_header_sz = kaezip_fmt_header_sz(kaezip_ctx->comp_alg_type); + if (kaezip_ctx->header_pos != fmt_header_sz) { + kaezip_deflate_set_fmt_header(strm, kaezip_ctx->comp_alg_type); + if (kaezip_ctx->header_pos != fmt_header_sz) { + return Z_OK; + } + } + + do { + ret = kaezip_do_deflate(strm, flush); + if (ret != KAEZIP_SUCCESS) { + US_ERR("kaezip failed to do deflate, flush %d", flush); + return Z_ERRNO; + } + + KAEZIP_UPDATE_ZSTREAM_IN(strm, kaezip_ctx->consumed); + KAEZIP_UPDATE_ZSTREAM_OUT(strm, kaezip_ctx->produced); + if (kaezip_ctx->status == KAEZIP_COMP_END) { + break; + } + } while (strm->avail_out != 0 && strm->avail_in != 0) ; + + if (kaezip_ctx->status == KAEZIP_COMP_END + && flush == Z_FINISH + && strm->avail_in == 0 + && kaezip_ctx->remain == 0) { + return Z_STREAM_END; + } else { + return Z_OK; + } +} + +int kz_deflateEnd(z_streamp strm) +{ + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getDeflateKaezipCtx(strm); + if (kaezip_ctx != NULL) { + US_DEBUG("kaezip deflate end"); + kaezip_put_ctx(kaezip_ctx); + } + + setDeflateKaezipCtx(strm, 0); + return lz_deflateEnd(strm); +} + +int ZEXPORT kz_deflateReset(z_streamp strm) +{ + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getDeflateKaezipCtx(strm); + if (kaezip_ctx != NULL) { + US_DEBUG("kaezip deflate reset"); + kaezip_init_ctx(kaezip_ctx); + } + + return lz_deflateReset(strm); +} + +static void kaezip_deflate_set_fmt_header(z_streamp strm, int comp_alg_type) +{ + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getDeflateKaezipCtx(strm); + const uint32_t fmt_header_sz = kaezip_fmt_header_sz(comp_alg_type); + const char* fmt_header = kaezip_get_fmt_header(comp_alg_type); + + //that means the outout avail buf is even not enough for a header + if (strm->avail_out < fmt_header_sz - kaezip_ctx->header_pos) { + kaezip_ctx->header_pos += strm->avail_out; + return; + } + + memcpy(strm->next_out, fmt_header, fmt_header_sz); + KAEZIP_UPDATE_ZSTREAM_OUT(strm, fmt_header_sz); + + kaezip_ctx->header_pos = fmt_header_sz; +} + +static int kaezip_do_deflate(z_streamp strm, int flush) +{ + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getDeflateKaezipCtx(strm); + KAEZIP_RETURN_FAIL_IF(kaezip_ctx == NULL, "kaezip ctx is NULL.", KAEZIP_FAILED); + KAEZIP_RETURN_FAIL_IF(kaezip_ctx->comp_alg_type != WCRYPTO_ZLIB && kaezip_ctx->comp_alg_type != WCRYPTO_GZIP, + "not support alg comp type!", KAEZIP_FAILED); + + kaezip_ctx->in = strm->next_in; + kaezip_ctx->in_len = (strm->avail_in < KAEZIP_STREAM_CHUNK_IN) ? strm->avail_in : KAEZIP_STREAM_CHUNK_IN; + kaezip_ctx->out = strm->next_out; + kaezip_ctx->consumed = 0; + kaezip_ctx->produced = 0; + kaezip_ctx->avail_out = strm->avail_out; + if (flush == Z_FINISH) { + kaezip_ctx->flush = (strm->avail_in <= KAEZIP_STREAM_CHUNK_IN) ? WCRYPTO_FINISH : WCRYPTO_SYNC_FLUSH; + } else { + kaezip_ctx->flush = WCRYPTO_SYNC_FLUSH; + } + + //if last stream(Z_FINISH) input len is zero, add a format tail for output, unlikely go here + if (kaezip_ctx->status != KAEZIP_COMP_END + && flush == Z_FINISH + && strm->avail_in == 0 + && kaezip_ctx->remain == 0) { + kaezip_set_fmt_tail(kaezip_ctx); + return KAEZIP_SUCCESS; + } + + int ret = kaezip_driver_do_comp(kaezip_ctx); + if (ret != KAEZIP_SUCCESS) { + US_ERR("kae zip do deflate impl fail!"); + return KAEZIP_FAILED; + } + + US_DEBUG("kaezip do deflate avail_in %u, avail_out %u, consumed %u, produced %u, remain %u, status %d, flush %d", + strm->avail_in, strm->avail_out, kaezip_ctx->consumed, kaezip_ctx->produced, + kaezip_ctx->remain, kaezip_ctx->status, flush); + + return KAEZIP_SUCCESS; +} diff --git a/kae_zip_engine/src/kaezip_deflate.h b/kae_zip_engine/src/kaezip_deflate.h new file mode 100644 index 0000000000000000000000000000000000000000..34314672708fa1a0a96893cddc8a857a60d2a714 --- /dev/null +++ b/kae_zip_engine/src/kaezip_deflate.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_inflate.h + * + * This file provides deflate function + * + *****************************************************************************/ + +#ifndef KAEZIP_DEFLATE_H +#define KAEZIP_DEFLATE_H + +extern int kz_deflateInit2_(z_streamp strm, int level, int metho, int windowBit, int memLevel, int strategy, + const char *version, int stream_size); +extern int kz_deflate(z_streamp strm, int flush); +extern int kz_deflateEnd(z_streamp strm); +extern int kz_deflateReset(z_streamp strm); + +extern int lz_deflateEnd(z_streamp strm); +extern int lz_deflateInit2_(z_streamp strm, int level, int metho, int windowBit, int memLevel, int strategy, + const char *version, int stream_size); +extern int lz_deflateReset(z_streamp strm); + +extern unsigned long getDeflateKaezipCtx(z_streamp strm); +extern void setDeflateKaezipCtx(z_streamp strm, unsigned long kaezip_ctx); +#endif + + diff --git a/kae_zip_engine/src/kaezip_inflate.c b/kae_zip_engine/src/kaezip_inflate.c new file mode 100644 index 0000000000000000000000000000000000000000..31f93ebe80c12d18a15ddcf1fbe751f85a4e9746 --- /dev/null +++ b/kae_zip_engine/src/kaezip_inflate.c @@ -0,0 +1,250 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_inflate.h + * + * This file provides inflate function + * + *****************************************************************************/ +#include +#include +#include +#include "zlib.h" +#include "kaezip_inflate.h" +#include "kaezip_ctx.h" +#include "kaezip_common.h" +#include "kaezip_utils.h" +#include "kaezip_log.h" + +#define KAEZIP_UPDATE_ZSTREAM_IN(zstrm, in_len) \ + do { \ + zstrm->next_in += in_len; \ + zstrm->avail_in -= in_len; \ + zstrm->total_in += in_len; \ + } while (0) + +#define KAEZIP_UPDATE_ZSTREAM_OUT(zstrm, out_len) \ + do { \ + zstrm->next_out += out_len; \ + zstrm->avail_out -= out_len; \ + zstrm->total_out += out_len; \ + } while (0) + +static int kaezip_check_strm_truely_end(z_streamp strm); +static int kaezip_do_inflate(z_streamp strm, int flush); + +int ZEXPORT kz_inflateInit2_(z_streamp strm, int windowBits, const char *version, int stream_size) +{ + static const int GZIP_INFLATE_MAX_AUTO_WBITS = 47; + static const int GZIP_INFLATE_MIN_AUTO_WBITS = 32; + + int ret = lz_inflateInit2_(strm, windowBits, version, stream_size); + if (unlikely(ret != Z_OK)) { + US_ERR("lz_inflateInit2_ error, windowBits %d!", windowBits); + return Z_ERRNO; + } + + setInflateKaezipCtx(strm, (unsigned long)0); + if (windowBits >= GZIP_INFLATE_MIN_AUTO_WBITS && windowBits <= GZIP_INFLATE_MAX_AUTO_WBITS) { + return Z_OK; + } else { + int alg_comp_type = kaezip_winbits2algtype(windowBits); + return kz_do_inflateInit(strm, alg_comp_type); + } + + return Z_OK; +} + +static int kaezip_check_strm_truely_end(z_streamp strm) +{ + KAEZIP_RETURN_FAIL_IF(strm == NULL, "strm is NULL.", Z_ERRNO); + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getInflateKaezipCtx(strm); + KAEZIP_RETURN_FAIL_IF(kaezip_ctx == NULL, "kaezip ctx is NULL.", Z_ERRNO); + + if (strm->avail_in == 1 + && kaezip_ctx->status == KAEZIP_DECOMP_END) { + strm->avail_in = 0; + } + + if (strm->avail_in == 0 + && kaezip_ctx->status == KAEZIP_DECOMP_END_BUT_DATAREMAIN) { + strm->avail_in = 1; + } + + return KAEZIP_SUCCESS; +} + +int ZEXPORT kz_inflate(z_streamp strm, int flush) +{ + int ret = -1; + KAEZIP_RETURN_FAIL_IF(strm == NULL, "strm is NULL.", Z_ERRNO); + + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getInflateKaezipCtx(strm); + KAEZIP_RETURN_FAIL_IF(kaezip_ctx == NULL, "kaezip ctx is NULL.", Z_ERRNO); + + do { + ret = kaezip_do_inflate(strm, flush); + if (ret != KAEZIP_SUCCESS) { + US_ERR("kaezip failed to do inflate, flush %d", flush); + return Z_ERRNO; + } + + KAEZIP_UPDATE_ZSTREAM_IN(strm, kaezip_ctx->consumed); + KAEZIP_UPDATE_ZSTREAM_OUT(strm, kaezip_ctx->produced); + + if (kaezip_ctx->status == KAEZIP_DECOMP_VERIFY_ERR) { + return Z_DATA_ERROR; + } + + if (kaezip_ctx->status == KAEZIP_DECOMP_END) { + (void)kaezip_check_strm_truely_end(strm); + return Z_STREAM_END; + } + } while (strm->avail_out != 0 && strm->avail_in != 0); + + // when test on rpm -i src.rpm, the strm may end with not true ended + (void)kaezip_check_strm_truely_end(strm); + + if (flush == Z_FINISH && strm->avail_in == 0 && kaezip_ctx->remain == 0) { + return Z_STREAM_END; + } else { + return Z_OK; + } +} + +int kz_inflateEnd(z_streamp strm) +{ + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getInflateKaezipCtx(strm); + if (kaezip_ctx != NULL) { + US_DEBUG("kaezip inflate end"); + kaezip_put_ctx(kaezip_ctx); + } + + setInflateKaezipCtx(strm, 0); + return lz_inflateEnd(strm); +} + +int ZEXPORT kz_inflateReset(z_streamp strm) +{ + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getInflateKaezipCtx(strm); + if (kaezip_ctx != NULL) { + US_DEBUG("kaezip inflate reset"); + kaezip_init_ctx(kaezip_ctx); + } + + return lz_inflateReset(strm); +} + +static int kaezip_do_inflate(z_streamp strm, int flush) +{ + kaezip_ctx_t *kaezip_ctx = (kaezip_ctx_t *)getInflateKaezipCtx(strm); + KAEZIP_RETURN_FAIL_IF(kaezip_ctx == NULL, "kaezip ctx is NULL.", KAEZIP_FAILED); + KAEZIP_RETURN_FAIL_IF(kaezip_ctx->comp_alg_type != WCRYPTO_ZLIB && kaezip_ctx->comp_alg_type != WCRYPTO_GZIP, + "not support alg comp type!", KAEZIP_FAILED); + + kaezip_ctx->in = strm->next_in; + kaezip_ctx->in_len = (strm->avail_in < KAEZIP_STREAM_CHUNK_IN) ? strm->avail_in : KAEZIP_STREAM_CHUNK_IN; + kaezip_ctx->out = strm->next_out; + kaezip_ctx->consumed = 0; + kaezip_ctx->produced = 0; + kaezip_ctx->avail_out = strm->avail_out; + if (flush == Z_FINISH) { + kaezip_ctx->flush = (strm->avail_in <= KAEZIP_STREAM_CHUNK_IN) ? WCRYPTO_FINISH : WCRYPTO_SYNC_FLUSH; + } else { + kaezip_ctx->flush = WCRYPTO_SYNC_FLUSH; + } + + //the firsh input z stream, should skip the format header, for hardware incompatible + if (kaezip_ctx->status == KAEZIP_DECOMP_INIT) { + const uint32_t fmt_header_sz = kaezip_fmt_header_sz(kaezip_ctx->comp_alg_type); + if (kaezip_ctx->header_pos + kaezip_ctx->in_len <= fmt_header_sz) { + kaezip_ctx->header_pos += kaezip_ctx->in_len; + kaezip_ctx->consumed = kaezip_ctx->in_len; + kaezip_ctx->produced = 0; + US_WARN("the z stream avail len is less then format header, skip!"); + return KAEZIP_SUCCESS; + } else { + kaezip_ctx->in += fmt_header_sz - kaezip_ctx->header_pos; + kaezip_ctx->in_len -= fmt_header_sz - kaezip_ctx->header_pos; + kaezip_ctx->consumed = fmt_header_sz - kaezip_ctx->header_pos; + kaezip_ctx->produced = fmt_header_sz; + KAEZIP_UPDATE_ZSTREAM_IN(strm, kaezip_ctx->consumed); //skip the strm format header here + kaezip_ctx->header_pos = 0; + } + } + + int ret = kaezip_driver_do_comp(kaezip_ctx); + if (ret != KAEZIP_SUCCESS) { + US_ERR("kae zip do inflate impl fail!"); + return KAEZIP_FAILED; + } + + US_DEBUG("kaezip do inflate avail_in %u, avail_out %u, consumed %u, produced %u, remain %u, status %d, flush %d", + strm->avail_in, strm->avail_out, kaezip_ctx->consumed, kaezip_ctx->produced, + kaezip_ctx->remain, kaezip_ctx->status, flush); + + return KAEZIP_SUCCESS; +} + +int kz_do_inflateInit(z_streamp strm, int alg_comp_type) +{ + unsigned long kaezip_ctx_value = getInflateKaezipCtx(strm); + if (kaezip_ctx_value != 0) { + return Z_OK; + } + + if (alg_comp_type != WCRYPTO_ZLIB && alg_comp_type != WCRYPTO_GZIP) { + US_WARN("unsupport alg_comp_type %d!", alg_comp_type); + setInflateKaezipCtx(strm, 0); + return Z_OK; + } + + kaezip_ctx_t* kaezip_ctx = kaezip_get_ctx(alg_comp_type, WCRYPTO_INFLATE); + if (kaezip_ctx == NULL) { + US_ERR("failed to get kaezip ctx, alg_comp_type %d!", alg_comp_type); + setInflateKaezipCtx(strm, 0); + return Z_OK; + } + + kaezip_ctx->status = KAEZIP_DECOMP_INIT; + setInflateKaezipCtx(strm, (uLong)kaezip_ctx); + + US_DEBUG("kae zip inflate init success, kaezip_ctx %p, kaezip_ctx->comp_alg_type %s!", + kaezip_ctx, kaezip_ctx->comp_alg_type == WCRYPTO_ZLIB ? "zlib" : "gzip"); + + return Z_OK; +} + +int kz_getAutoInflateAlgType(z_streamp strm) +{ + if (strm->next_in == NULL || strm->avail_in == 0) { + return WCRYPTO_NONE; + } + + int wrap = getInflateStateWrap(strm); + // wrap bit 0 true for zlib, bit 1 true for gzip, bit 2 true to validate check value + if ((wrap & 0x7) == 0x7) { + // gzip head + if (strm->next_in[0] == 0x1F && strm->next_in[1] == 0x8B) { + return WCRYPTO_GZIP; + } else { + return WCRYPTO_ZLIB; + } + } else { + return WCRYPTO_NONE; + } +} + diff --git a/kae_zip_engine/src/kaezip_inflate.h b/kae_zip_engine/src/kaezip_inflate.h new file mode 100644 index 0000000000000000000000000000000000000000..d230c92e647deee3b2fa6c9d788e0134ff3fb103 --- /dev/null +++ b/kae_zip_engine/src/kaezip_inflate.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_inflate.h + * + * This file provides inflate function + * + *****************************************************************************/ + +#ifndef KAEZIP_INFLATE_H +#define KAEZIP_INFLATE_H + +extern int kz_inflateInit2_(z_streamp strm, int windowBits, const char *version, int stream_size); +extern int kz_inflate(z_streamp strm, int flush); +extern int kz_inflateEnd(z_streamp strm); +extern int kz_inflateReset(z_streamp strm); +extern int kz_do_inflateInit(z_streamp strm, int alg_comp_type); + +extern int lz_inflateEnd(z_streamp strm); +extern int lz_inflateInit2_(z_streamp strm, int windowBits, const char *version, int stream_size); +extern int lz_inflateReset(z_streamp strm); + +extern int getInflateStateWrap(z_streamp strm); +extern unsigned long getInflateKaezipCtx(z_streamp strm); +extern void setInflateKaezipCtx(z_streamp strm, unsigned long kaezip_ctx); +#endif + + diff --git a/kae_zip_engine/src/kaezip_log.c b/kae_zip_engine/src/kaezip_log.c new file mode 100644 index 0000000000000000000000000000000000000000..41a622b1293418d4e5194e06f3587ca1ec12317d --- /dev/null +++ b/kae_zip_engine/src/kaezip_log.c @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_log.c + * + * This file provides the log funtion; + * + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include "kaezip_log.h" +#include "kaezip_conf.h" + +#define KAE_CONFIG_FILE_NAME "/kaezip.cnf" +#define MAX_LEVEL_LEN 10 +#define MAX_CONFIG_LEN 512 + +static const char *g_kaezip_conf_env = "KAEZIP_CONF_ENV"; + +FILE *g_kaezip_debug_log_file = (FILE *)NULL; +pthread_mutex_t g_kaezip_debug_file_mutex = PTHREAD_MUTEX_INITIALIZER; +int g_kaezip_debug_file_ref_count = 0; +int g_kaezip_log_init_times = 0; +int g_kaezip_log_level = 0; + +const char *g_kaezip_log_level_string[] = { + "none", + "error", + "warning", + "info", + "debug", +}; + +static char *kae_getenv(const char *name) +{ + return getenv(name); +} + +static void kae_set_conf_debuglevel() +{ + char *conf_path = kae_getenv(g_kaezip_conf_env); + unsigned int i = 0; + const char *filename = KAE_CONFIG_FILE_NAME; + char *file_path = (char *)NULL; + char *debuglev = (char *)NULL; + if (conf_path == NULL || strlen(conf_path) > MAX_CONFIG_LEN) { + goto err; + } + file_path = (char *)malloc(strlen(conf_path) + strlen(filename) + 1); + debuglev = (char *)malloc(MAX_LEVEL_LEN); + if (!file_path || !debuglev) { + goto err; + } + memset(debuglev, 0, MAX_LEVEL_LEN); + memset(file_path, 0, sizeof(conf_path) + sizeof(filename) + 1); + strcat(file_path, conf_path); + strcat(file_path, filename); + int ret = kaezip_drv_get_item(file_path, "LogSection", "debug_level", debuglev); + if (ret != 0) { + goto err; + } + + for (i = 0; i < sizeof(g_kaezip_log_level_string) / sizeof(g_kaezip_log_level_string[0]); i++) { + if (strncmp(g_kaezip_log_level_string[i], debuglev, strlen(debuglev) - 1) == 0) { + g_kaezip_log_level = i; + free(file_path); + free(debuglev); + return; + } + } + +err: + g_kaezip_log_level = KAE_NONE; + if (debuglev != NULL) { + free(debuglev); + debuglev = (char *)NULL; + } + if (file_path != NULL) { + free(file_path); + file_path = (char *)NULL; + } + + return; +} + +void kaezip_debug_init_log() +{ + pthread_mutex_lock(&g_kaezip_debug_file_mutex); + kae_set_conf_debuglevel(); + if (!g_kaezip_debug_file_ref_count && g_kaezip_log_level != KAE_NONE) { + g_kaezip_debug_log_file = fopen(KAEZIP_DEBUG_FILE_PATH, "a+"); + if (g_kaezip_debug_log_file == NULL) { + g_kaezip_debug_log_file = stderr; + fprintf(stderr, "unable to open %s, %s\n", KAEZIP_DEBUG_FILE_PATH, strerror(errno)); + } else { + g_kaezip_debug_file_ref_count++; + } + } + g_kaezip_log_init_times++; + pthread_mutex_unlock(&g_kaezip_debug_file_mutex); +} + +void kaezip_debug_close_log() +{ + pthread_mutex_lock(&g_kaezip_debug_file_mutex); + g_kaezip_log_init_times--; + if (g_kaezip_debug_file_ref_count && (g_kaezip_log_init_times == 0)) { + if (g_kaezip_debug_log_file != NULL) { + fclose(g_kaezip_debug_log_file); + g_kaezip_debug_file_ref_count--; + g_kaezip_debug_log_file = stderr; + } + } + pthread_mutex_unlock(&g_kaezip_debug_file_mutex); +} + +void kaezip_save_log(FILE *src) +{ + int size = 0; + char buf[1024] = {0}; // buf length:1024 + + if (src == NULL) { + return; + } + + FILE *dst = fopen(KAEZIP_DEBUG_FILE_PATH_OLD, "w"); + if (dst == NULL) { + return; + } + + fseek(src, 0, SEEK_SET); + while (1) { + size = fread(buf, sizeof(char), 1024, src); // buf length:1024 + fwrite(buf, sizeof(char), size, dst); + if (!size) { + break; + } + } + + fclose(dst); +} diff --git a/kae_zip_engine/src/kaezip_log.h b/kae_zip_engine/src/kaezip_log.h new file mode 100644 index 0000000000000000000000000000000000000000..e3f00984de988a9b6eb5de725872d23f059e49b1 --- /dev/null +++ b/kae_zip_engine/src/kaezip_log.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_log.h + * + * This file provides the log funtion; + * + *****************************************************************************/ + +#ifndef KAEZIP_LOG_H +#define KAEZIP_LOG_H +#include +#include +#include +#include +#include + +#define KAEZIP_DEBUG_FILE_PATH "/var/log/kaezip.log" +#define KAEZIP_DEBUG_FILE_PATH_OLD "/var/log/kaezip.log.old" +#define KAE_LOG_MAX_SIZE 209715200 + +extern FILE *g_kaezip_debug_log_file; +extern pthread_mutex_t g_kaezip_debug_file_mutex; +extern const char *g_kaezip_log_level_string[]; +extern int g_kaezip_log_level; + +enum KAE_LOG_LEVEL { + KAE_NONE = 0, + KAE_ERROR, + KAE_WARNING, + KAE_INFO, + KAE_DEBUG, +}; + +void ENGINE_LOG_LIMIT(int level, int times, int limit, const char *fmt, ...); + +#define KAEZIP_CRYPTO(LEVEL, fmt, args...) \ + do { \ + if (LEVEL > g_kaezip_log_level) { \ + break; \ + } \ + struct tm *log_tm_p = NULL; \ + time_t timep = time((time_t *)NULL); \ + log_tm_p = localtime(&timep); \ + flock(g_kaezip_debug_log_file->_fileno, LOCK_EX); \ + pthread_mutex_lock(&g_kaezip_debug_file_mutex); \ + fseek(g_kaezip_debug_log_file, 0, SEEK_END); \ + if (log_tm_p != NULL) { \ + fprintf(g_kaezip_debug_log_file, "[%4d-%02d-%02d %02d:%02d:%02d][%s][%s:%d:%s()] " fmt "\n", \ + (1900 + log_tm_p->tm_year), (1 + log_tm_p->tm_mon), log_tm_p->tm_mday, \ + log_tm_p->tm_hour, log_tm_p->tm_min, log_tm_p->tm_sec, \ + g_kaezip_log_level_string[LEVEL], __FILE__, __LINE__, __func__, ##args); \ + } else { \ + fprintf(g_kaezip_debug_log_file, "[%s][%s:%d:%s()] " fmt "\n", \ + g_kaezip_log_level_string[LEVEL], __FILE__, __LINE__, __func__, ##args); \ + } \ + if (ftell(g_kaezip_debug_log_file) > KAE_LOG_MAX_SIZE) { \ + kaezip_save_log(g_kaezip_debug_log_file); \ + ftruncate(g_kaezip_debug_log_file->_fileno, 0); \ + fseek(g_kaezip_debug_log_file, 0, SEEK_SET); \ + } \ + pthread_mutex_unlock(&g_kaezip_debug_file_mutex); \ + flock(g_kaezip_debug_log_file->_fileno, LOCK_UN); \ + } while (0) + +#define US_ERR(fmt, args...) KAEZIP_CRYPTO(KAE_ERROR, fmt, ##args) +#define US_WARN(fmt, args...) KAEZIP_CRYPTO(KAE_WARNING, fmt, ##args) +#define US_INFO(fmt, args...) KAEZIP_CRYPTO(KAE_INFO, fmt, ##args) +#define US_DEBUG(fmt, args...) KAEZIP_CRYPTO(KAE_DEBUG, fmt, ##args) + +void kaezip_debug_init_log(); +void kaezip_debug_close_log(); +void kaezip_save_log(FILE *src); + +#endif diff --git a/kae_zip_engine/src/kaezip_utils.h b/kae_zip_engine/src/kaezip_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..c9e2c93dcee2bd6d8f179057107ec69ba4068633 --- /dev/null +++ b/kae_zip_engine/src/kaezip_utils.h @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file kaezip_utils.h + * + * This file provides the utils funtion; + * + *****************************************************************************/ + +#ifndef KAEZIP_UTILS_H +#define KAEZIP_UTILS_H +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define gettid() syscall(SYS_gettid) +#define PRINTPID \ + US_DEBUG("pid=%d, ptid=%lu, tid=%d", getpid(), pthread_self(), gettid()) + +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) + +#ifndef true +#define true (0 == 0) +#endif + +#ifndef false +#define false (0 == 1) +#endif + +#define KAEZIP_FAILED (-1) +#define KAEZIP_SUCCESS (0) + +#define KAEZIP_RETURN_FAIL_IF(cond, mesg, ret) \ + if (unlikely(cond)) {\ + US_ERR(mesg); \ + return (ret); \ + }\ + +#define UNUSED(x) (void)(x) + +#define BLOCKSIZES_OF(data) (sizeof((data)) / sizeof(((data)[0]))) + +#define KAE_SPIN_INIT(q) kae_spinlock_init(&(q)) +#define KAE_SPIN_LOCK(q) kae_spinlock_lock(&(q)) +#define KAE_SPIN_TRYLOCK(q) kae_spinlock_trylock(&(q)) +#define KAE_SPIN_UNLOCK(q) kae_spinlock_unlock(&(q)) + +#define kae_free(addr) \ + do { \ + if (addr != NULL) { \ + free(addr); \ + addr = NULL; \ + } \ + } while (0) + +static inline void *kae_malloc(unsigned int size) +{ + return malloc(size); +} + +struct kae_spinlock { + int lock; +}; + +static inline void kae_spinlock_init(struct kae_spinlock *lock) +{ + lock->lock = 0; +} + +static inline void kae_spinlock_lock(struct kae_spinlock *lock) +{ + while (__sync_lock_test_and_set(&lock->lock, 1)) {} +} + +static inline int kae_spinlock_trylock(struct kae_spinlock *lock) +{ + return __sync_lock_test_and_set(&lock->lock, 1) == 0; +} + +static inline void kae_spinlock_unlock(struct kae_spinlock *lock) +{ + __sync_lock_release(&lock->lock); +} + +#endif diff --git a/kae_zip_engine/src/wd_queue_memory.c b/kae_zip_engine/src/wd_queue_memory.c new file mode 100644 index 0000000000000000000000000000000000000000..2c80cb5a6e90d1e59114e709a148fb31199c3f26 --- /dev/null +++ b/kae_zip_engine/src/wd_queue_memory.c @@ -0,0 +1,441 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file wd_queue_memory.h + * + * This file provides the queue and mempool for zlib; + * + *****************************************************************************/ + +#include +#include "wd_queue_memory.h" +#include "kaezip_log.h" +#include "uadk/v1/wd_bmm.h" +#include "uadk/v1/wd_comp.h" +#include "kaezip_ctx.h" + +void kaezip_wd_free_queue(struct wd_queue* queue); +struct wd_queue* kaezip_wd_new_queue(int comp_alg_type, int comp_optype); + +struct wd_queue* kaezip_wd_new_queue(int comp_alg_type, int comp_optype) +{ + struct wd_queue* queue = (struct wd_queue *)kae_malloc(sizeof(struct wd_queue)); + if (queue == NULL) { + US_ERR("malloc failed"); + return NULL; + } + + memset(queue, 0, sizeof(struct wd_queue)); + switch (comp_alg_type) { + case WCRYPTO_ZLIB: + queue->capa.alg = "zlib"; + break; + case WCRYPTO_GZIP: + queue->capa.alg = "gzip"; + break; + default: + kae_free(queue); + return NULL; + } + queue->capa.latency = 0; + queue->capa.throughput = 0; + + struct wcrypto_paras *priv = (struct wcrypto_paras *)&(queue->capa.priv); + priv->direction = comp_optype; + int ret = wd_request_queue(queue); + if (ret) { + US_ERR("request wd queue fail!errno:%d", ret); + kae_free(queue); + queue = NULL; + return NULL; + } + + return queue; +} + +void kaezip_wd_free_queue(struct wd_queue* queue) +{ + if (queue != NULL) { + wd_release_queue(queue); + kae_free(queue); + queue = NULL; + } +} + +void* kaezip_create_alg_wd_queue_mempool(struct wd_queue *q) +{ + unsigned int block_size = COMP_BLOCK_SIZE; + unsigned int block_num = COMP_BLOCK_NUM; + struct wd_blkpool_setup setup; + + memset(&setup, 0, sizeof(setup)); + setup.block_size = block_size; + setup.block_num = block_num; + setup.align_size = 64; // align with 64 + + void *mempool = wd_blkpool_create(q, &setup); + + return mempool; +} + +void kaezip_wd_queue_mempool_destroy(void *pool) +{ + return wd_blkpool_destroy(pool); +} + +void *kaezip_dma_map(void *usr, void *va, size_t sz) +{ + return wd_blk_iova_map(usr, va); +} + +void kaezip_dma_unmap(void *usr, void *va, void *dma, size_t sz) +{ + return wd_blk_iova_unmap(usr, dma, va); +} + +void *kaezip_wd_alloc_blk(void *pool, size_t size) +{ + if (pool == NULL) { + US_ERR("mem pool empty!"); + return NULL; + } + + return wd_alloc_blk(pool); +} + +void kaezip_wd_free_blk(void *pool, void *blk) +{ + return wd_free_blk(pool, blk); +} + +KAE_QUEUE_POOL_HEAD_S* kaezip_init_queue_pool(int algtype) +{ + KAE_QUEUE_POOL_HEAD_S *kae_pool = NULL; + + kae_pool = (KAE_QUEUE_POOL_HEAD_S *)kae_malloc(sizeof(KAE_QUEUE_POOL_HEAD_S)); + if (kae_pool == NULL) { + US_ERR("malloc pool head fail!"); + return NULL; + } + + /* fill data of head */ + kae_pool->algtype = algtype; + kae_pool->next = NULL; + kae_pool->pool_use_num = 0; + + /* malloc a pool */ + kae_pool->kae_queue_pool = (KAE_QUEUE_POOL_NODE_S *) + kae_malloc(KAE_QUEUE_POOL_MAX_SIZE * sizeof(KAE_QUEUE_POOL_NODE_S)); + if (kae_pool->kae_queue_pool == NULL) { + US_ERR("malloc failed"); + kae_free(kae_pool); + return NULL; + } + memset(kae_pool->kae_queue_pool, 0, KAE_QUEUE_POOL_MAX_SIZE * sizeof(KAE_QUEUE_POOL_NODE_S)); + + pthread_mutex_init(&kae_pool->kae_queue_mutex, NULL); + pthread_mutex_init(&kae_pool->destroy_mutex, NULL); + + return kae_pool; +} + +static KAE_QUEUE_DATA_NODE_S* kaezip_get_queue_data_from_list(KAE_QUEUE_POOL_HEAD_S* pool_head, int type) +{ + int i = 0; + KAE_QUEUE_DATA_NODE_S *queue_data_node = NULL; + KAE_QUEUE_POOL_HEAD_S *temp_pool = pool_head; + + if ((pool_head->pool_use_num == 0) && (pool_head->next == NULL)) { + return queue_data_node; + } + + while (temp_pool != NULL) { + for (i = 0; i < temp_pool->pool_use_num; i++) { + if (temp_pool->kae_queue_pool[i].node_data == NULL) { + continue; + } + + if (KAE_SPIN_TRYLOCK(temp_pool->kae_queue_pool[i].spinlock)) { + if (temp_pool->kae_queue_pool[i].node_data == NULL) { + KAE_SPIN_UNLOCK(temp_pool->kae_queue_pool[i].spinlock); + continue; + } + + if (temp_pool->kae_queue_pool[i].node_data->comp_alg_type != type) { + KAE_SPIN_UNLOCK(temp_pool->kae_queue_pool[i].spinlock); + continue; + } + + queue_data_node = temp_pool->kae_queue_pool[i].node_data; + temp_pool->kae_queue_pool[i].node_data = NULL; + KAE_SPIN_UNLOCK(temp_pool->kae_queue_pool[i].spinlock); + + US_DEBUG("kaezip get queue from pool success. queue_node id =%d", i); + return queue_data_node; + } + } + /* next pool */ + temp_pool = temp_pool->next; + } + + return queue_data_node; +} + +static void kaezip_free_wd_queue_memory(KAE_QUEUE_DATA_NODE_S *queue_node, kae_release_priv_ctx_cb release_fn) +{ + if (queue_node != NULL) { + if (release_fn != NULL && queue_node->priv_ctx != NULL) { + release_fn(queue_node->priv_ctx); + queue_node->priv_ctx = NULL; + } + + if (queue_node->kae_queue_mem_pool != NULL) { + kaezip_wd_queue_mempool_destroy(queue_node->kae_queue_mem_pool); + queue_node->kae_queue_mem_pool = NULL; + } + if (queue_node->kae_wd_queue != NULL) { + kaezip_wd_free_queue(queue_node->kae_wd_queue); + queue_node->kae_wd_queue = NULL; + } + + kae_free(queue_node); + queue_node = NULL; + } + + US_DEBUG("free wd queue success"); +} + +static KAE_QUEUE_DATA_NODE_S* kaezip_new_wd_queue_memory(int comp_alg_type, int comp_type) +{ + KAE_QUEUE_DATA_NODE_S *queue_node = NULL; + + queue_node = (KAE_QUEUE_DATA_NODE_S *)kae_malloc(sizeof(KAE_QUEUE_DATA_NODE_S)); + if (queue_node == NULL) { + US_ERR("malloc failed"); + return NULL; + } + memset(queue_node, 0, sizeof(KAE_QUEUE_DATA_NODE_S)); + + queue_node->kae_wd_queue = kaezip_wd_new_queue(comp_alg_type, comp_type); + if (queue_node->kae_wd_queue == NULL) { + US_ERR("new wd queue fail"); + goto err; + } + + queue_node->kae_queue_mem_pool = kaezip_create_alg_wd_queue_mempool(queue_node->kae_wd_queue); + if (queue_node->kae_queue_mem_pool == NULL) { + US_ERR("request mempool fail!"); + goto err; + } + + queue_node->comp_alg_type = comp_alg_type; + return queue_node; + +err: + kaezip_free_wd_queue_memory(queue_node, NULL); + return NULL; +} + +KAE_QUEUE_DATA_NODE_S* kaezip_get_node_from_pool(KAE_QUEUE_POOL_HEAD_S* pool_head, int comp_alg_type, int comp_type) +{ + KAE_QUEUE_DATA_NODE_S *queue_data_node = NULL; + + if (pool_head == NULL) { + US_ERR("input params pool_head is null"); + return NULL; + } + + queue_data_node = kaezip_get_queue_data_from_list(pool_head, comp_alg_type); + if (queue_data_node == NULL) { + queue_data_node = kaezip_new_wd_queue_memory(comp_alg_type, comp_type); + } + + return queue_data_node; +} + +static void kaezip_set_pool_use_num(KAE_QUEUE_POOL_HEAD_S *pool, int set_num) +{ + pthread_mutex_lock(&pool->kae_queue_mutex); + if (set_num > pool->pool_use_num) { + pool->pool_use_num = set_num; + } + (void)pthread_mutex_unlock(&pool->kae_queue_mutex); +} + +int kaezip_put_node_to_pool(KAE_QUEUE_POOL_HEAD_S* pool_head, KAE_QUEUE_DATA_NODE_S* node_data) +{ + int i = 0; + KAE_QUEUE_POOL_HEAD_S *temp_pool = pool_head; + KAE_QUEUE_POOL_HEAD_S *last_pool = NULL; + + if (node_data == NULL || pool_head == NULL) { + return 0; + } + + while (temp_pool != NULL) { + for (i = 0; i < KAE_QUEUE_POOL_MAX_SIZE; i++) { + if (temp_pool->kae_queue_pool[i].node_data) { + continue; + } + + if (KAE_SPIN_TRYLOCK(temp_pool->kae_queue_pool[i].spinlock)) { + if (temp_pool->kae_queue_pool[i].node_data) { + KAE_SPIN_UNLOCK(temp_pool->kae_queue_pool[i].spinlock); + continue; + } else { + temp_pool->kae_queue_pool[i].node_data = node_data; + temp_pool->kae_queue_pool[i].add_time = time((time_t *)NULL); + KAE_SPIN_UNLOCK(temp_pool->kae_queue_pool[i].spinlock); + if (i >= temp_pool->pool_use_num) { + kaezip_set_pool_use_num(temp_pool, i + 1); + } + + US_DEBUG("kaezip put queue node to pool, queue node id is %d.", i); + return 1; + } + } + } + last_pool = temp_pool; + temp_pool = temp_pool->next; + /* if no empty pool to add,new a pool */ + if (temp_pool == NULL) { + pthread_mutex_lock(&last_pool->destroy_mutex); + if (last_pool->next == NULL) { + temp_pool = kaezip_init_queue_pool(last_pool->algtype); + if (temp_pool == NULL) { + (void)pthread_mutex_unlock(&last_pool->destroy_mutex); + break; + } + last_pool->next = temp_pool; + } + (void)pthread_mutex_unlock(&last_pool->destroy_mutex); + } + } + /* if not added,free it */ + kaezip_free_wd_queue_memory(node_data, kaezip_free_ctx); + return 0; +} + +void kaezip_queue_pool_reset(KAE_QUEUE_POOL_HEAD_S* pool_head) +{ + (void)pool_head; + return; +} + +void kaezip_queue_pool_destroy(KAE_QUEUE_POOL_HEAD_S* pool_head, kae_release_priv_ctx_cb release_fn) +{ + int error = 0; + int i = 0; + KAE_QUEUE_DATA_NODE_S *queue_data_node = (KAE_QUEUE_DATA_NODE_S *)NULL; + KAE_QUEUE_POOL_HEAD_S *temp_pool = NULL; + KAE_QUEUE_POOL_HEAD_S *cur_pool = pool_head; + + while (cur_pool != NULL) { + error = pthread_mutex_lock(&cur_pool->destroy_mutex); + if (error != 0) { + (void)pthread_mutex_unlock(&cur_pool->destroy_mutex); + return; + } + + error = pthread_mutex_lock(&cur_pool->kae_queue_mutex); + if (error != 0) { + (void)pthread_mutex_unlock(&cur_pool->destroy_mutex); + return; + } + for (i = 0; i < cur_pool->pool_use_num; i++) { + queue_data_node = cur_pool->kae_queue_pool[i].node_data; + if (queue_data_node != NULL) { + kaezip_free_wd_queue_memory(queue_data_node, release_fn); + US_DEBUG("kae queue node destroy success. queue_node id =%d", i); + cur_pool->kae_queue_pool[i].node_data = NULL; + } + } + US_DEBUG("pool use num :%d.", cur_pool->pool_use_num); + + kae_free(cur_pool->kae_queue_pool); + + (void)pthread_mutex_unlock(&cur_pool->kae_queue_mutex); + (void)pthread_mutex_unlock(&cur_pool->destroy_mutex); + + pthread_mutex_destroy(&cur_pool->kae_queue_mutex); + pthread_mutex_destroy(&cur_pool->destroy_mutex); + + temp_pool = cur_pool; + + kae_free(cur_pool); + + cur_pool = temp_pool->next; + } + + return; +} + +void kaezip_queue_pool_check_and_release(KAE_QUEUE_POOL_HEAD_S* pool_head, kae_release_priv_ctx_cb release_fn) +{ + int i = 0; + int error; + time_t current_time; + KAE_QUEUE_DATA_NODE_S *queue_data_node = NULL; + KAE_QUEUE_POOL_HEAD_S *cur_pool = pool_head; + + current_time = time((time_t *)NULL); + + while (cur_pool != NULL) { + error = pthread_mutex_lock(&cur_pool->destroy_mutex); + if (error != 0) { + cur_pool = cur_pool->next; + (void)pthread_mutex_unlock(&cur_pool->destroy_mutex); + continue; + } + if (cur_pool->kae_queue_pool == NULL) { + (void)pthread_mutex_unlock(&cur_pool->destroy_mutex); + cur_pool = cur_pool->next; + continue; + } + + for (i = cur_pool->pool_use_num - 1; i >= 0; i--) { + if (cur_pool->kae_queue_pool[i].node_data == NULL) { + continue; + } + + if (difftime(current_time, cur_pool->kae_queue_pool[i].add_time) < CHECK_QUEUE_TIME_SECONDS) { + continue; + } + + if (KAE_SPIN_TRYLOCK(cur_pool->kae_queue_pool[i].spinlock)) { + if ((cur_pool->kae_queue_pool[i].node_data == NULL) || + (difftime(current_time, cur_pool->kae_queue_pool[i].add_time) < CHECK_QUEUE_TIME_SECONDS)) { + KAE_SPIN_UNLOCK(cur_pool->kae_queue_pool[i].spinlock); + continue; + } else { + queue_data_node = cur_pool->kae_queue_pool[i].node_data; + cur_pool->kae_queue_pool[i].node_data = (KAE_QUEUE_DATA_NODE_S *)NULL; + KAE_SPIN_UNLOCK(cur_pool->kae_queue_pool[i].spinlock); + + kaezip_free_wd_queue_memory(queue_data_node, release_fn); + + US_DEBUG("hpre queue list release success. queue node id =%d", i); + } + } + } + + (void)pthread_mutex_unlock(&cur_pool->destroy_mutex); + cur_pool = cur_pool->next; + } + + return; +} + diff --git a/kae_zip_engine/src/wd_queue_memory.h b/kae_zip_engine/src/wd_queue_memory.h new file mode 100644 index 0000000000000000000000000000000000000000..1f6b674c2d3e1d8a1d45e6a0e2df4e60e15b1ab4 --- /dev/null +++ b/kae_zip_engine/src/wd_queue_memory.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2019. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the zlib License. + * You may obtain a copy of the License at + * + * https://www.zlib.net/zlib_license.html + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * zlib License for more details. + */ + +/***************************************************************************** + * @file wd_queue_memory.h + * + * This file provides the queue and mempool for zlib; + * + *****************************************************************************/ + +#ifndef __KAEZIP_QUEUE_MEMORY_H +#define __KAEZIP_QUEUE_MEMORY_H + +#include +#include "uadk/v1/wd.h" +#include "kaezip_utils.h" + +#define KAE_QUEUE_POOL_MAX_SIZE (512) +#define CHECK_QUEUE_TIME_SECONDS (60) // seconds + +#define COMP_BLOCK_NUM (4) +#define COMP_BLOCK_SIZE (1024 * 1024) + +typedef void (*kae_release_priv_ctx_cb)(void* priv_ctx); + +typedef struct KAE_QUEUE_DATA_NODE { + struct wd_queue *kae_wd_queue; + void *kae_queue_mem_pool; + int comp_alg_type; + void *priv_ctx; +} KAE_QUEUE_DATA_NODE_S; + +typedef struct KAE_QUEUE_POOL_NODE { + // int using_flag; /* used:true,nouse:false */ + struct kae_spinlock spinlock; + time_t add_time; + // int index; /* index of node,init:-1 */ + KAE_QUEUE_DATA_NODE_S *node_data; + // KAE_QUEUE_POOL_NODE_S *next; +} KAE_QUEUE_POOL_NODE_S; + +typedef struct KAE_QUEUE_POOL_HEAD { + // int init_flag; + int pool_use_num; + int algtype; /* alg type,just init at init pool */ + pthread_mutex_t destroy_mutex; + pthread_mutex_t kae_queue_mutex; + struct KAE_QUEUE_POOL_HEAD *next; /* next pool */ + KAE_QUEUE_POOL_NODE_S *kae_queue_pool; /* point to a attray */ +} KAE_QUEUE_POOL_HEAD_S; + +void kaezip_wd_free_blk(void *pool, void *blk); +void *kaezip_wd_alloc_blk(void *pool, size_t size); +void *kaezip_dma_map(void *usr, void *va, size_t sz); +void kaezip_dma_unmap(void *usr, void *va, void *dma, size_t sz); + +KAE_QUEUE_POOL_HEAD_S* kaezip_init_queue_pool (int algtype); +KAE_QUEUE_DATA_NODE_S* kaezip_get_node_from_pool(KAE_QUEUE_POOL_HEAD_S* pool_head, int alg_comp_type, int comp_optype); +int kaezip_put_node_to_pool (KAE_QUEUE_POOL_HEAD_S* pool_head, KAE_QUEUE_DATA_NODE_S* node_data); +void kaezip_queue_pool_reset(KAE_QUEUE_POOL_HEAD_S* pool_head); +void kaezip_queue_pool_destroy(KAE_QUEUE_POOL_HEAD_S* pool_head, kae_release_priv_ctx_cb release_fn); +void kaezip_queue_pool_check_and_release(KAE_QUEUE_POOL_HEAD_S* pool_head, kae_release_priv_ctx_cb release_ectx_fn); + +#endif + diff --git a/kae_zip_engine/test/kaezip_perf.c b/kae_zip_engine/test/kaezip_perf.c new file mode 100644 index 0000000000000000000000000000000000000000..f042492071eed892c2c0c8eaf656e113487d6eb7 --- /dev/null +++ b/kae_zip_engine/test/kaezip_perf.c @@ -0,0 +1,241 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +uint8_t *get_compress_input(size_t input_sz) +{ + uint8_t *inbuf = (uint8_t *)malloc(input_sz * sizeof(uint8_t)); + if (inbuf == NULL) { + return NULL; + } + + memset(inbuf, 0, input_sz); + srand((unsigned int)time(NULL)); + int i = 0; + for (i = 0; i < input_sz; i++) { + inbuf[i] = (uint8_t)rand() % 254 + 1; + } + + return inbuf; +} + +uint32_t *get_decompress_input(size_t input_sz, uLong *pblen) +{ + uint32_t *inbuf = (uint32_t *)malloc(input_sz * sizeof(uint32_t)); + if (inbuf == NULL) { + return NULL; + } + + memset(inbuf, 0, input_sz); + srand((unsigned int)time(NULL)); + int i = 0; + for (i = 0; i < input_sz; i++) { + inbuf[i] = (uint32_t)rand() % 254 + 1; + } + + uLong blen = compressBound(input_sz); + uint32_t *outbuf = (uint32_t *)malloc(blen * sizeof(uint32_t)); + memset(outbuf, 0, blen); + int cret = compress2((Bytef *)outbuf, (uLongf *)&blen, (Bytef *)inbuf, (uLong)input_sz, 1); + if (cret != Z_OK && cret != Z_BUF_ERROR) { + free(outbuf); + outbuf = NULL; + } + + free(inbuf); + *pblen = blen; + return outbuf; +} + +int do_multi_perf(int multi, int stream_len, int loop_times, int compress, + void* output, uLong output_sz, void* inbuf, uLong blen) +{ + int i,j; + pid_t pid_child = 0; + struct timeval start, stop; + gettimeofday(&start, NULL); + for (i = 0; i < multi; i++) { + pid_child = fork(); + if (pid_child == 0 || pid_child == -1) { + break; + } + } + + if (pid_child == 0) { + for(j = 0;j < loop_times;j++) + { + int ret = -1; + if (compress) { + blen = compressBound(stream_len); + ret = compress2((Bytef *)output, (uLongf *)&blen, (Bytef *)inbuf, (uLong)stream_len, 1); + if (ret != Z_OK && ret != Z_BUF_ERROR) { + printf("compres error, ret = %d\n", ret); + return -1; + } + } else { + ret = uncompress((Bytef *)output, &output_sz, (const Bytef *)inbuf, blen); + if (ret < 0) { + printf("uncompres error, ret = %d\n", ret); + return -1; + } + } + } + } + + if (pid_child > 0) { + int ret = -1; + while (1) { + ret = wait(NULL); + if (ret == -1) { + if (errno == EINTR) { + continue; + } + break; + } + } + } + + if (pid_child > 0 || multi == 0) { + if (multi == 0) { multi = 1; } + gettimeofday(&stop, NULL); + uLong time1 = (stop.tv_sec - start.tv_sec) * 1000000 + stop.tv_usec - start.tv_usec; + float speed1 = 1000000.0 / time1 * loop_times * multi * stream_len / 1000 / 1000 / 1000; + printf("kaezip %s perf result:\n", compress ? "compress" : "decompress"); + printf(" time used: %lu us, speed = %.3f GB/s\n", time1, speed1); + } + + return 0; +} + +int do_compress_perf(int multi, int stream_len, int loop_times) +{ + int i = 0; + uint8_t *inbuf = get_compress_input(stream_len); + if (inbuf == NULL) { + return -1; + } + + uLong blen = compressBound(stream_len); + uLong output_sz = blen; + uint8_t *outbuf = (uint8_t *)malloc(output_sz * sizeof(uint8_t)); + if (outbuf == NULL) { + return -1; + } + memset(outbuf, 0, output_sz); + + int ret = do_multi_perf(multi, stream_len, loop_times, 1, outbuf, output_sz, inbuf, blen); + + free(inbuf); + inbuf = NULL; + free(outbuf); + outbuf = NULL; + return ret; +} + +int do_decompress_perf(int multi, int stream_len, int loop_times) +{ + int i, j; + uLong blen = 0; + uint32_t *inbuf = get_decompress_input(stream_len, &blen); + if (inbuf == NULL) { + return -1; + } + + uLong output_sz = stream_len; + uint32_t *output = malloc(output_sz * sizeof(uint32_t)); + if (output == NULL) { + return -1; + } + + int ret = do_multi_perf(multi, stream_len, loop_times, 0, output, output_sz, inbuf, blen); + + free(inbuf); + inbuf = NULL; + free(output); + output = NULL; + return ret; +} + +void usage(void) +{ + printf("usage: \n"); + printf(" -m: multi process \n"); + printf(" -l: stream length(KB)\n"); + printf(" -n: loop times\n"); + printf(" -d: compress or decompress\n"); + printf(" example: ./kaezip_perf -m 2 -l 1024 -n 1000\n"); + printf(" ./kaezip_perf -d -m 2 -l 1024 -n 1000\n"); +} + +// [root@localhost test]# ./kaezip_perf +// usage: +// -m: multi process +// -l: stream length(KB) +// -n: loop times +// -d: compress or decompress +// example: ./kaezip_perf -m 2 -l 1024 -n 1000 +// ./kaezip_perf -d -m 2 -l 1024 -n 1000 + +// default input parameter used +// kaezip perf input parameter: multi process 2, stream length: 1024(KB), loop times: 1000 +// kaezip compress perf result: +// time used: 509004 us, speed = 4.024 GB/s +// +// [root@localhost test]# ./kaezip_perf -d +// kaezip perf parameter: multi process 2, stream length: 1024(KB), loop times: 1000 +// kaezip decompress perf result: +// time used: 810318 us, speed = 2.527 GB/s + +int main(int argc, char **argv) +{ + int o = 0; + const char *optstring = "dm:l:n:h"; + int multi = 2; + int stream_len = 1024; + int loop_times = 1000; + int compress = 1; + while ((o = getopt(argc, argv, optstring)) != -1) { + if(optstring == NULL) continue; + switch (o) { + case 'm': + multi = atoi(optarg); + break; + case 'l': + stream_len = atoi(optarg); + break; + case 'n': + loop_times = atoi(optarg); + break; + case 'd': + compress = 0; + break; + case 'h': + usage(); + return 0; + } + } + + if (argc <= 1) { + usage(); + printf("\ndefault input parameter used\n"); + } + + printf("kaezip perf parameter: multi process %d, stream length: %d(KB), loop times: %d\n", multi, stream_len, loop_times); + + stream_len = 1000 * stream_len; + if (compress) { + return do_compress_perf(multi, stream_len, loop_times); + } else { + return do_decompress_perf(multi, stream_len, loop_times); + } +} diff --git a/kae_zip_engine/test/makefile b/kae_zip_engine/test/makefile new file mode 100644 index 0000000000000000000000000000000000000000..baaaf3693b08930637adaeede1d33bf31f156614 --- /dev/null +++ b/kae_zip_engine/test/makefile @@ -0,0 +1,18 @@ + +INCDIR += -I /usr/local/kaezip/include/ +# Include Libs. +LIBDIR := -L/usr/local/kaezip/lib + +# The flags +CFLAGS := -g -lz -Wall +LDFLAGS := $(LIBDIR) + +all: kaezip_perf +objects = kaezip_perf.c + +kaezip_perf : $(objects) + gcc $(CFLAGS) $(LDFLAGS) $(INCDIR) -o kaezip_perf $(objects) + +.PHONY : clean +clean : + -rm -f kaezip_perf diff --git a/uadk/Makefile.am b/uadk/Makefile.am index 66da853b22a1597b7b7b1396e41bd4f6e74b6f16..7b81168a5cb73091be4e31a7f8b86f36bf0b4273 100644 --- a/uadk/Makefile.am +++ b/uadk/Makefile.am @@ -36,7 +36,7 @@ pkginclude_HEADERS = include/wd.h include/wd_cipher.h include/wd_aead.h \ include/wd_common.h include/wd_ecc.h include/wd_sched.h nobase_pkginclude_HEADERS = v1/wd.h v1/wd_cipher.h v1/wd_aead.h v1/uacce.h v1/wd_dh.h \ - v1/wd_digest.h v1/wd_rsa.h v1/wd_bmm.h + v1/wd_digest.h v1/wd_rsa.h v1/wd_bmm.h v1/wd_comp.h lib_LTLIBRARIES=libwd.la libwd_comp.la libwd_crypto.la libhisi_zip.la \ libhisi_hpre.la libhisi_sec.la