From dc347506d21b086635b5d08592d4dff04b1e9d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E5=87=A4?= Date: Thu, 24 Jul 2025 09:05:29 +0000 Subject: [PATCH] fix: lz4 perftest patch support lz77_only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 白凤 --- KAELz4/test/perftest/lzbench_KAELz4.patch | 103 ++++++++++------------ 1 file changed, 47 insertions(+), 56 deletions(-) diff --git a/KAELz4/test/perftest/lzbench_KAELz4.patch b/KAELz4/test/perftest/lzbench_KAELz4.patch index 086cbb2..5c5fdef 100644 --- a/KAELz4/test/perftest/lzbench_KAELz4.patch +++ b/KAELz4/test/perftest/lzbench_KAELz4.patch @@ -376,19 +376,19 @@ diff -uprN lzbench-master/lz4/lz4.c lzbench-master-KAELz4/lz4/lz4.c return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } -@@ -980,6 +1023,303 @@ LZ4_FORCE_INLINE int LZ4_compress_generi +@@ -980,6 +1023,294 @@ LZ4_FORCE_INLINE int LZ4_compress_generi if (inputSize hardwareBlockSize) { ++ // US_DEBUG("remainingLength:%d, hardware:%d\n", remainingLength, hardwareBlockSize); ++ if (remainingLength >= hardwareBlockSize + MFLIMIT) { + srcSize = hardwareBlockSize; + // part1.基于KAE接口进行硬化压缩 + result = kaelz4_compress(&ctxBody, src, srcSize); @@ -472,41 +472,41 @@ diff -uprN lzbench-master/lz4/lz4.c lzbench-master-KAELz4/lz4/lz4.c + offset_hist[0] = 1; offset_hist[1] = 4; offset_hist[2] = 8; + + while (sequenceCount < ctxBody.seqnum) { -+ US_DEBUG("sequenceCount:%d, ctxBody.seqnum:%d.", sequenceCount, ctxBody.seqnum); ++ // US_DEBUG("sequenceCount:%d, ctxBody.seqnum:%d.", sequenceCount, ctxBody.seqnum); + sequenceCount++; // sequence计数 -+ offBase = sequencespoint->offBase; ++ offBase = sequencespoint->offBase + 1; + litLength = sequencespoint->litLength; + mlBase = sequencespoint->mlBase; -+ US_DEBUG("%d:Before trans: offBase %d-litLength %d-mlbase %d-remainingLength %d\n", i++, offBase, litLength, mlBase, remainingLength); ++ // US_DEBUG("%d:Before trans: offBase %d-litLength %d-mlbase %d-remainingLength %d\n", i++, offBase, litLength, mlBase, remainingLength); + + /* 1.从repcode还原出raw offset，根据的zstd的table的逻辑，具体查看mannual文档 */ -+ size_t realoffset; -+ if (offBase <= 3) { -+ idx = offBase - 1; -+ if (litLength == 0) { -+ idx++; -+ } -+ if (idx == 0) { -+ realoffset = offset_hist[0]; -+ } else { -+ realoffset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1; -+ if (idx > 1) { -+ offset_hist[2] = offset_hist[1]; -+ } -+ offset_hist[1] = offset_hist[0]; -+ offset_hist[0] = realoffset; -+ } -+ } else { -+ realoffset = offBase - 3; -+ offset_hist[2] = offset_hist[1]; -+ offset_hist[1] = offset_hist[0]; -+ offset_hist[0] = realoffset; -+ } -+ offBase = realoffset; // 真正的offset ++ //size_t realoffset; ++ //if (offBase <= 3) { ++ // idx = offBase - 1; ++ // if (litLength == 0) { ++ // idx++; ++ // } ++ // if (idx == 0) { ++ // realoffset = offset_hist[0]; ++ // } else { ++ // realoffset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1; ++ // if (idx > 1) { ++ // offset_hist[2] = offset_hist[1]; ++ // } ++ // offset_hist[1] = offset_hist[0]; ++ // offset_hist[0] = realoffset; ++ // } ++ //} else { ++ // realoffset = offBase - 3; ++ // offset_hist[2] = offset_hist[1]; ++ // offset_hist[1] = offset_hist[0]; ++ // offset_hist[0] = realoffset; ++ //} ++ // offBase = realoffset; // 真正的offset + + /* 2.基于mlBase是否为0（即3），对offset和literal length进行校正 */ + if (mlBase <= TOKEN_NUM_CONTROL) { -+ US_DEBUG("litLength:%d, tempLiteralLength:%d", litLength, tempLiteralLength); ++ // US_DEBUG("litLength:%d, tempLiteralLength:%d", litLength, tempLiteralLength); + // 针对zstd硬算返回match length为3的情况，为保证存量可解，准备还原为纯literal形式的sequence + translit = ip + litLength - offBase; // 定位到3match所在位置 + // 就是说并不是补-offset就能找到，事实上找不到，得用src！！！ @@ -520,9 +520,9 @@ diff -uprN lzbench-master/lz4/lz4.c lzbench-master-KAELz4/lz4/lz4.c + LZ4_wildCopy8(temp_point, translit, temp_point + mlBase + 3); + temp_point += mlBase + 3; + tempLiteralLength += mlBase + 3; -+ US_DEBUG("%d:3match After trans: offBase %d-litLength %d-mlbase %d\n", i++, offBase, litLength, mlBase); ++ // US_DEBUG("%d:3match After trans: offBase %d-litLength %d-mlbase %d\n", i++, offBase, litLength, mlBase); + sequencespoint++; -+ // 追踪 ++ // 追踪 + ip += litLength + mlBase + 3; + + continue; // 存完即可开启下一轮，因为无需组织token了 @@ -534,7 +534,7 @@ diff -uprN lzbench-master/lz4/lz4.c lzbench-master-KAELz4/lz4/lz4.c + // 追踪 + ip += litLength - tempLiteralLength + mlBase + 4; + } -+ US_DEBUG("%d:After trans: offBase %d-litLength %d-mlbase %d\n", i++, offBase, litLength, mlBase); ++ // US_DEBUG("%d:After trans: offBase %d-litLength %d-mlbase %d\n", i++, offBase, litLength, mlBase); + + sequencespoint++; + // 特殊场景判断，倒数第二个sequence的mlBase小于12Bytes，则解压会异常(暂未触发) @@ -595,7 +595,7 @@ diff -uprN lzbench-master/lz4/lz4.c lzbench-master-KAELz4/lz4/lz4.c + } else { // 这个分支就是每轮正常的最后一个无三元组的literal字段处理 + lastRun = (U32)(ctxBody.seqStore.lit - litpoint); + } -+ US_DEBUG("Deal with last literal: %d， remaining-length:%d.\n", lastRun, remainingLength); ++ // US_DEBUG("Deal with last literal: %d， remaining-length:%d.\n", lastRun, remainingLength); + /* 对每个子块，最后会余下暂存数组中的字符和最后literal的字符，对于这两部分，需要拼起来放到下一个子块的literal前: + (1) 对于非最后一个子块，则将最后的literal先暂存到暂存数组，并更新tempLiteralLength即可 + (2) 对最后一个子块，就按原先最后的处理逻辑来处理 */ @@ -603,16 +603,12 @@ diff -uprN lzbench-master/lz4/lz4.c lzbench-master-KAELz4/lz4/lz4.c + LZ4_wildCopy8(temp_point, litpoint, temp_point + lastRun); + temp_point += lastRun; + tempLiteralLength += lastRun; -+ US_DEBUG("Remaining for next subblock, tempLiteralLength:%d.", tempLiteralLength); ++ // US_DEBUG("Remaining for next subblock, tempLiteralLength:%d.", tempLiteralLength); + // 更新参数 + src += hardwareBlockSize; // 因为前面是关于本128K的三元组处理完了，tempLiteralLength是三元组剩的，而这个128k的最后的literal还没处理 + remainingLength -= hardwareBlockSize; // 要和src对齐 -+ -+ if (remainingLength < hardwareBlockSize) { -+ srcSize = remainingLength; -+ } else { -+ srcSize = hardwareBlockSize; -+ } ++ ++ srcSize = remainingLength; + ip = src; + if (remainingLength <= MFLIMIT) { + specialFlag = 1; @@ -620,12 +616,7 @@ diff -uprN lzbench-master/lz4/lz4.c lzbench-master-KAELz4/lz4/lz4.c + } + } else { // 场景2：最后一个子块，则需要按原流程处理暂存数组和literal + lastRun += tempLiteralLength; // 把暂存的补进来 -+ int saveChar = 0; -+ if (remainingLength >= MFLIMIT) { -+ saveChar = MFLIMIT; // 把预留的12个原始字符纳入统计 -+ } else { -+ saveChar = remainingLength; -+ } ++ int saveChar = MFLIMIT; // 把预留的12个原始字符纳入统计 + lastRun += saveChar; + + // 更新参数 @@ -641,7 +632,7 @@ diff -uprN lzbench-master/lz4/lz4.c lzbench-master-KAELz4/lz4/lz4.c + } else { + *op++ = (BYTE)(lastRun< 0); -+ US_DEBUG("LZ4_compress_generic: compressed %ld bytes into %ld bytes\n", inputSize, result); ++ // US_DEBUG("LZ4_compress_generic: compressed %ld bytes into %ld bytes\n", inputSize, result); + // 压缩硬件资源释放 + ctxBody.seqStore.lit = ctxBody.seqStore.litStart = NULL; + ctxBody.seqStore.sequences = ctxBody.seqStore.sequencesStart = NULL; -- Gitee