Skip to content

Commit fd17d3b

Browse files
marty-emmanuelmarty-emmanuel
committed
Reduce token count by 1% without changing the ratio
1 parent cff0bad commit fd17d3b

File tree

1 file changed

+29
-19
lines changed

1 file changed

+29
-19
lines changed

‎src/shrink.c‎

Lines changed: 29 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@
4949
#define LAST_MATCH_OFFSET 12
5050
#define LAST_LITERALS 5
5151

52+
#define MODESWITCH_PENALTY 1
53+
5254
/** One match */
5355
typedef struct _lz4ultra_match {
5456
unsigned int length;
@@ -388,14 +390,14 @@ static void lz4ultra_find_all_matches(lsza_compressor *pCompressor, const int nS
388390
}
389391

390392
/**
391-
* Get the number of extra bytes required to represent a literals length
393+
* Get the number of extra bits required to represent a literals length
392394
*
393395
* @param nLength literals length
394396
*
395-
* @return number of extra bytes required
397+
* @return number of extra bits required
396398
*/
397399
static inline int lz4ultra_get_literals_varlen_size(const int nLength) {
398-
return ((nLength - LITERALS_RUN_LEN + 255) / 255);
400+
return ((nLength - LITERALS_RUN_LEN + 255) / 255) << 3;
399401
}
400402

401403
/**
@@ -420,14 +422,14 @@ static inline int lz4ultra_write_literals_varlen(unsigned char *pOutData, int nO
420422
}
421423

422424
/**
423-
* Get the number of extra bytes required to represent an encoded match length
425+
* Get the number of extra bits required to represent an encoded match length
424426
*
425427
* @param nLength encoded match length (actual match length - MIN_MATCH_SIZE)
426428
*
427-
* @return number of extra bytes required
429+
* @return number of extra bits required
428430
*/
429431
static inline int lz4ultra_get_match_varlen_size(const int nLength) {
430-
return ((nLength - MATCH_RUN_LEN + 255) / 255);
432+
return ((nLength - MATCH_RUN_LEN + 255) / 255) << 3;
431433
}
432434

433435
/**
@@ -463,19 +465,21 @@ static void lz4ultra_optimize_matches(lsza_compressor *pCompressor, const int nS
463465
int nLastLiteralsOffset;
464466
int i;
465467

466-
cost[nEndOffset - 1] = 1;
468+
cost[nEndOffset - 1] = 8;
467469
nLastLiteralsOffset = nEndOffset;
468470

469471
for (i = nEndOffset - 2; i != (nStartOffset - 1); i--) {
470472
int nBestCost, nBestMatchLen, nBestMatchOffset;
471473

472474
int nLiteralsLen = nLastLiteralsOffset - i;
473-
nBestCost = 1 + cost[i + 1];
475+
nBestCost = 8 + cost[i + 1];
474476
if (nLiteralsLen >= LITERALS_RUN_LEN && ((nLiteralsLen - LITERALS_RUN_LEN) % 255) == 0) {
475477
/* Add to the cost of encoding literals as their number crosses a variable length encoding boundary.
476478
* The cost automatically accumulates down the chain. */
477-
nBestCost++;
479+
nBestCost += 8;
478480
}
481+
if (pCompressor->match[(i + 1) << MATCHES_PER_OFFSET_SHIFT].length >= MIN_MATCH_SIZE)
482+
nBestCost += MODESWITCH_PENALTY;
479483
nBestMatchLen = 0;
480484
nBestMatchOffset = 0;
481485

@@ -490,8 +494,10 @@ static void lz4ultra_optimize_matches(lsza_compressor *pCompressor, const int nS
490494
if ((i + nMatchLen) > (nEndOffset - LAST_LITERALS))
491495
nMatchLen = nEndOffset - LAST_LITERALS - i;
492496

493-
nCurCost = 1 + 2 + lz4ultra_get_match_varlen_size(nMatchLen - MIN_MATCH_SIZE);
497+
nCurCost = 8 + 16 + lz4ultra_get_match_varlen_size(nMatchLen - MIN_MATCH_SIZE);
494498
nCurCost += cost[i + nMatchLen];
499+
if (pCompressor->match[(i + nMatchLen) << MATCHES_PER_OFFSET_SHIFT].length >= MIN_MATCH_SIZE)
500+
nCurCost += MODESWITCH_PENALTY;
495501

496502
if (nBestCost >= nCurCost) {
497503
nBestCost = nCurCost;
@@ -513,8 +519,10 @@ static void lz4ultra_optimize_matches(lsza_compressor *pCompressor, const int nS
513519
for (k = MIN_MATCH_SIZE; k < nMatchRunLen; k++) {
514520
int nCurCost;
515521

516-
nCurCost = 1 + 2 /* no extra match len bytes */;
522+
nCurCost = 8 + 16 /* no extra match len bytes */;
517523
nCurCost += cost[i + k];
524+
if (pCompressor->match[(i + k) << MATCHES_PER_OFFSET_SHIFT].length >= MIN_MATCH_SIZE)
525+
nCurCost += MODESWITCH_PENALTY;
518526

519527
if (nBestCost >= nCurCost) {
520528
nBestCost = nCurCost;
@@ -526,8 +534,10 @@ static void lz4ultra_optimize_matches(lsza_compressor *pCompressor, const int nS
526534
for (; k <= nMatchLen; k++) {
527535
int nCurCost;
528536

529-
nCurCost = 1 + 2 + lz4ultra_get_match_varlen_size(k - MIN_MATCH_SIZE);
537+
nCurCost = 8 + 16 + lz4ultra_get_match_varlen_size(k - MIN_MATCH_SIZE);
530538
nCurCost += cost[i + k];
539+
if (pCompressor->match[(i + k) << MATCHES_PER_OFFSET_SHIFT].length >= MIN_MATCH_SIZE)
540+
nCurCost += MODESWITCH_PENALTY;
531541

532542
if (nBestCost >= nCurCost) {
533543
nBestCost = nCurCost;
@@ -569,10 +579,10 @@ static void lz4ultra_optimize_command_count(lsza_compressor *pCompressor, const
569579
if (nMatchLen <= 19 && (i + nMatchLen) < nEndOffset) {
570580
int nMatchOffset = pMatch->offset;
571581
int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE;
572-
int nCommandSize = 1 /* token */ + lz4ultra_get_literals_varlen_size(nNumLiterals) + 2 /* match offset */ + lz4ultra_get_match_varlen_size(nEncodedMatchLen);
582+
int nCommandSize = 8 /* token */ + lz4ultra_get_literals_varlen_size(nNumLiterals) + 16 /* match offset */ + lz4ultra_get_match_varlen_size(nEncodedMatchLen);
573583

574584
if (pCompressor->match[(i + nMatchLen) << MATCHES_PER_OFFSET_SHIFT].length >= MIN_MATCH_SIZE) {
575-
if (nCommandSize >= (nMatchLen + lz4ultra_get_literals_varlen_size(nNumLiterals + nMatchLen))) {
585+
if (nCommandSize >= ((nMatchLen << 3) + lz4ultra_get_literals_varlen_size(nNumLiterals + nMatchLen))) {
576586
/* This command is a match; the next command is also a match. The next command currently has no literals; replacing this command by literals will
577587
* make the next command eat the cost of encoding the current number of literals, + nMatchLen extra literals. The size of the current match command is
578588
* at least as much as the number of literal bytes + the extra cost of encoding them in the next match command, so we can safely replace the current
@@ -589,7 +599,7 @@ static void lz4ultra_optimize_command_count(lsza_compressor *pCompressor, const
589599
nNextNumLiterals++;
590600
} while (nCurIndex < nEndOffset && pCompressor->match[nCurIndex << MATCHES_PER_OFFSET_SHIFT].length < MIN_MATCH_SIZE);
591601

592-
if (nCommandSize >= (nMatchLen + lz4ultra_get_literals_varlen_size(nNumLiterals + nNextNumLiterals + nMatchLen) - lz4ultra_get_literals_varlen_size(nNextNumLiterals))) {
602+
if (nCommandSize >= ((nMatchLen << 3) + lz4ultra_get_literals_varlen_size(nNumLiterals + nNextNumLiterals + nMatchLen) - lz4ultra_get_literals_varlen_size(nNextNumLiterals))) {
593603
/* This command is a match, and is followed by literals, and then another match or the end of the input data. If encoding this match as literals doesn't take
594604
* more room than the match, and doesn't grow the next match command's literals encoding, go ahead and remove the command. */
595605
nReduce = 1;
@@ -655,9 +665,9 @@ static int lz4ultra_write_block(lsza_compressor *pCompressor, const unsigned cha
655665
int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE;
656666
int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN) ? LITERALS_RUN_LEN : nNumLiterals;
657667
int nTokenMatchLen = (nEncodedMatchLen >= MATCH_RUN_LEN) ? MATCH_RUN_LEN : nEncodedMatchLen;
658-
int nCommandSize = 1 /* token */ + lz4ultra_get_literals_varlen_size(nNumLiterals) + nNumLiterals + 2 /* match offset */ + lz4ultra_get_match_varlen_size(nEncodedMatchLen);
668+
int nCommandSize = 8 /* token */ + lz4ultra_get_literals_varlen_size(nNumLiterals) + (nNumLiterals << 3) + 16 /* match offset */ + lz4ultra_get_match_varlen_size(nEncodedMatchLen);
659669

660-
if ((nOutOffset + nCommandSize) > nMaxOutDataSize)
670+
if ((nOutOffset + (nCommandSize >> 3)) > nMaxOutDataSize)
661671
return -1;
662672
if (nMatchOffset < MIN_OFFSET || nMatchOffset > MAX_OFFSET)
663673
return -1;
@@ -688,9 +698,9 @@ static int lz4ultra_write_block(lsza_compressor *pCompressor, const unsigned cha
688698

689699
{
690700
int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN) ? LITERALS_RUN_LEN : nNumLiterals;
691-
int nCommandSize = 1 /* token */ + lz4ultra_get_literals_varlen_size(nNumLiterals) + nNumLiterals;
701+
int nCommandSize = 8 /* token */ + lz4ultra_get_literals_varlen_size(nNumLiterals) + (nNumLiterals << 3);
692702

693-
if ((nOutOffset + nCommandSize) > nMaxOutDataSize)
703+
if ((nOutOffset + (nCommandSize >> 3)) > nMaxOutDataSize)
694704
return -1;
695705

696706
pOutData[nOutOffset++] = (nTokenLiteralsLen << 4);

0 commit comments

Comments
 (0)