Counter Strike : Global Offensive Source Code
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

801 lines
24 KiB

  1. // zdeflate.cpp - written and placed in the public domain by Wei Dai
  2. // Many of the algorithms and tables used here came from the deflate implementation
  3. // by Jean-loup Gailly, which was included in Crypto++ 4.0 and earlier. I completely
  4. // rewrote it in order to fix a bug that I could not figure out. This code
  5. // is less clever, but hopefully more understandable and maintainable.
  6. #include "pch.h"
  7. #include "zdeflate.h"
  8. #include <functional>
  9. #if _MSC_VER >= 1600
  10. // for make_unchecked_array_iterator
  11. #include <iterator>
  12. #endif
  13. NAMESPACE_BEGIN(CryptoPP)
  14. using namespace std;
  15. LowFirstBitWriter::LowFirstBitWriter(BufferedTransformation *attachment)
  16. : Filter(attachment), m_counting(false), m_buffer(0), m_bitsBuffered(0), m_bytesBuffered(0)
  17. {
  18. }
  19. void LowFirstBitWriter::StartCounting()
  20. {
  21. assert(!m_counting);
  22. m_counting = true;
  23. m_bitCount = 0;
  24. }
  25. unsigned long LowFirstBitWriter::FinishCounting()
  26. {
  27. assert(m_counting);
  28. m_counting = false;
  29. return m_bitCount;
  30. }
  31. void LowFirstBitWriter::PutBits(unsigned long value, unsigned int length)
  32. {
  33. if (m_counting)
  34. m_bitCount += length;
  35. else
  36. {
  37. m_buffer |= value << m_bitsBuffered;
  38. m_bitsBuffered += length;
  39. assert(m_bitsBuffered <= sizeof(unsigned long)*8);
  40. while (m_bitsBuffered >= 8)
  41. {
  42. m_outputBuffer[m_bytesBuffered++] = (byte)m_buffer;
  43. if (m_bytesBuffered == m_outputBuffer.size())
  44. {
  45. AttachedTransformation()->PutModifiable(m_outputBuffer, m_bytesBuffered);
  46. m_bytesBuffered = 0;
  47. }
  48. m_buffer >>= 8;
  49. m_bitsBuffered -= 8;
  50. }
  51. }
  52. }
  53. void LowFirstBitWriter::FlushBitBuffer()
  54. {
  55. if (m_counting)
  56. m_bitCount += 8*(m_bitsBuffered > 0);
  57. else
  58. {
  59. if (m_bytesBuffered > 0)
  60. {
  61. AttachedTransformation()->PutModifiable(m_outputBuffer, m_bytesBuffered);
  62. m_bytesBuffered = 0;
  63. }
  64. if (m_bitsBuffered > 0)
  65. {
  66. AttachedTransformation()->Put((byte)m_buffer);
  67. m_buffer = 0;
  68. m_bitsBuffered = 0;
  69. }
  70. }
  71. }
  72. void LowFirstBitWriter::ClearBitBuffer()
  73. {
  74. m_buffer = 0;
  75. m_bytesBuffered = 0;
  76. m_bitsBuffered = 0;
  77. }
  78. HuffmanEncoder::HuffmanEncoder(const unsigned int *codeBits, unsigned int nCodes)
  79. {
  80. Initialize(codeBits, nCodes);
  81. }
  82. struct HuffmanNode
  83. {
  84. size_t symbol;
  85. union {size_t parent; unsigned depth, freq;};
  86. };
  87. struct FreqLessThan
  88. {
  89. inline bool operator()(unsigned int lhs, const HuffmanNode &rhs) {return lhs < rhs.freq;}
  90. inline bool operator()(const HuffmanNode &lhs, const HuffmanNode &rhs) const {return lhs.freq < rhs.freq;}
  91. // needed for MSVC .NET 2005
  92. inline bool operator()(const HuffmanNode &lhs, unsigned int rhs) {return lhs.freq < rhs;}
  93. };
  94. void HuffmanEncoder::GenerateCodeLengths(unsigned int *codeBits, unsigned int maxCodeBits, const unsigned int *codeCounts, size_t nCodes)
  95. {
  96. assert(nCodes > 0);
  97. assert(nCodes <= ((size_t)1 << maxCodeBits));
  98. size_t i;
  99. SecBlockWithHint<HuffmanNode, 2*286> tree(nCodes);
  100. for (i=0; i<nCodes; i++)
  101. {
  102. tree[i].symbol = i;
  103. tree[i].freq = codeCounts[i];
  104. }
  105. sort(tree.begin(), tree.end(), FreqLessThan());
  106. size_t treeBegin = upper_bound(tree.begin(), tree.end(), 0, FreqLessThan()) - tree.begin();
  107. if (treeBegin == nCodes)
  108. { // special case for no codes
  109. fill(codeBits, codeBits+nCodes, 0);
  110. return;
  111. }
  112. tree.resize(nCodes + nCodes - treeBegin - 1);
  113. size_t leastLeaf = treeBegin, leastInterior = nCodes;
  114. for (i=nCodes; i<tree.size(); i++)
  115. {
  116. size_t least;
  117. least = (leastLeaf == nCodes || (leastInterior < i && tree[leastInterior].freq < tree[leastLeaf].freq)) ? leastInterior++ : leastLeaf++;
  118. tree[i].freq = tree[least].freq;
  119. tree[least].parent = i;
  120. least = (leastLeaf == nCodes || (leastInterior < i && tree[leastInterior].freq < tree[leastLeaf].freq)) ? leastInterior++ : leastLeaf++;
  121. tree[i].freq += tree[least].freq;
  122. tree[least].parent = i;
  123. }
  124. tree[tree.size()-1].depth = 0;
  125. if (tree.size() >= 2)
  126. for (i=tree.size()-2; i>=nCodes; i--)
  127. tree[i].depth = tree[tree[i].parent].depth + 1;
  128. unsigned int sum = 0;
  129. SecBlockWithHint<unsigned int, 15+1> blCount(maxCodeBits+1);
  130. fill(blCount.begin(), blCount.end(), 0);
  131. for (i=treeBegin; i<nCodes; i++)
  132. {
  133. size_t depth = STDMIN(maxCodeBits, tree[tree[i].parent].depth + 1);
  134. blCount[depth]++;
  135. sum += 1 << (maxCodeBits - depth);
  136. }
  137. unsigned int overflow = sum > (unsigned int)(1 << maxCodeBits) ? sum - (1 << maxCodeBits) : 0;
  138. while (overflow--)
  139. {
  140. unsigned int bits = maxCodeBits-1;
  141. while (blCount[bits] == 0)
  142. bits--;
  143. blCount[bits]--;
  144. blCount[bits+1] += 2;
  145. assert(blCount[maxCodeBits] > 0);
  146. blCount[maxCodeBits]--;
  147. }
  148. for (i=0; i<treeBegin; i++)
  149. codeBits[tree[i].symbol] = 0;
  150. unsigned int bits = maxCodeBits;
  151. for (i=treeBegin; i<nCodes; i++)
  152. {
  153. while (blCount[bits] == 0)
  154. bits--;
  155. codeBits[tree[i].symbol] = bits;
  156. blCount[bits]--;
  157. }
  158. assert(blCount[bits] == 0);
  159. }
  160. void HuffmanEncoder::Initialize(const unsigned int *codeBits, unsigned int nCodes)
  161. {
  162. assert(nCodes > 0);
  163. unsigned int maxCodeBits = *max_element(codeBits, codeBits+nCodes);
  164. if (maxCodeBits == 0)
  165. return; // assume this object won't be used
  166. SecBlockWithHint<unsigned int, 15+1> blCount(maxCodeBits+1);
  167. fill(blCount.begin(), blCount.end(), 0);
  168. unsigned int i;
  169. for (i=0; i<nCodes; i++)
  170. blCount[codeBits[i]]++;
  171. code_t code = 0;
  172. SecBlockWithHint<code_t, 15+1> nextCode(maxCodeBits+1);
  173. nextCode[1] = 0;
  174. for (i=2; i<=maxCodeBits; i++)
  175. {
  176. code = (code + blCount[i-1]) << 1;
  177. nextCode[i] = code;
  178. }
  179. assert(maxCodeBits == 1 || code == (1 << maxCodeBits) - blCount[maxCodeBits]);
  180. m_valueToCode.resize(nCodes);
  181. for (i=0; i<nCodes; i++)
  182. {
  183. unsigned int len = m_valueToCode[i].len = codeBits[i];
  184. if (len != 0)
  185. m_valueToCode[i].code = BitReverse(nextCode[len]++) >> (8*sizeof(code_t)-len);
  186. }
  187. }
  188. inline void HuffmanEncoder::Encode(LowFirstBitWriter &writer, value_t value) const
  189. {
  190. assert(m_valueToCode[value].len > 0);
  191. writer.PutBits(m_valueToCode[value].code, m_valueToCode[value].len);
  192. }
  193. Deflator::Deflator(BufferedTransformation *attachment, int deflateLevel, int log2WindowSize, bool detectUncompressible)
  194. : LowFirstBitWriter(attachment)
  195. , m_deflateLevel(-1)
  196. {
  197. InitializeStaticEncoders();
  198. IsolatedInitialize(MakeParameters("DeflateLevel", deflateLevel)("Log2WindowSize", log2WindowSize)("DetectUncompressible", detectUncompressible));
  199. }
  200. Deflator::Deflator(const NameValuePairs &parameters, BufferedTransformation *attachment)
  201. : LowFirstBitWriter(attachment)
  202. , m_deflateLevel(-1)
  203. {
  204. InitializeStaticEncoders();
  205. IsolatedInitialize(parameters);
  206. }
  207. void Deflator::InitializeStaticEncoders()
  208. {
  209. unsigned int codeLengths[288];
  210. fill(codeLengths + 0, codeLengths + 144, 8);
  211. fill(codeLengths + 144, codeLengths + 256, 9);
  212. fill(codeLengths + 256, codeLengths + 280, 7);
  213. fill(codeLengths + 280, codeLengths + 288, 8);
  214. m_staticLiteralEncoder.Initialize(codeLengths, 288);
  215. fill(codeLengths + 0, codeLengths + 32, 5);
  216. m_staticDistanceEncoder.Initialize(codeLengths, 32);
  217. }
  218. void Deflator::IsolatedInitialize(const NameValuePairs &parameters)
  219. {
  220. int log2WindowSize = parameters.GetIntValueWithDefault("Log2WindowSize", DEFAULT_LOG2_WINDOW_SIZE);
  221. if (!(MIN_LOG2_WINDOW_SIZE <= log2WindowSize && log2WindowSize <= MAX_LOG2_WINDOW_SIZE))
  222. throw InvalidArgument("Deflator: " + IntToString(log2WindowSize) + " is an invalid window size");
  223. m_log2WindowSize = log2WindowSize;
  224. DSIZE = 1 << m_log2WindowSize;
  225. DMASK = DSIZE - 1;
  226. HSIZE = 1 << m_log2WindowSize;
  227. HMASK = HSIZE - 1;
  228. m_byteBuffer.New(2*DSIZE);
  229. m_head.New(HSIZE);
  230. m_prev.New(DSIZE);
  231. m_matchBuffer.New(DSIZE/2);
  232. Reset(true);
  233. SetDeflateLevel(parameters.GetIntValueWithDefault("DeflateLevel", DEFAULT_DEFLATE_LEVEL));
  234. bool detectUncompressible = parameters.GetValueWithDefault("DetectUncompressible", true);
  235. m_compressibleDeflateLevel = detectUncompressible ? m_deflateLevel : 0;
  236. }
  237. void Deflator::Reset(bool forceReset)
  238. {
  239. if (forceReset)
  240. ClearBitBuffer();
  241. else
  242. assert(m_bitsBuffered == 0);
  243. m_headerWritten = false;
  244. m_matchAvailable = false;
  245. m_dictionaryEnd = 0;
  246. m_stringStart = 0;
  247. m_lookahead = 0;
  248. m_minLookahead = MAX_MATCH;
  249. m_matchBufferEnd = 0;
  250. m_blockStart = 0;
  251. m_blockLength = 0;
  252. m_detectCount = 1;
  253. m_detectSkip = 0;
  254. // m_prev will be initialized automaticly in InsertString
  255. fill(m_head.begin(), m_head.end(), 0);
  256. fill(m_literalCounts.begin(), m_literalCounts.end(), 0);
  257. fill(m_distanceCounts.begin(), m_distanceCounts.end(), 0);
  258. }
  259. void Deflator::SetDeflateLevel(int deflateLevel)
  260. {
  261. if (!(MIN_DEFLATE_LEVEL <= deflateLevel && deflateLevel <= MAX_DEFLATE_LEVEL))
  262. throw InvalidArgument("Deflator: " + IntToString(deflateLevel) + " is an invalid deflate level");
  263. if (deflateLevel == m_deflateLevel)
  264. return;
  265. EndBlock(false);
  266. static const unsigned int configurationTable[10][4] = {
  267. /* good lazy nice chain */
  268. /* 0 */ {0, 0, 0, 0}, /* store only */
  269. /* 1 */ {4, 3, 8, 4}, /* maximum speed, no lazy matches */
  270. /* 2 */ {4, 3, 16, 8},
  271. /* 3 */ {4, 3, 32, 32},
  272. /* 4 */ {4, 4, 16, 16}, /* lazy matches */
  273. /* 5 */ {8, 16, 32, 32},
  274. /* 6 */ {8, 16, 128, 128},
  275. /* 7 */ {8, 32, 128, 256},
  276. /* 8 */ {32, 128, 258, 1024},
  277. /* 9 */ {32, 258, 258, 4096}}; /* maximum compression */
  278. GOOD_MATCH = configurationTable[deflateLevel][0];
  279. MAX_LAZYLENGTH = configurationTable[deflateLevel][1];
  280. MAX_CHAIN_LENGTH = configurationTable[deflateLevel][3];
  281. m_deflateLevel = deflateLevel;
  282. }
  283. unsigned int Deflator::FillWindow(const byte *str, size_t length)
  284. {
  285. unsigned int maxBlockSize = (unsigned int)STDMIN(2UL*DSIZE, 0xffffUL);
  286. if (m_stringStart >= maxBlockSize - MAX_MATCH)
  287. {
  288. if (m_blockStart < DSIZE)
  289. EndBlock(false);
  290. memcpy(m_byteBuffer, m_byteBuffer + DSIZE, DSIZE);
  291. m_dictionaryEnd = m_dictionaryEnd < DSIZE ? 0 : m_dictionaryEnd-DSIZE;
  292. assert(m_stringStart >= DSIZE);
  293. m_stringStart -= DSIZE;
  294. assert(!m_matchAvailable || m_previousMatch >= DSIZE);
  295. m_previousMatch -= DSIZE;
  296. assert(m_blockStart >= DSIZE);
  297. m_blockStart -= DSIZE;
  298. unsigned int i;
  299. for (i=0; i<HSIZE; i++)
  300. m_head[i] = SaturatingSubtract(m_head[i], DSIZE);
  301. for (i=0; i<DSIZE; i++)
  302. m_prev[i] = SaturatingSubtract(m_prev[i], DSIZE);
  303. }
  304. assert(maxBlockSize > m_stringStart+m_lookahead);
  305. unsigned int accepted = UnsignedMin(maxBlockSize-(m_stringStart+m_lookahead), length);
  306. assert(accepted > 0);
  307. memcpy(m_byteBuffer + m_stringStart + m_lookahead, str, accepted);
  308. m_lookahead += accepted;
  309. return accepted;
  310. }
  311. inline unsigned int Deflator::ComputeHash(const byte *str) const
  312. {
  313. assert(str+3 <= m_byteBuffer + m_stringStart + m_lookahead);
  314. return ((str[0] << 10) ^ (str[1] << 5) ^ str[2]) & HMASK;
  315. }
  316. unsigned int Deflator::LongestMatch(unsigned int &bestMatch) const
  317. {
  318. assert(m_previousLength < MAX_MATCH);
  319. bestMatch = 0;
  320. unsigned int bestLength = STDMAX(m_previousLength, (unsigned int)MIN_MATCH-1);
  321. if (m_lookahead <= bestLength)
  322. return 0;
  323. const byte *scan = m_byteBuffer + m_stringStart, *scanEnd = scan + STDMIN((unsigned int)MAX_MATCH, m_lookahead);
  324. unsigned int limit = m_stringStart > (DSIZE-MAX_MATCH) ? m_stringStart - (DSIZE-MAX_MATCH) : 0;
  325. unsigned int current = m_head[ComputeHash(scan)];
  326. unsigned int chainLength = MAX_CHAIN_LENGTH;
  327. if (m_previousLength >= GOOD_MATCH)
  328. chainLength >>= 2;
  329. while (current > limit && --chainLength > 0)
  330. {
  331. const byte *match = m_byteBuffer + current;
  332. assert(scan + bestLength < m_byteBuffer + m_stringStart + m_lookahead);
  333. if (scan[bestLength-1] == match[bestLength-1] && scan[bestLength] == match[bestLength] && scan[0] == match[0] && scan[1] == match[1])
  334. {
  335. assert(scan[2] == match[2]);
  336. unsigned int len = (unsigned int)(
  337. #if defined(_STDEXT_BEGIN) && !(defined(_MSC_VER) && (_MSC_VER < 1400 || _MSC_VER >= 1600)) && !defined(_STLPORT_VERSION)
  338. stdext::unchecked_mismatch
  339. #else
  340. std::mismatch
  341. #endif
  342. #if _MSC_VER >= 1600
  343. (stdext::make_unchecked_array_iterator(scan)+3, stdext::make_unchecked_array_iterator(scanEnd), stdext::make_unchecked_array_iterator(match)+3).first - stdext::make_unchecked_array_iterator(scan));
  344. #else
  345. (scan+3, scanEnd, match+3).first - scan);
  346. #endif
  347. assert(len != bestLength);
  348. if (len > bestLength)
  349. {
  350. bestLength = len;
  351. bestMatch = current;
  352. if (len == (scanEnd - scan))
  353. break;
  354. }
  355. }
  356. current = m_prev[current & DMASK];
  357. }
  358. return (bestMatch > 0) ? bestLength : 0;
  359. }
  360. inline void Deflator::InsertString(unsigned int start)
  361. {
  362. unsigned int hash = ComputeHash(m_byteBuffer + start);
  363. m_prev[start & DMASK] = m_head[hash];
  364. m_head[hash] = start;
  365. }
  366. void Deflator::ProcessBuffer()
  367. {
  368. if (!m_headerWritten)
  369. {
  370. WritePrestreamHeader();
  371. m_headerWritten = true;
  372. }
  373. if (m_deflateLevel == 0)
  374. {
  375. m_stringStart += m_lookahead;
  376. m_lookahead = 0;
  377. m_blockLength = m_stringStart - m_blockStart;
  378. m_matchAvailable = false;
  379. return;
  380. }
  381. while (m_lookahead > m_minLookahead)
  382. {
  383. while (m_dictionaryEnd < m_stringStart && m_dictionaryEnd+3 <= m_stringStart+m_lookahead)
  384. InsertString(m_dictionaryEnd++);
  385. if (m_matchAvailable)
  386. {
  387. unsigned int matchPosition, matchLength;
  388. bool usePreviousMatch;
  389. if (m_previousLength >= MAX_LAZYLENGTH)
  390. usePreviousMatch = true;
  391. else
  392. {
  393. matchLength = LongestMatch(matchPosition);
  394. usePreviousMatch = (matchLength == 0);
  395. }
  396. if (usePreviousMatch)
  397. {
  398. MatchFound(m_stringStart-1-m_previousMatch, m_previousLength);
  399. m_stringStart += m_previousLength-1;
  400. m_lookahead -= m_previousLength-1;
  401. m_matchAvailable = false;
  402. }
  403. else
  404. {
  405. m_previousLength = matchLength;
  406. m_previousMatch = matchPosition;
  407. LiteralByte(m_byteBuffer[m_stringStart-1]);
  408. m_stringStart++;
  409. m_lookahead--;
  410. }
  411. }
  412. else
  413. {
  414. m_previousLength = 0;
  415. m_previousLength = LongestMatch(m_previousMatch);
  416. if (m_previousLength)
  417. m_matchAvailable = true;
  418. else
  419. LiteralByte(m_byteBuffer[m_stringStart]);
  420. m_stringStart++;
  421. m_lookahead--;
  422. }
  423. assert(m_stringStart - (m_blockStart+m_blockLength) == (unsigned int)m_matchAvailable);
  424. }
  425. if (m_minLookahead == 0 && m_matchAvailable)
  426. {
  427. LiteralByte(m_byteBuffer[m_stringStart-1]);
  428. m_matchAvailable = false;
  429. }
  430. }
  431. size_t Deflator::Put2(const byte *str, size_t length, int messageEnd, bool blocking)
  432. {
  433. if (!blocking)
  434. throw BlockingInputOnly("Deflator");
  435. size_t accepted = 0;
  436. while (accepted < length)
  437. {
  438. unsigned int newAccepted = FillWindow(str+accepted, length-accepted);
  439. ProcessBuffer();
  440. // call ProcessUncompressedData() after WritePrestreamHeader()
  441. ProcessUncompressedData(str+accepted, newAccepted);
  442. accepted += newAccepted;
  443. }
  444. assert(accepted == length);
  445. if (messageEnd)
  446. {
  447. m_minLookahead = 0;
  448. ProcessBuffer();
  449. EndBlock(true);
  450. FlushBitBuffer();
  451. WritePoststreamTail();
  452. Reset();
  453. }
  454. Output(0, NULL, 0, messageEnd, blocking);
  455. return 0;
  456. }
  457. bool Deflator::IsolatedFlush(bool hardFlush, bool blocking)
  458. {
  459. if (!blocking)
  460. throw BlockingInputOnly("Deflator");
  461. m_minLookahead = 0;
  462. ProcessBuffer();
  463. m_minLookahead = MAX_MATCH;
  464. EndBlock(false);
  465. if (hardFlush)
  466. EncodeBlock(false, STORED);
  467. return false;
  468. }
  469. void Deflator::LiteralByte(byte b)
  470. {
  471. if (m_matchBufferEnd == m_matchBuffer.size())
  472. EndBlock(false);
  473. m_matchBuffer[m_matchBufferEnd++].literalCode = b;
  474. m_literalCounts[b]++;
  475. m_blockLength++;
  476. }
  477. void Deflator::MatchFound(unsigned int distance, unsigned int length)
  478. {
  479. if (m_matchBufferEnd == m_matchBuffer.size())
  480. EndBlock(false);
  481. static const unsigned int lengthCodes[] = {
  482. 257, 258, 259, 260, 261, 262, 263, 264, 265, 265, 266, 266, 267, 267, 268, 268,
  483. 269, 269, 269, 269, 270, 270, 270, 270, 271, 271, 271, 271, 272, 272, 272, 272,
  484. 273, 273, 273, 273, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274,
  485. 275, 275, 275, 275, 275, 275, 275, 275, 276, 276, 276, 276, 276, 276, 276, 276,
  486. 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, 277,
  487. 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278,
  488. 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279,
  489. 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280,
  490. 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281,
  491. 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281,
  492. 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282,
  493. 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282, 282,
  494. 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283,
  495. 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, 283,
  496. 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284,
  497. 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, 285};
  498. static const unsigned int lengthBases[] = {3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258};
  499. static const unsigned int distanceBases[30] =
  500. {1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577};
  501. EncodedMatch &m = m_matchBuffer[m_matchBufferEnd++];
  502. assert(length >= 3);
  503. unsigned int lengthCode = lengthCodes[length-3];
  504. m.literalCode = lengthCode;
  505. m.literalExtra = length - lengthBases[lengthCode-257];
  506. unsigned int distanceCode = (unsigned int)(upper_bound(distanceBases, distanceBases+30, distance) - distanceBases - 1);
  507. m.distanceCode = distanceCode;
  508. m.distanceExtra = distance - distanceBases[distanceCode];
  509. m_literalCounts[lengthCode]++;
  510. m_distanceCounts[distanceCode]++;
  511. m_blockLength += length;
  512. }
  513. inline unsigned int CodeLengthEncode(const unsigned int *begin,
  514. const unsigned int *end,
  515. const unsigned int *& p,
  516. unsigned int &extraBits,
  517. unsigned int &extraBitsLength)
  518. {
  519. unsigned int v = *p;
  520. if ((end-p) >= 3)
  521. {
  522. const unsigned int *oldp = p;
  523. if (v==0 && p[1]==0 && p[2]==0)
  524. {
  525. for (p=p+3; p!=end && *p==0 && p!=oldp+138; p++) {}
  526. unsigned int repeat = (unsigned int)(p - oldp);
  527. if (repeat <= 10)
  528. {
  529. extraBits = repeat-3;
  530. extraBitsLength = 3;
  531. return 17;
  532. }
  533. else
  534. {
  535. extraBits = repeat-11;
  536. extraBitsLength = 7;
  537. return 18;
  538. }
  539. }
  540. else if (p!=begin && v==p[-1] && v==p[1] && v==p[2])
  541. {
  542. for (p=p+3; p!=end && *p==v && p!=oldp+6; p++) {}
  543. unsigned int repeat = (unsigned int)(p - oldp);
  544. extraBits = repeat-3;
  545. extraBitsLength = 2;
  546. return 16;
  547. }
  548. }
  549. p++;
  550. extraBits = 0;
  551. extraBitsLength = 0;
  552. return v;
  553. }
  554. void Deflator::EncodeBlock(bool eof, unsigned int blockType)
  555. {
  556. PutBits(eof, 1);
  557. PutBits(blockType, 2);
  558. if (blockType == STORED)
  559. {
  560. assert(m_blockStart + m_blockLength <= m_byteBuffer.size());
  561. assert(m_blockLength <= 0xffff);
  562. FlushBitBuffer();
  563. AttachedTransformation()->PutWord16(m_blockLength, LITTLE_ENDIAN_ORDER);
  564. AttachedTransformation()->PutWord16(~m_blockLength, LITTLE_ENDIAN_ORDER);
  565. AttachedTransformation()->Put(m_byteBuffer + m_blockStart, m_blockLength);
  566. }
  567. else
  568. {
  569. if (blockType == DYNAMIC)
  570. {
  571. #if defined(_MSC_VER) && !defined(__MWERKS__) && (_MSC_VER <= 1300)
  572. // VC60 and VC7 workaround: built-in reverse_iterator has two template parameters, Dinkumware only has one
  573. typedef reverse_bidirectional_iterator<unsigned int *, unsigned int> RevIt;
  574. #elif defined(_RWSTD_NO_CLASS_PARTIAL_SPEC)
  575. typedef reverse_iterator<unsigned int *, random_access_iterator_tag, unsigned int> RevIt;
  576. #else
  577. typedef reverse_iterator<unsigned int *> RevIt;
  578. #endif
  579. FixedSizeSecBlock<unsigned int, 286> literalCodeLengths;
  580. FixedSizeSecBlock<unsigned int, 30> distanceCodeLengths;
  581. m_literalCounts[256] = 1;
  582. HuffmanEncoder::GenerateCodeLengths(literalCodeLengths, 15, m_literalCounts, 286);
  583. m_dynamicLiteralEncoder.Initialize(literalCodeLengths, 286);
  584. unsigned int hlit = (unsigned int)(find_if(RevIt(literalCodeLengths.end()), RevIt(literalCodeLengths.begin()+257), bind2nd(not_equal_to<unsigned int>(), 0)).base() - (literalCodeLengths.begin()+257));
  585. HuffmanEncoder::GenerateCodeLengths(distanceCodeLengths, 15, m_distanceCounts, 30);
  586. m_dynamicDistanceEncoder.Initialize(distanceCodeLengths, 30);
  587. unsigned int hdist = (unsigned int)(find_if(RevIt(distanceCodeLengths.end()), RevIt(distanceCodeLengths.begin()+1), bind2nd(not_equal_to<unsigned int>(), 0)).base() - (distanceCodeLengths.begin()+1));
  588. SecBlockWithHint<unsigned int, 286+30> combinedLengths(hlit+257+hdist+1);
  589. memcpy(combinedLengths, literalCodeLengths, (hlit+257)*sizeof(unsigned int));
  590. memcpy(combinedLengths+hlit+257, distanceCodeLengths, (hdist+1)*sizeof(unsigned int));
  591. FixedSizeSecBlock<unsigned int, 19> codeLengthCodeCounts, codeLengthCodeLengths;
  592. fill(codeLengthCodeCounts.begin(), codeLengthCodeCounts.end(), 0);
  593. const unsigned int *p = combinedLengths.begin(), *begin = combinedLengths.begin(), *end = combinedLengths.end();
  594. while (p != end)
  595. {
  596. unsigned int code, extraBits, extraBitsLength;
  597. code = CodeLengthEncode(begin, end, p, extraBits, extraBitsLength);
  598. codeLengthCodeCounts[code]++;
  599. }
  600. HuffmanEncoder::GenerateCodeLengths(codeLengthCodeLengths, 7, codeLengthCodeCounts, 19);
  601. HuffmanEncoder codeLengthEncoder(codeLengthCodeLengths, 19);
  602. static const unsigned int border[] = { // Order of the bit length code lengths
  603. 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
  604. unsigned int hclen = 19;
  605. while (hclen > 4 && codeLengthCodeLengths[border[hclen-1]] == 0)
  606. hclen--;
  607. hclen -= 4;
  608. PutBits(hlit, 5);
  609. PutBits(hdist, 5);
  610. PutBits(hclen, 4);
  611. for (unsigned int i=0; i<hclen+4; i++)
  612. PutBits(codeLengthCodeLengths[border[i]], 3);
  613. p = combinedLengths.begin();
  614. while (p != end)
  615. {
  616. unsigned int code, extraBits, extraBitsLength;
  617. code = CodeLengthEncode(begin, end, p, extraBits, extraBitsLength);
  618. codeLengthEncoder.Encode(*this, code);
  619. PutBits(extraBits, extraBitsLength);
  620. }
  621. }
  622. static const unsigned int lengthExtraBits[] = {
  623. 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2,
  624. 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0};
  625. static const unsigned int distanceExtraBits[] = {
  626. 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6,
  627. 7, 7, 8, 8, 9, 9, 10, 10, 11, 11,
  628. 12, 12, 13, 13};
  629. const HuffmanEncoder &literalEncoder = (blockType == STATIC) ? m_staticLiteralEncoder : m_dynamicLiteralEncoder;
  630. const HuffmanEncoder &distanceEncoder = (blockType == STATIC) ? m_staticDistanceEncoder : m_dynamicDistanceEncoder;
  631. for (unsigned int i=0; i<m_matchBufferEnd; i++)
  632. {
  633. unsigned int literalCode = m_matchBuffer[i].literalCode;
  634. literalEncoder.Encode(*this, literalCode);
  635. if (literalCode >= 257)
  636. {
  637. assert(literalCode <= 285);
  638. PutBits(m_matchBuffer[i].literalExtra, lengthExtraBits[literalCode-257]);
  639. unsigned int distanceCode = m_matchBuffer[i].distanceCode;
  640. distanceEncoder.Encode(*this, distanceCode);
  641. PutBits(m_matchBuffer[i].distanceExtra, distanceExtraBits[distanceCode]);
  642. }
  643. }
  644. literalEncoder.Encode(*this, 256); // end of block
  645. }
  646. }
  647. void Deflator::EndBlock(bool eof)
  648. {
  649. if (m_blockLength == 0 && !eof)
  650. return;
  651. if (m_deflateLevel == 0)
  652. {
  653. EncodeBlock(eof, STORED);
  654. if (m_compressibleDeflateLevel > 0 && ++m_detectCount == m_detectSkip)
  655. {
  656. m_deflateLevel = m_compressibleDeflateLevel;
  657. m_detectCount = 1;
  658. }
  659. }
  660. else
  661. {
  662. unsigned long storedLen = 8*((unsigned long)m_blockLength+4) + RoundUpToMultipleOf(m_bitsBuffered+3, 8U)-m_bitsBuffered;
  663. StartCounting();
  664. EncodeBlock(eof, STATIC);
  665. unsigned long staticLen = FinishCounting();
  666. unsigned long dynamicLen;
  667. if (m_blockLength < 128 && m_deflateLevel < 8)
  668. dynamicLen = ULONG_MAX;
  669. else
  670. {
  671. StartCounting();
  672. EncodeBlock(eof, DYNAMIC);
  673. dynamicLen = FinishCounting();
  674. }
  675. if (storedLen <= staticLen && storedLen <= dynamicLen)
  676. {
  677. EncodeBlock(eof, STORED);
  678. if (m_compressibleDeflateLevel > 0)
  679. {
  680. if (m_detectSkip)
  681. m_deflateLevel = 0;
  682. m_detectSkip = m_detectSkip ? STDMIN(2*m_detectSkip, 128U) : 1;
  683. }
  684. }
  685. else
  686. {
  687. if (staticLen <= dynamicLen)
  688. EncodeBlock(eof, STATIC);
  689. else
  690. EncodeBlock(eof, DYNAMIC);
  691. if (m_compressibleDeflateLevel > 0)
  692. m_detectSkip = 0;
  693. }
  694. }
  695. m_matchBufferEnd = 0;
  696. m_blockStart += m_blockLength;
  697. m_blockLength = 0;
  698. fill(m_literalCounts.begin(), m_literalCounts.end(), 0);
  699. fill(m_distanceCounts.begin(), m_distanceCounts.end(), 0);
  700. }
  701. NAMESPACE_END