initial commit, 4.5 stable

core/io/compression.cpp

@@ -0,0 +1,371 @@
+/**************************************************************************/
+/*  compression.cpp                                                       */
+/**************************************************************************/
+/*                         This file is part of:                          */
+/*                             GODOT ENGINE                               */
+/*                        https://godotengine.org                         */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                  */
+/*                                                                        */
+/* Permission is hereby granted, free of charge, to any person obtaining  */
+/* a copy of this software and associated documentation files (the        */
+/* "Software"), to deal in the Software without restriction, including    */
+/* without limitation the rights to use, copy, modify, merge, publish,    */
+/* distribute, sublicense, and/or sell copies of the Software, and to     */
+/* permit persons to whom the Software is furnished to do so, subject to  */
+/* the following conditions:                                              */
+/*                                                                        */
+/* The above copyright notice and this permission notice shall be         */
+/* included in all copies or substantial portions of the Software.        */
+/*                                                                        */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,        */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF     */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY   */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,   */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE      */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
+/**************************************************************************/
+
+#include "compression.h"
+
+#include "core/config/project_settings.h"
+#include "core/io/zip_io.h"
+
+#include "thirdparty/misc/fastlz.h"
+
+#include <zstd.h>
+
+#ifdef BROTLI_ENABLED
+#include <brotli/decode.h>
+#endif
+
+// Caches for zstd.
+static BinaryMutex mutex;
+static ZSTD_DCtx *current_zstd_d_ctx = nullptr;
+static bool current_zstd_long_distance_matching;
+static int current_zstd_window_log_size;
+
+int64_t Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
+	switch (p_mode) {
+		case MODE_BROTLI: {
+			ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
+		} break;
+		case MODE_FASTLZ: {
+			ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
+			if (p_src_size < 16) {
+				uint8_t src[16];
+				memset(&src[p_src_size], 0, 16 - p_src_size);
+				memcpy(src, p_src, p_src_size);
+				return fastlz_compress(src, 16, p_dst);
+			} else {
+				return fastlz_compress(p_src, p_src_size, p_dst);
+			}
+
+		} break;
+		case MODE_DEFLATE:
+		case MODE_GZIP: {
+			ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
+			int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
+
+			z_stream strm;
+			strm.zalloc = zipio_alloc;
+			strm.zfree = zipio_free;
+			strm.opaque = Z_NULL;
+			int level = p_mode == MODE_DEFLATE ? zlib_level : gzip_level;
+			int err = deflateInit2(&strm, level, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY);
+			if (err != Z_OK) {
+				return -1;
+			}
+
+			strm.avail_in = p_src_size;
+			int aout = deflateBound(&strm, p_src_size);
+			strm.avail_out = aout;
+			strm.next_in = (Bytef *)p_src;
+			strm.next_out = p_dst;
+			deflate(&strm, Z_FINISH);
+			aout = aout - strm.avail_out;
+			deflateEnd(&strm);
+			return aout;
+
+		} break;
+		case MODE_ZSTD: {
+			ZSTD_CCtx *cctx = ZSTD_createCCtx();
+			ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, zstd_level);
+			if (zstd_long_distance_matching) {
+				ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1);
+				ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, zstd_window_log_size);
+			}
+			const int64_t max_dst_size = get_max_compressed_buffer_size(p_src_size, MODE_ZSTD);
+			const size_t ret = ZSTD_compressCCtx(cctx, p_dst, max_dst_size, p_src, p_src_size, zstd_level);
+			ZSTD_freeCCtx(cctx);
+			return (int64_t)ret;
+		} break;
+	}
+
+	ERR_FAIL_V(-1);
+}
+
+int64_t Compression::get_max_compressed_buffer_size(int64_t p_src_size, Mode p_mode) {
+	switch (p_mode) {
+		case MODE_BROTLI: {
+			ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
+		} break;
+		case MODE_FASTLZ: {
+			ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
+			int ss = p_src_size + p_src_size * 6 / 100;
+			if (ss < 66) {
+				ss = 66;
+			}
+			return ss;
+
+		} break;
+		case MODE_DEFLATE:
+		case MODE_GZIP: {
+			ERR_FAIL_COND_V_MSG(p_src_size > INT32_MAX, -1, "Cannot compress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
+			int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
+
+			z_stream strm;
+			strm.zalloc = zipio_alloc;
+			strm.zfree = zipio_free;
+			strm.opaque = Z_NULL;
+			int err = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY);
+			if (err != Z_OK) {
+				return -1;
+			}
+			int aout = deflateBound(&strm, p_src_size);
+			deflateEnd(&strm);
+			return aout;
+		} break;
+		case MODE_ZSTD: {
+			return ZSTD_compressBound(p_src_size);
+		} break;
+	}
+
+	ERR_FAIL_V(-1);
+}
+
+int64_t Compression::decompress(uint8_t *p_dst, int64_t p_dst_max_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
+	switch (p_mode) {
+		case MODE_BROTLI: {
+#ifdef BROTLI_ENABLED
+			size_t ret_size = p_dst_max_size;
+			BrotliDecoderResult res = BrotliDecoderDecompress(p_src_size, p_src, &ret_size, p_dst);
+			ERR_FAIL_COND_V(res != BROTLI_DECODER_RESULT_SUCCESS, -1);
+			return ret_size;
+#else
+			ERR_FAIL_V_MSG(-1, "Godot was compiled without brotli support.");
+#endif
+		} break;
+		case MODE_FASTLZ: {
+			ERR_FAIL_COND_V_MSG(p_dst_max_size > INT32_MAX, -1, "Cannot decompress a FastLZ/LZ77 file 2 GiB or larger. LZ77 supports larger files, but FastLZ's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
+			int ret_size = 0;
+
+			if (p_dst_max_size < 16) {
+				uint8_t dst[16];
+				fastlz_decompress(p_src, p_src_size, dst, 16);
+				memcpy(p_dst, dst, p_dst_max_size);
+				ret_size = p_dst_max_size;
+			} else {
+				ret_size = fastlz_decompress(p_src, p_src_size, p_dst, p_dst_max_size);
+			}
+			return ret_size;
+		} break;
+		case MODE_DEFLATE:
+		case MODE_GZIP: {
+			ERR_FAIL_COND_V_MSG(p_dst_max_size > INT32_MAX, -1, "Cannot decompress a Deflate or GZip file 2 GiB or larger. Deflate and GZip are both limited to 4 GiB, and ZLib's implementation uses C++ `int` so is limited to 2 GiB. Consider using Zstd instead.");
+			int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
+
+			z_stream strm;
+			strm.zalloc = zipio_alloc;
+			strm.zfree = zipio_free;
+			strm.opaque = Z_NULL;
+			strm.avail_in = 0;
+			strm.next_in = Z_NULL;
+			int err = inflateInit2(&strm, window_bits);
+			ERR_FAIL_COND_V(err != Z_OK, -1);
+
+			strm.avail_in = p_src_size;
+			strm.avail_out = p_dst_max_size;
+			strm.next_in = (Bytef *)p_src;
+			strm.next_out = p_dst;
+
+			err = inflate(&strm, Z_FINISH);
+			int total = strm.total_out;
+			inflateEnd(&strm);
+			ERR_FAIL_COND_V(err != Z_STREAM_END, -1);
+			return total;
+		} break;
+		case MODE_ZSTD: {
+			MutexLock lock(mutex);
+
+			if (!current_zstd_d_ctx || current_zstd_long_distance_matching != zstd_long_distance_matching || current_zstd_window_log_size != zstd_window_log_size) {
+				if (current_zstd_d_ctx) {
+					ZSTD_freeDCtx(current_zstd_d_ctx);
+				}
+
+				current_zstd_d_ctx = ZSTD_createDCtx();
+				if (zstd_long_distance_matching) {
+					ZSTD_DCtx_setParameter(current_zstd_d_ctx, ZSTD_d_windowLogMax, zstd_window_log_size);
+				}
+				current_zstd_long_distance_matching = zstd_long_distance_matching;
+				current_zstd_window_log_size = zstd_window_log_size;
+			}
+
+			size_t ret = ZSTD_decompressDCtx(current_zstd_d_ctx, p_dst, p_dst_max_size, p_src, p_src_size);
+			return (int64_t)ret;
+		} break;
+	}
+
+	ERR_FAIL_V(-1);
+}
+
+/**
+	This will handle both Gzip and Deflate streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector.
+	This is required for compressed data whose final uncompressed size is unknown, as is the case for HTTP response bodies.
+	This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
+*/
+int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int64_t p_max_dst_size, const uint8_t *p_src, int64_t p_src_size, Mode p_mode) {
+	uint8_t *dst = nullptr;
+	int out_mark = 0;
+
+	ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
+
+	if (p_mode == MODE_BROTLI) {
+#ifdef BROTLI_ENABLED
+		BrotliDecoderResult ret;
+		BrotliDecoderState *state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
+		ERR_FAIL_NULL_V(state, Z_DATA_ERROR);
+
+		// Setup the stream inputs.
+		const uint8_t *next_in = p_src;
+		size_t avail_in = p_src_size;
+		uint8_t *next_out = nullptr;
+		size_t avail_out = 0;
+		size_t total_out = 0;
+
+		// Ensure the destination buffer is empty.
+		p_dst_vect->clear();
+
+		// Decompress until stream ends or end of file.
+		do {
+			// Add another chunk size to the output buffer.
+			// This forces a copy of the whole buffer.
+			p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
+			// Get pointer to the actual output buffer.
+			dst = p_dst_vect->ptrw();
+
+			// Set the stream to the new output stream.
+			// Since it was copied, we need to reset the stream to the new buffer.
+			next_out = &(dst[out_mark]);
+			avail_out += gzip_chunk;
+
+			ret = BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, &next_out, &total_out);
+			if (ret == BROTLI_DECODER_RESULT_ERROR) {
+				WARN_PRINT(BrotliDecoderErrorString(BrotliDecoderGetErrorCode(state)));
+				BrotliDecoderDestroyInstance(state);
+				p_dst_vect->clear();
+				return Z_DATA_ERROR;
+			}
+
+			out_mark += gzip_chunk - avail_out;
+
+			// Enforce max output size.
+			if (p_max_dst_size > -1 && total_out > (uint64_t)p_max_dst_size) {
+				BrotliDecoderDestroyInstance(state);
+				p_dst_vect->clear();
+				return Z_BUF_ERROR;
+			}
+		} while (ret != BROTLI_DECODER_RESULT_SUCCESS);
+
+		// If all done successfully, resize the output if it's larger than the actual output.
+		if ((unsigned long)p_dst_vect->size() > total_out) {
+			p_dst_vect->resize(total_out);
+		}
+
+		// Clean up and return.
+		BrotliDecoderDestroyInstance(state);
+		return Z_OK;
+#else
+		ERR_FAIL_V_MSG(Z_ERRNO, "Godot was compiled without brotli support.");
+#endif
+	} else {
+		// This function only supports GZip and Deflate.
+		ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
+
+		int ret;
+		z_stream strm;
+		int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
+
+		// Initialize the stream.
+		strm.zalloc = Z_NULL;
+		strm.zfree = Z_NULL;
+		strm.opaque = Z_NULL;
+		strm.avail_in = 0;
+		strm.next_in = Z_NULL;
+
+		int err = inflateInit2(&strm, window_bits);
+		ERR_FAIL_COND_V(err != Z_OK, -1);
+
+		// Setup the stream inputs.
+		strm.next_in = (Bytef *)p_src;
+		strm.avail_in = p_src_size;
+
+		// Ensure the destination buffer is empty.
+		p_dst_vect->clear();
+
+		// Decompress until deflate stream ends or end of file.
+		do {
+			// Add another chunk size to the output buffer.
+			// This forces a copy of the whole buffer.
+			p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
+			// Get pointer to the actual output buffer.
+			dst = p_dst_vect->ptrw();
+
+			// Set the stream to the new output stream.
+			// Since it was copied, we need to reset the stream to the new buffer.
+			strm.next_out = &(dst[out_mark]);
+			strm.avail_out = gzip_chunk;
+
+			// Run inflate() on input until output buffer is full and needs to be resized or input runs out.
+			do {
+				ret = inflate(&strm, Z_SYNC_FLUSH);
+
+				switch (ret) {
+					case Z_NEED_DICT:
+						ret = Z_DATA_ERROR;
+						[[fallthrough]];
+					case Z_DATA_ERROR:
+					case Z_MEM_ERROR:
+					case Z_STREAM_ERROR:
+					case Z_BUF_ERROR:
+						if (strm.msg) {
+							WARN_PRINT(strm.msg);
+						}
+						(void)inflateEnd(&strm);
+						p_dst_vect->clear();
+						return ret;
+				}
+			} while (strm.avail_out > 0 && strm.avail_in > 0);
+
+			out_mark += gzip_chunk;
+
+			// Enforce max output size.
+			if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
+				(void)inflateEnd(&strm);
+				p_dst_vect->clear();
+				return Z_BUF_ERROR;
+			}
+		} while (ret != Z_STREAM_END);
+
+		// If all done successfully, resize the output if it's larger than the actual output.
+		if ((unsigned long)p_dst_vect->size() > strm.total_out) {
+			p_dst_vect->resize(strm.total_out);
+		}
+
+		// Clean up and return.
+		(void)inflateEnd(&strm);
+		return Z_OK;
+	}
+}