Change: replace liblzma with libarchive for OTTX savegame format

This is a drop-in replacement. Savegames with both can be read
by both.

.github/workflows/ci-linux.yml

@@ -53,12 +53,12 @@ jobs:
 
         echo "::group::Install dependencies"
         sudo apt-get install -y --no-install-recommends \
+          libarchive-dev \
           liballegro4-dev \
           libcurl4-openssl-dev \
           libfontconfig-dev \
           libharfbuzz-dev \
           libicu-dev \
-          liblzma-dev \
           liblzo2-dev \
           ${{ inputs.libraries }} \
           zlib1g-dev \

.github/workflows/codeql.yml

@@ -47,12 +47,12 @@ jobs:
 
         echo "::group::Install dependencies"
         sudo apt-get install -y --no-install-recommends \
+          libarchive-dev \
           liballegro4-dev \
           libcurl4-openssl-dev \
           libfontconfig-dev \
           libharfbuzz-dev \
           libicu-dev \
-          liblzma-dev \
           liblzo2-dev \
           libsdl2-dev \
           zlib1g-dev \

CMakeLists.txt

@@ -121,9 +121,9 @@ set(CMAKE_THREAD_PREFER_PTHREAD YES)
 find_package(Threads REQUIRED)
 
 find_package(ZLIB)
-find_package(LibLZMA)
 find_package(LZO)
 find_package(PNG)
+find_package(LibArchive)
 
 if(WIN32 OR EMSCRIPTEN)
     # Windows uses WinHttp for HTTP requests.
@@ -310,7 +310,7 @@ process_compile_flags()
 include(LinkPackage)
 link_package(PNG TARGET PNG::PNG ENCOURAGED)
 link_package(ZLIB TARGET ZLIB::ZLIB ENCOURAGED)
-link_package(LIBLZMA TARGET LibLZMA::LibLZMA ENCOURAGED)
+link_package(LibArchive TARGET LibArchive::LibArchive ENCOURAGED)
 link_package(LZO)
 
 if(NOT WIN32 AND NOT EMSCRIPTEN)

COMPILING.md

@@ -7,7 +7,7 @@ OpenTTD makes use of the following external libraries:
 - (encouraged) breakpad: creates minidumps on crash
 - (encouraged) zlib: (de)compressing of old (0.3.0-1.0.5) savegames, content downloads,
    heightmaps
-- (encouraged) liblzma: (de)compressing of savegames (1.1.0 and later)
+- (encouraged) libarchive: (de)compressing of savegames (1.1.0 and later)
 - (encouraged) libpng: making screenshots and loading heightmaps
 - (optional) liblzo2: (de)compressing of old (pre 0.3.0) savegames
 
@@ -24,7 +24,7 @@ For Linux, the following additional libraries are used:
 If you are building a dedicated-server only, you don't need the last four.
 
 OpenTTD does not require any of the libraries to be present, but without
-liblzma you cannot open most recent savegames and without zlib you cannot
+libarchive you cannot open most recent savegames and without zlib you cannot
 open most older savegames or use the content downloading system.
 
 ## Windows
@@ -51,7 +51,7 @@ After this, you can install the dependencies OpenTTD needs. We advise to use
 the `static` versions, and OpenTTD currently needs the following dependencies:
 
 - breakpad
-- liblzma
+- libarchive
 - libpng
 - lzo
 - zlib

Doxyfile.in

@@ -287,7 +287,7 @@ INCLUDE_PATH           =
 INCLUDE_FILE_PATTERNS  =
 PREDEFINED             = WITH_ZLIB \
                          WITH_LZO \
-                         WITH_LIBLZMA \
+                         WITH_LIBARCHIVE \
                          WITH_SDL \
                          WITH_PNG \
                          WITH_FONTCONFIG \

src/network/network_server.cpp

@@ -141,7 +141,7 @@ struct PacketWriter : SaveFilter {
 		return false;
 	}
 
-	void Write(uint8_t *buf, size_t size) override
+	void Write(const uint8_t *buf, size_t size) override
 	{
 		std::lock_guard<std::mutex> lock(this->mutex);
 

src/saveload/saveload.cpp

@@ -2224,7 +2224,7 @@ struct FileWriter : SaveFilter {
 		this->Finish();
 	}
 
-	void Write(uint8_t *buf, size_t size) override
+	void Write(const uint8_t *buf, size_t size) override
 	{
 		/* We're in the process of shutting down, i.e. in "failure" mode. */
 		if (this->file == nullptr) return;
@@ -2310,7 +2310,7 @@ struct LZOSaveFilter : SaveFilter {
 		if (lzo_init() != LZO_E_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
 	}
 
-	void Write(uint8_t *buf, size_t size) override
+	void Write(const uint8_t *buf, size_t size) override
 	{
 		const lzo_bytep in = buf;
 		/* Buffer size is from the LZO docs plus the chunk header size. */
@@ -2365,7 +2365,7 @@ struct NoCompSaveFilter : SaveFilter {
 	{
 	}
 
-	void Write(uint8_t *buf, size_t size) override
+	void Write(const uint8_t *buf, size_t size) override
 	{
 		this->chain->Write(buf, size);
 	}
@@ -2450,10 +2450,10 @@ struct ZlibSaveFilter : SaveFilter {
 	 * @param len  Amount of bytes to write.
 	 * @param mode Mode for deflate.
 	 */
-	void WriteLoop(uint8_t *p, size_t len, int mode)
+	void WriteLoop(const uint8_t *p, size_t len, int mode)
 	{
 		uint n;
-		this->z.next_in = p;
+		this->z.next_in = const_cast<uint8_t *>(p);
 		this->z.avail_in = (uInt)len;
 		do {
 			this->z.next_out = this->fwrite_buf;
@@ -2478,7 +2478,7 @@ struct ZlibSaveFilter : SaveFilter {
 		} while (this->z.avail_in || !this->z.avail_out);
 	}
 
-	void Write(uint8_t *buf, size_t size) override
+	void Write(const uint8_t *buf, size_t size) override
 	{
 		this->WriteLoop(buf, size, 0);
 	}
@@ -2496,63 +2496,61 @@ struct ZlibSaveFilter : SaveFilter {
  ********** START OF LZMA CODE **************
  ********************************************/
 
-#if defined(WITH_LIBLZMA)
+#if defined(WITH_LIBARCHIVE)
-#include <lzma.h>
+#include <archive.h>
-
+#include <archive_entry.h>
-/**
- * Have a copy of an initialised LZMA stream. We need this as it's
- * impossible to "re"-assign LZMA_STREAM_INIT to a variable in some
- * compilers, i.e. LZMA_STREAM_INIT can't be used to set something.
- * This var has to be used instead.
- */
-static const lzma_stream _lzma_init = LZMA_STREAM_INIT;
 
 /** Filter without any compression. */
 struct LZMALoadFilter : LoadFilter {
-	lzma_stream lzma;                  ///< Stream state that we are reading from.
+	struct archive *archive; ///< Archive state.
 	uint8_t fread_buf[MEMORY_CHUNK_SIZE]; ///< Buffer for reading from the file.
 
 	/**
 	 * Initialise this filter.
 	 * @param chain The next filter in this chain.
 	 */
-	LZMALoadFilter(std::shared_ptr<LoadFilter> chain) : LoadFilter(chain), lzma(_lzma_init)
+	LZMALoadFilter(std::shared_ptr<LoadFilter> chain) : LoadFilter(chain), archive(archive_read_new())
 	{
-		/* Allow saves up to 256 MB uncompressed */
+		archive_read_support_filter_xz(this->archive);
-		if (lzma_auto_decoder(&this->lzma, 1 << 28, 0) != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize decompressor");
+		archive_read_support_format_raw(this->archive);
+
+		archive_read_set_read_callback(this->archive, [](struct archive *, void *client_data, const void **buffer) -> ssize_t {
+			LZMALoadFilter *this_ = static_cast<LZMALoadFilter *>(client_data);
+
+			*buffer = this_->fread_buf;
+
+			ssize_t res = this_->chain->Read(this_->fread_buf, sizeof(this_->fread_buf));
+			return res;
+		});
+
+		archive_read_append_callback_data(this->archive, this);
+		int res = archive_read_open1(this->archive);
+		if (res != ARCHIVE_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize decompressor");
+
+		/* Read the next "header". As the format is set to raw(), this just runs
+		 * the format auto-detection so archive knows this is a lzma file. */
+		struct archive_entry *entry;
+		res = archive_read_next_header(this->archive, &entry);
+		if (res != ARCHIVE_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize decompressor");
 	}
 
 	/** Clean everything up. */
 	~LZMALoadFilter()
 	{
-		lzma_end(&this->lzma);
+		archive_read_free(this->archive);
 	}
 
 	size_t Read(uint8_t *buf, size_t size) override
 	{
-		this->lzma.next_out  = buf;
+		int res = archive_read_data(this->archive, buf, size);
-		this->lzma.avail_out = size;
+		if (res < 0) SlError(STR_GAME_SAVELOAD_ERROR_FILE_NOT_READABLE, "archive_read_data returned error code");
-
+		return res;
-		do {
-			/* read more bytes from the file? */
-			if (this->lzma.avail_in == 0) {
-				this->lzma.next_in  = this->fread_buf;
-				this->lzma.avail_in = this->chain->Read(this->fread_buf, sizeof(this->fread_buf));
-			}
-
-			/* inflate the data */
-			lzma_ret r = lzma_code(&this->lzma, LZMA_RUN);
-			if (r == LZMA_STREAM_END) break;
-			if (r != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "liblzma returned error code");
-		} while (this->lzma.avail_out != 0);
-
-		return size - this->lzma.avail_out;
 	}
 };
 
 /** Filter using LZMA compression. */
 struct LZMASaveFilter : SaveFilter {
-	lzma_stream lzma; ///< Stream state that we are writing to.
+	struct archive *archive; ///< Archive state.
 	uint8_t fwrite_buf[MEMORY_CHUNK_SIZE]; ///< Buffer for writing to the file.
 
 	/**
@@ -2560,56 +2558,53 @@ struct LZMASaveFilter : SaveFilter {
 	 * @param chain             The next filter in this chain.
 	 * @param compression_level The requested level of compression.
 	 */
-	LZMASaveFilter(std::shared_ptr<SaveFilter> chain, uint8_t compression_level) : SaveFilter(chain), lzma(_lzma_init)
+	LZMASaveFilter(std::shared_ptr<SaveFilter> chain, uint8_t compression_level) : SaveFilter(chain), archive(archive_write_new())
 	{
-		if (lzma_easy_encoder(&this->lzma, compression_level, LZMA_CHECK_CRC32) != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
+		archive_write_add_filter_xz(this->archive);
+		archive_write_set_format_raw(this->archive);
+		archive_write_set_filter_option(this->archive, nullptr, "compression-level", std::to_string(compression_level).c_str());
+
+		int res = archive_write_open2(this->archive, this, nullptr, [](struct archive *, void *client_data, const void *buffer, size_t length) -> ssize_t {
+			LZMASaveFilter *this_ = static_cast<LZMASaveFilter *>(client_data);
+
+			this_->chain->Write(reinterpret_cast<const uint8_t *>(buffer), length);
+			return length;
+		}, nullptr, nullptr);
+		if (res != ARCHIVE_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
+
+		/* Write a "header". As the format is set to raw(), this doesn't do
+		 * anything, but archive needs it before you can write data. */
+		struct archive_entry *entry = archive_entry_new();
+		archive_entry_set_pathname(entry, "");
+		archive_entry_set_size(entry, 1);
+		archive_entry_set_filetype(entry, AE_IFREG);
+		archive_entry_set_perm(entry, 0644);
+
+		res = archive_write_header(this->archive, entry);
+		if (res != ARCHIVE_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
+
+		archive_entry_free(entry);
 	}
 
 	/** Clean up what we allocated. */
 	~LZMASaveFilter()
 	{
-		lzma_end(&this->lzma);
+		archive_write_free(this->archive);
 	}
 
-	/**
+	void Write(const uint8_t *buf, size_t size) override
-	 * Helper loop for writing the data.
-	 * @param p      The bytes to write.
-	 * @param len    Amount of bytes to write.
-	 * @param action Action for lzma_code.
-	 */
-	void WriteLoop(uint8_t *p, size_t len, lzma_action action)
 	{
-		size_t n;
+		int res = archive_write_data(this->archive, buf, size);
-		this->lzma.next_in = p;
+		if (res < 0) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "archive_write_data returned error code");
-		this->lzma.avail_in = len;
-		do {
-			this->lzma.next_out = this->fwrite_buf;
-			this->lzma.avail_out = sizeof(this->fwrite_buf);
-
-			lzma_ret r = lzma_code(&this->lzma, action);
-
-			/* bytes were emitted? */
-			if ((n = sizeof(this->fwrite_buf) - this->lzma.avail_out) != 0) {
-				this->chain->Write(this->fwrite_buf, n);
-			}
-			if (r == LZMA_STREAM_END) break;
-			if (r != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "liblzma returned error code");
-		} while (this->lzma.avail_in || !this->lzma.avail_out);
-	}
-
-	void Write(uint8_t *buf, size_t size) override
-	{
-		this->WriteLoop(buf, size, LZMA_RUN);
 	}
 
 	void Finish() override
 	{
-		this->WriteLoop(nullptr, 0, LZMA_FINISH);
+		archive_write_close(this->archive);
-		this->chain->Finish();
 	}
 };
 
-#endif /* WITH_LIBLZMA */
+#endif /* WITH_LIBARCHIVE */
 
 /*******************************************
  ************* END OF CODE *****************
@@ -2646,7 +2641,7 @@ static const SaveLoadFormat _saveload_formats[] = {
 #else
 	{"zlib",   TO_BE32X('OTTZ'), nullptr,                            nullptr,                            0, 0, 0},
 #endif
-#if defined(WITH_LIBLZMA)
+#if defined(WITH_LIBARCHIVE)
 	/* Level 2 compression is speed wise as fast as zlib level 6 compression (old default), but results in ~10% smaller saves.
 	 * Higher compression levels are possible, and might improve savegame size by up to 25%, but are also up to 10 times slower.
 	 * The next significant reduction in file size is at level 4, but that is already 4 times slower. Level 3 is primarily 50%

src/saveload/saveload_filter.h

@@ -78,7 +78,7 @@ struct SaveFilter {
 	 * @param buf The bytes to write.
 	 * @param len The number of bytes to write.
 	 */
-	virtual void Write(uint8_t *buf, size_t len) = 0;
+	virtual void Write(const uint8_t *buf, size_t len) = 0;
 
 	/**
 	 * Prepare everything to finish writing the savegame.

src/stdafx.h

@@ -147,13 +147,6 @@
 #		if !defined(FT_EXPORT)
 #			define FT_EXPORT( x )  extern "C"  x CDECL
 #		endif
-#	endif
-
-	/* liblzma from vcpkg (before 5.2.4-2) used to patch lzma.h to define LZMA_API_STATIC for static builds */
-#	if defined(WITH_LIBLZMA)
-#		if !defined(LZMA_API_STATIC)
-#			define LZMA_API_STATIC
-#		endif
 #	endif
 
 	/* MSVC doesn't have these :( */

src/survey.cpp

@@ -57,8 +57,8 @@
 #ifdef WITH_ICU_I18N
 #	include <unicode/uversion.h>
 #endif /* WITH_ICU_I18N */
-#ifdef WITH_LIBLZMA
+#ifdef WITH_LIBARCHIVE
-#	include <lzma.h>
+#	include <archive.h>
 #endif
 #ifdef WITH_LZO
 #include <lzo/lzo1x.h>
@@ -429,8 +429,8 @@ void SurveyLibraries(nlohmann::json &survey)
 	survey["icu_i18n"] = buf;
 #endif /* WITH_ICU_I18N */
 
-#ifdef WITH_LIBLZMA
+#ifdef WITH_LIBARCHIVE
-	survey["lzma"] = lzma_version_string();
+	survey["archive"] = archive_version_string();
 #endif
 
 #ifdef WITH_LZO

src/textfile_gui.cpp

@@ -29,10 +29,6 @@
 #include <zlib.h>
 #endif
 
-#if defined(WITH_LIBLZMA)
-#include <lzma.h>
-#endif
-
 #include <regex>
 
 #include "safeguards.h"
@@ -704,61 +700,6 @@ static void Gunzip(uint8_t **bufp, size_t *sizep)
 }
 #endif
 
-#if defined(WITH_LIBLZMA)
-
-/**
- * Do an in-memory xunzip operation. This works on a .xz or (legacy)
- * .lzma file.
- * @param bufp  A pointer to a buffer containing the input data. This
- *              buffer will be freed and replaced by a buffer containing
- *              the uncompressed data.
- * @param sizep A pointer to the buffer size. Before the call, the value
- *              pointed to should contain the size of the input buffer.
- *              After the call, it contains the size of the uncompressed
- *              data.
- *
- * When decompressing fails, *bufp is set to nullptr and *sizep to 0. The
- * compressed buffer passed in is still freed in this case.
- */
-static void Xunzip(uint8_t **bufp, size_t *sizep)
-{
-	static const int BLOCKSIZE  = 8192;
-	uint8_t             *buf       = nullptr;
-	size_t           alloc_size = 0;
-	lzma_stream      z = LZMA_STREAM_INIT;
-	int              res;
-
-	z.next_in = *bufp;
-	z.avail_in = *sizep;
-
-	res = lzma_auto_decoder(&z, UINT64_MAX, LZMA_CONCATENATED);
-	/* Z_BUF_ERROR just means we need more space */
-	while (res == LZMA_OK || (res == LZMA_BUF_ERROR && z.avail_out == 0)) {
-		/* When we get here, we're either just starting, or
-		 * inflate is out of output space - allocate more */
-		alloc_size += BLOCKSIZE;
-		z.avail_out += BLOCKSIZE;
-		buf = ReallocT(buf, alloc_size);
-		z.next_out = buf + alloc_size - z.avail_out;
-		res = lzma_code(&z, LZMA_FINISH);
-	}
-
-	free(*bufp);
-	lzma_end(&z);
-
-	if (res == LZMA_STREAM_END) {
-		*bufp = buf;
-		*sizep = alloc_size - z.avail_out;
-	} else {
-		/* Something went wrong */
-		*bufp = nullptr;
-		*sizep = 0;
-		free(buf);
-	}
-}
-#endif
-
-
 /**
  * Loads the textfile text from file and setup #lines.
  */
@@ -791,12 +732,6 @@ static void Xunzip(uint8_t **bufp, size_t *sizep)
 	/* In-place gunzip */
 	if (textfile.ends_with(".gz")) Gunzip((uint8_t**)&buf, &filesize);
 #endif
-
-#if defined(WITH_LIBLZMA)
-	/* In-place xunzip */
-	if (textfile.ends_with(".xz")) Xunzip((uint8_t**)&buf, &filesize);
-#endif
-
 	if (buf == nullptr) return;
 
 	std::string_view sv_buf(buf, filesize);
@@ -889,10 +824,6 @@ std::optional<std::string> GetTextfile(TextfileType type, Subdirectory dir, cons
 #if defined(WITH_ZLIB)
 		"txt.gz",
 		"md.gz",
-#endif
-#if defined(WITH_LIBLZMA)
-		"txt.xz",
-		"md.xz",
 #endif
 	};
 

vcpkg.json

@@ -34,7 +34,11 @@
       "platform": "linux"
     },
     {
-      "name": "liblzma"
+      "name": "libarchive",
+      "default-features": false,
+      "features": [
+        "lzma"
+      ]
     },
     {
       "name": "libpng"