mirror
/
OpenTTD
mirror of https://github.com/OpenTTD/OpenTTD
1
0
Fork 0
Code Issues

Change: replace liblzma with libarchive for OTTX savegame format 

This is a drop-in replacement. Savegames with both can be read
by both.
pull/12417/head
Patric Stout 2024-04-03 23:06:48 +02:00
parent 3d2a8fb60c
commit 8ee7fafdfe
12 changed files with 88 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci-linux.yml vendored
View File

@ -53,12 +53,12 @@ jobs:
echo "::group::Install dependencies"
sudo apt-get install -y --no-install-recommends \
libarchive-dev \
liballegro4-dev \
libcurl4-openssl-dev \
libfontconfig-dev \
libharfbuzz-dev \
libicu-dev \
liblzma-dev \
liblzo2-dev \
${{ inputs.libraries }} \
zlib1g-dev \

2
.github/workflows/codeql.yml vendored
View File

@ -47,12 +47,12 @@ jobs:
echo "::group::Install dependencies"
sudo apt-get install -y --no-install-recommends \
libarchive-dev \
liballegro4-dev \
libcurl4-openssl-dev \
libfontconfig-dev \
libharfbuzz-dev \
libicu-dev \
liblzma-dev \
liblzo2-dev \
libsdl2-dev \
zlib1g-dev \

4
CMakeLists.txt
View File

@ -121,9 +121,9 @@ set(CMAKE_THREAD_PREFER_PTHREAD YES)
find_package(Threads REQUIRED)
find_package(ZLIB)
find_package(LibLZMA)
find_package(LZO)
find_package(PNG)
find_package(LibArchive)
if(WIN32 OR EMSCRIPTEN)
# Windows uses WinHttp for HTTP requests.
@ -310,7 +310,7 @@ process_compile_flags()
include(LinkPackage)
link_package(PNG TARGET PNG::PNG ENCOURAGED)
link_package(ZLIB TARGET ZLIB::ZLIB ENCOURAGED)
link_package(LIBLZMA TARGET LibLZMA::LibLZMA ENCOURAGED)
link_package(LibArchive TARGET LibArchive::LibArchive ENCOURAGED)
link_package(LZO)
if(NOT WIN32 AND NOT EMSCRIPTEN)

6
COMPILING.md
View File

@ -7,7 +7,7 @@ OpenTTD makes use of the following external libraries:
- (encouraged) breakpad: creates minidumps on crash
- (encouraged) zlib: (de)compressing of old (0.3.0-1.0.5) savegames, content downloads,
heightmaps
- (encouraged) liblzma: (de)compressing of savegames (1.1.0 and later)
- (encouraged) libarchive: (de)compressing of savegames (1.1.0 and later)
- (encouraged) libpng: making screenshots and loading heightmaps
- (optional) liblzo2: (de)compressing of old (pre 0.3.0) savegames
@ -24,7 +24,7 @@ For Linux, the following additional libraries are used:
If you are building a dedicated-server only, you don't need the last four.
OpenTTD does not require any of the libraries to be present, but without
liblzma you cannot open most recent savegames and without zlib you cannot
libarchive you cannot open most recent savegames and without zlib you cannot
open most older savegames or use the content downloading system.
## Windows
@ -51,7 +51,7 @@ After this, you can install the dependencies OpenTTD needs. We advise to use
the `static` versions, and OpenTTD currently needs the following dependencies:
- breakpad
- liblzma
- libarchive
- libpng
- lzo
- zlib

2
Doxyfile.in
View File

@ -287,7 +287,7 @@ INCLUDE_PATH =
INCLUDE_FILE_PATTERNS =
PREDEFINED = WITH_ZLIB \
WITH_LZO \
WITH_LIBLZMA \
WITH_LIBARCHIVE \
WITH_SDL \
WITH_PNG \
WITH_FONTCONFIG \

2
src/network/network_server.cpp
View File

@ -141,7 +141,7 @@ struct PacketWriter : SaveFilter {
return false;
}
void Write(uint8_t *buf, size_t size) override
void Write(const uint8_t *buf, size_t size) override
{
std::lock_guard<std::mutex> lock(this->mutex);

143
src/saveload/saveload.cpp
View File

@ -2224,7 +2224,7 @@ struct FileWriter : SaveFilter {
this->Finish();
}
void Write(uint8_t *buf, size_t size) override
void Write(const uint8_t *buf, size_t size) override
{
/* We're in the process of shutting down, i.e. in "failure" mode. */
if (this->file == nullptr) return;
@ -2310,7 +2310,7 @@ struct LZOSaveFilter : SaveFilter {
if (lzo_init() != LZO_E_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
}
void Write(uint8_t *buf, size_t size) override
void Write(const uint8_t *buf, size_t size) override
{
const lzo_bytep in = buf;
/* Buffer size is from the LZO docs plus the chunk header size. */
@ -2365,7 +2365,7 @@ struct NoCompSaveFilter : SaveFilter {
{
}
void Write(uint8_t *buf, size_t size) override
void Write(const uint8_t *buf, size_t size) override
{
this->chain->Write(buf, size);
}
@ -2450,10 +2450,10 @@ struct ZlibSaveFilter : SaveFilter {
* @param len Amount of bytes to write.
* @param mode Mode for deflate.
*/
void WriteLoop(uint8_t *p, size_t len, int mode)
void WriteLoop(const uint8_t *p, size_t len, int mode)
{
uint n;
this->z.next_in = p;
this->z.next_in = const_cast<uint8_t *>(p);
this->z.avail_in = (uInt)len;
do {
this->z.next_out = this->fwrite_buf;
@ -2478,7 +2478,7 @@ struct ZlibSaveFilter : SaveFilter {
} while (this->z.avail_in || !this->z.avail_out);
}
void Write(uint8_t *buf, size_t size) override
void Write(const uint8_t *buf, size_t size) override
{
this->WriteLoop(buf, size, 0);
}
@ -2496,63 +2496,61 @@ struct ZlibSaveFilter : SaveFilter {
********** START OF LZMA CODE **************
********************************************/
#if defined(WITH_LIBLZMA)
#include <lzma.h>
/**
* Have a copy of an initialised LZMA stream. We need this as it's
* impossible to "re"-assign LZMA_STREAM_INIT to a variable in some
* compilers, i.e. LZMA_STREAM_INIT can't be used to set something.
* This var has to be used instead.
*/
static const lzma_stream _lzma_init = LZMA_STREAM_INIT;
#if defined(WITH_LIBARCHIVE)
#include <archive.h>
#include <archive_entry.h>
/** Filter without any compression. */
struct LZMALoadFilter : LoadFilter {
lzma_stream lzma; ///< Stream state that we are reading from.
struct archive *archive; ///< Archive state.
uint8_t fread_buf[MEMORY_CHUNK_SIZE]; ///< Buffer for reading from the file.
/**
* Initialise this filter.
* @param chain The next filter in this chain.
*/
LZMALoadFilter(std::shared_ptr<LoadFilter> chain) : LoadFilter(chain), lzma(_lzma_init)
LZMALoadFilter(std::shared_ptr<LoadFilter> chain) : LoadFilter(chain), archive(archive_read_new())
{
/* Allow saves up to 256 MB uncompressed */
if (lzma_auto_decoder(&this->lzma, 1 << 28, 0) != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize decompressor");
archive_read_support_filter_xz(this->archive);
archive_read_support_format_raw(this->archive);
archive_read_set_read_callback(this->archive, [](struct archive *, void *client_data, const void **buffer) -> ssize_t {
LZMALoadFilter *this_ = static_cast<LZMALoadFilter *>(client_data);
*buffer = this_->fread_buf;
ssize_t res = this_->chain->Read(this_->fread_buf, sizeof(this_->fread_buf));
return res;
});
archive_read_append_callback_data(this->archive, this);
int res = archive_read_open1(this->archive);
if (res != ARCHIVE_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize decompressor");
/* Read the next "header". As the format is set to raw(), this just runs
* the format auto-detection so archive knows this is a lzma file. */
struct archive_entry *entry;
res = archive_read_next_header(this->archive, &entry);
if (res != ARCHIVE_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize decompressor");
}
/** Clean everything up. */
~LZMALoadFilter()
{
lzma_end(&this->lzma);
archive_read_free(this->archive);
}
size_t Read(uint8_t *buf, size_t size) override
{
this->lzma.next_out = buf;
this->lzma.avail_out = size;
do {
/* read more bytes from the file? */
if (this->lzma.avail_in == 0) {
this->lzma.next_in = this->fread_buf;
this->lzma.avail_in = this->chain->Read(this->fread_buf, sizeof(this->fread_buf));
}
/* inflate the data */
lzma_ret r = lzma_code(&this->lzma, LZMA_RUN);
if (r == LZMA_STREAM_END) break;
if (r != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "liblzma returned error code");
} while (this->lzma.avail_out != 0);
return size - this->lzma.avail_out;
int res = archive_read_data(this->archive, buf, size);
if (res < 0) SlError(STR_GAME_SAVELOAD_ERROR_FILE_NOT_READABLE, "archive_read_data returned error code");
return res;
}
};
/** Filter using LZMA compression. */
struct LZMASaveFilter : SaveFilter {
lzma_stream lzma; ///< Stream state that we are writing to.
struct archive *archive; ///< Archive state.
uint8_t fwrite_buf[MEMORY_CHUNK_SIZE]; ///< Buffer for writing to the file.
/**
@ -2560,56 +2558,53 @@ struct LZMASaveFilter : SaveFilter {
* @param chain The next filter in this chain.
* @param compression_level The requested level of compression.
*/
LZMASaveFilter(std::shared_ptr<SaveFilter> chain, uint8_t compression_level) : SaveFilter(chain), lzma(_lzma_init)
LZMASaveFilter(std::shared_ptr<SaveFilter> chain, uint8_t compression_level) : SaveFilter(chain), archive(archive_write_new())
{
if (lzma_easy_encoder(&this->lzma, compression_level, LZMA_CHECK_CRC32) != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
archive_write_add_filter_xz(this->archive);
archive_write_set_format_raw(this->archive);
archive_write_set_filter_option(this->archive, nullptr, "compression-level", std::to_string(compression_level).c_str());
int res = archive_write_open2(this->archive, this, nullptr, [](struct archive *, void *client_data, const void *buffer, size_t length) -> ssize_t {
LZMASaveFilter *this_ = static_cast<LZMASaveFilter *>(client_data);
this_->chain->Write(reinterpret_cast<const uint8_t *>(buffer), length);
return length;
}, nullptr, nullptr);
if (res != ARCHIVE_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
/* Write a "header". As the format is set to raw(), this doesn't do
* anything, but archive needs it before you can write data. */
struct archive_entry *entry = archive_entry_new();
archive_entry_set_pathname(entry, "");
archive_entry_set_size(entry, 1);
archive_entry_set_filetype(entry, AE_IFREG);
archive_entry_set_perm(entry, 0644);
res = archive_write_header(this->archive, entry);
if (res != ARCHIVE_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor");
archive_entry_free(entry);
}
/** Clean up what we allocated. */
~LZMASaveFilter()
{
lzma_end(&this->lzma);
archive_write_free(this->archive);
}
/**
* Helper loop for writing the data.
* @param p The bytes to write.
* @param len Amount of bytes to write.
* @param action Action for lzma_code.
*/
void WriteLoop(uint8_t *p, size_t len, lzma_action action)
void Write(const uint8_t *buf, size_t size) override
{
size_t n;
this->lzma.next_in = p;
this->lzma.avail_in = len;
do {
this->lzma.next_out = this->fwrite_buf;
this->lzma.avail_out = sizeof(this->fwrite_buf);
lzma_ret r = lzma_code(&this->lzma, action);
/* bytes were emitted? */
if ((n = sizeof(this->fwrite_buf) - this->lzma.avail_out) != 0) {
this->chain->Write(this->fwrite_buf, n);
}
if (r == LZMA_STREAM_END) break;
if (r != LZMA_OK) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "liblzma returned error code");
} while (this->lzma.avail_in || !this->lzma.avail_out);
}
void Write(uint8_t *buf, size_t size) override
{
this->WriteLoop(buf, size, LZMA_RUN);
int res = archive_write_data(this->archive, buf, size);
if (res < 0) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "archive_write_data returned error code");
}
void Finish() override
{
this->WriteLoop(nullptr, 0, LZMA_FINISH);
this->chain->Finish();
archive_write_close(this->archive);
}
};
#endif /* WITH_LIBLZMA */
#endif /* WITH_LIBARCHIVE */
/*******************************************
************* END OF CODE *****************
@ -2646,7 +2641,7 @@ static const SaveLoadFormat _saveload_formats[] = {
#else
{"zlib", TO_BE32X('OTTZ'), nullptr, nullptr, 0, 0, 0},
#endif
#if defined(WITH_LIBLZMA)
#if defined(WITH_LIBARCHIVE)
/* Level 2 compression is speed wise as fast as zlib level 6 compression (old default), but results in ~10% smaller saves.
* Higher compression levels are possible, and might improve savegame size by up to 25%, but are also up to 10 times slower.
* The next significant reduction in file size is at level 4, but that is already 4 times slower. Level 3 is primarily 50%

2
src/saveload/saveload_filter.h
View File

@ -78,7 +78,7 @@ struct SaveFilter {
* @param buf The bytes to write.
* @param len The number of bytes to write.
*/
virtual void Write(uint8_t *buf, size_t len) = 0;
virtual void Write(const uint8_t *buf, size_t len) = 0;
/**
* Prepare everything to finish writing the savegame.

7
src/stdafx.h
View File

@ -147,13 +147,6 @@
# if !defined(FT_EXPORT)
# define FT_EXPORT( x ) extern "C" x CDECL
# endif
# endif
/* liblzma from vcpkg (before 5.2.4-2) used to patch lzma.h to define LZMA_API_STATIC for static builds */
# if defined(WITH_LIBLZMA)
# if !defined(LZMA_API_STATIC)
# define LZMA_API_STATIC
# endif
# endif
/* MSVC doesn't have these :( */

8
src/survey.cpp
View File

@ -57,8 +57,8 @@
#ifdef WITH_ICU_I18N
# include <unicode/uversion.h>
#endif /* WITH_ICU_I18N */
#ifdef WITH_LIBLZMA
# include <lzma.h>
#ifdef WITH_LIBARCHIVE
# include <archive.h>
#endif
#ifdef WITH_LZO
#include <lzo/lzo1x.h>
@ -429,8 +429,8 @@ void SurveyLibraries(nlohmann::json &survey)
survey["icu_i18n"] = buf;
#endif /* WITH_ICU_I18N */
#ifdef WITH_LIBLZMA
survey["lzma"] = lzma_version_string();
#ifdef WITH_LIBARCHIVE
survey["archive"] = archive_version_string();
#endif
#ifdef WITH_LZO

69
src/textfile_gui.cpp
View File

@ -29,10 +29,6 @@
#include <zlib.h>
#endif
#if defined(WITH_LIBLZMA)
#include <lzma.h>
#endif
#include <regex>
#include "safeguards.h"
@ -704,61 +700,6 @@ static void Gunzip(uint8_t **bufp, size_t *sizep)
}
#endif
#if defined(WITH_LIBLZMA)
/**
* Do an in-memory xunzip operation. This works on a .xz or (legacy)
* .lzma file.
* @param bufp A pointer to a buffer containing the input data. This
* buffer will be freed and replaced by a buffer containing
* the uncompressed data.
* @param sizep A pointer to the buffer size. Before the call, the value
* pointed to should contain the size of the input buffer.
* After the call, it contains the size of the uncompressed
* data.
*
* When decompressing fails, *bufp is set to nullptr and *sizep to 0. The
* compressed buffer passed in is still freed in this case.
*/
static void Xunzip(uint8_t **bufp, size_t *sizep)
{
static const int BLOCKSIZE = 8192;
uint8_t *buf = nullptr;
size_t alloc_size = 0;
lzma_stream z = LZMA_STREAM_INIT;
int res;
z.next_in = *bufp;
z.avail_in = *sizep;
res = lzma_auto_decoder(&z, UINT64_MAX, LZMA_CONCATENATED);
/* Z_BUF_ERROR just means we need more space */
while (res == LZMA_OK || (res == LZMA_BUF_ERROR && z.avail_out == 0)) {
/* When we get here, we're either just starting, or
* inflate is out of output space - allocate more */
alloc_size += BLOCKSIZE;
z.avail_out += BLOCKSIZE;
buf = ReallocT(buf, alloc_size);
z.next_out = buf + alloc_size - z.avail_out;
res = lzma_code(&z, LZMA_FINISH);
}
free(*bufp);
lzma_end(&z);
if (res == LZMA_STREAM_END) {
*bufp = buf;
*sizep = alloc_size - z.avail_out;
} else {
/* Something went wrong */
*bufp = nullptr;
*sizep = 0;
free(buf);
}
}
#endif
/**
* Loads the textfile text from file and setup #lines.
*/
@ -791,12 +732,6 @@ static void Xunzip(uint8_t **bufp, size_t *sizep)
/* In-place gunzip */
if (textfile.ends_with(".gz")) Gunzip((uint8_t**)&buf, &filesize);
#endif
#if defined(WITH_LIBLZMA)
/* In-place xunzip */
if (textfile.ends_with(".xz")) Xunzip((uint8_t**)&buf, &filesize);
#endif
if (buf == nullptr) return;
std::string_view sv_buf(buf, filesize);
@ -889,10 +824,6 @@ std::optional<std::string> GetTextfile(TextfileType type, Subdirectory dir, cons
#if defined(WITH_ZLIB)
"txt.gz",
"md.gz",
#endif
#if defined(WITH_LIBLZMA)
"txt.xz",
"md.xz",
#endif
};

6
vcpkg.json
View File

@ -34,7 +34,11 @@
"platform": "linux"
},
{
"name": "liblzma"
"name": "libarchive",
"default-features": false,
"features": [
"lzma"
]
},
{
"name": "libpng"