Index: trunk/Simutrans-GDI.vcxproj
===================================================================
--- trunk/Simutrans-GDI.vcxproj (revision 8887)
+++ trunk/Simutrans-GDI.vcxproj (working copy)
@@ -83,7 +83,7 @@
EnableFastChecks
- kernel32.lib;user32.lib;gdi32.lib;shell32.lib;winmm.lib;zlibstat.lib;advapi32.lib;ws2_32.lib;imm32.lib;libbz2.lib;libpthreadVC3.lib;miniupnpc.lib;freetype.lib
+ kernel32.lib;user32.lib;gdi32.lib;shell32.lib;winmm.lib;zlibstat.lib;advapi32.lib;ws2_32.lib;imm32.lib;libbz2.lib;libpthreadVC3.lib;miniupnpc.lib;freetype.lib;libzstd_static.lib
libcmtd.lib
true
false
Index: trunk/Simutrans-SDL2.vcxproj
===================================================================
--- trunk/Simutrans-SDL2.vcxproj (revision 8887)
+++ trunk/Simutrans-SDL2.vcxproj (working copy)
@@ -1,4 +1,4 @@
-
+
@@ -83,7 +83,7 @@
EnableFastChecks
- SDL2main.lib;SDL2.lib;kernel32.lib;user32.lib;shell32.lib;winmm.lib;zlibstat.lib;advapi32.lib;ws2_32.lib;imm32.lib;libbz2.lib;libpthreadVC3d.lib;miniupnpc.lib;freetype.lib
+ SDL2main.lib;SDL2.lib;kernel32.lib;user32.lib;shell32.lib;winmm.lib;zlibstat.lib;advapi32.lib;ws2_32.lib;imm32.lib;libbz2.lib;libpthreadVC3d.lib;miniupnpc.lib;freetype.lib;libzstd_static.lib
libcmtd.lib
true
false
Index: trunk/dataobj/loadsave.cc
===================================================================
--- trunk/dataobj/loadsave.cc (revision 8887)
+++ trunk/dataobj/loadsave.cc (working copy)
@@ -16,15 +16,19 @@
#include "../utils/simstring.h"
+#include
#include
#include
+#define USE_ZSTD
+
#define INVALID_RDWR_ID (-1)
//#undef MULTI_THREAD
// buffer size for read/write - bzip2 gains up to 8M for non-threaded, 1M for threaded. binary, zipped ok with 256K or smaller.
-#define LS_BUF_SIZE (1024*1024)
+// zstd need their own buffer size ...
+#define LS_BUF_SIZE (1024 * 1024)
#ifdef MULTI_THREAD
#include "../utils/simthread.h"
@@ -184,7 +188,16 @@
gzFile gzfp;
BZFILE *bzfp;
int bse;
+#ifdef USE_ZSTD
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ void *zbuff;
+ ZSTD_CCtx *cctx;
+ ZSTD_DCtx *dctx;
+ file_descriptors_t() : fp( NULL ), gzfp( NULL ), bzfp( NULL ), bse( BZ_OK + 1 ), zbuff(NULL), cctx( NULL ), dctx( NULL ) {}
+#else
file_descriptors_t() : fp(NULL), gzfp(NULL), bzfp(NULL), bse(BZ_OK+1) {}
+#endif
};
@@ -220,6 +233,18 @@
buf_pos[0] = buf_pos[1] = 0;
buf_len[0] = buf_len[1] = 0;
ls_buf[0] = new char[LS_BUF_SIZE];
+#ifdef USE_ZSTD
+ if( is_zstd() ) {
+ if( saving ) {
+ fd->zout = { fd->zbuff, LS_BUF_SIZE, 0 };
+ fd->zin = { NULL, 0, 0 };
+ }
+ else {
+ fd->zout = { NULL, 0, 0 };
+ fd->zin = { fd->zbuff, LS_BUF_SIZE, 0 };
+ }
+ }
+#endif
#ifdef MULTI_THREAD
ls_buf[1] = new char[LS_BUF_SIZE]; // second buffer only when multithreaded
@@ -274,7 +299,9 @@
last_error = FILE_ERROR_OK; // no error
version = 0;
- mode = zipped;
+ mode = binary;
+ saving = false;
+
fd->fp = dr_fopen(filename_utf8, "rb");
if( fd->fp==NULL ) {
// most likely not existing
@@ -283,39 +310,51 @@
}
// now check for BZ2 format
char buf[80];
- if( fread( buf, 1, 80, fd->fp )==80 ) {
+ if( fread( buf, 1, 2, fd->fp )==2 ) {
if( buf[0]=='B' && buf[1]=='Z' ) {
mode = bzip2;
}
- fseek(fd->fp,0,SEEK_SET);
+ if( buf[0]=='Z' && buf[1]=='D' ) {
+ mode = zstd;
+ }
}
if( mode==bzip2 ) {
+ fseek(fd->fp,0,SEEK_SET);
fd->bse = BZ_OK+1;
fd->bzfp = NULL;
fd->bzfp = BZ2_bzReadOpen( &fd->bse, fd->fp, 0, 0, NULL, 0 );
- bool ok = false;
- if( fd->bse==BZ_OK ) {
- // else: use zlib
+ if( fd->bse!=BZ_OK ) {
MEMZERO(buf);
- if( BZ2_bzRead( &fd->bse, fd->bzfp, buf, sizeof(SAVEGAME_PREFIX) )==sizeof(SAVEGAME_PREFIX) && fd->bse==BZ_OK ) {
- // get the rest of the string
- for( int i=sizeof(SAVEGAME_PREFIX); (uint8)buf[i-1] >= 32 && i<79; i++ ) {
- buf[i] = lsgetc();
- }
- ok = fd->bse==BZ_OK;
- }
+ last_error = FILE_ERROR_BZ_CORRUPT;
+ close();
+ return false;
}
- // BZ-Header but wrong data ...
- if( !ok ) {
+ }
+
+ if( mode==zstd ) {
+#ifdef USE_ZSTD
+ bool ok = false;
+ fd->zbuff = xmalloc(LS_BUF_SIZE);
+
+ fd->dctx = ZSTD_createDCtx();
+ if( fd->dctx==NULL ) {
+ // zstd could not init
+ bool ok = false;
last_error = FILE_ERROR_BZ_CORRUPT;
close();
return false;
}
+ set_buffered( true );
+ fd->zin.size = 0;
+#else
+ dbg->fatal( "loadsave_t::rd_open", "Compiled without zstd support!" );
+#endif
}
- if( mode!=bzip2 ) {
+ if( !is_bzip2() && !is_zstd() ) {
fclose(fd->fp);
+ mode = zipped;
// and now with zlib ...
fd->gzfp = dr_gzopen(filename_utf8, "rb");
if(fd->gzfp==NULL) {
@@ -322,10 +361,26 @@
last_error = FILE_ERROR_GZ_CORRUPT;
return false;
}
- gzgets(fd->gzfp, buf, 80);
}
- saving = false;
+ if( read( buf, sizeof( SAVEGAME_PREFIX ) ) == sizeof( SAVEGAME_PREFIX ) ) {
+ // get the rest of the string
+ for( int i = sizeof( SAVEGAME_PREFIX ); i < 79; ) {
+ int ch = lsgetc();
+ if( ch < 32 ) {
+ break;
+ }
+ buf[ i++ ] = (char)ch;
+ buf[ i ] = 0;
+ }
+ }
+ else {
+ // could not even read start of file
+ last_error = FILE_ERROR_BZ_CORRUPT;
+ close();
+ return false;
+ }
+
if (strstart(buf, SAVEGAME_PREFIX)) {
version = int_version(buf + sizeof(SAVEGAME_PREFIX) - 1, pak_extension);
if( version == 0 ) {
@@ -417,14 +472,38 @@
last_error = FILE_ERROR_OK; // no error
close();
+ saving = true;
if( is_zipped() ) {
- // using zlib in lowest compression for highest speed (on servers)
- fd->gzfp = dr_gzopen(filename_utf8, "wb1");
+ // using zlib on servers, since 3x times faster saving than bz2
+ fd->gzfp = dr_gzopen(filename_utf8, "wb");
}
else if( mode==binary ) {
// no compression
fd->fp = dr_fopen(filename_utf8, "wb");
}
+ else if( is_zstd() ) {
+#ifdef USE_ZSTD
+ fd->cctx = ZSTD_createCCtx();
+ if( fd->cctx==NULL ) {
+ // zstd could not init
+ bool ok = false;
+ last_error = FILE_ERROR_BZ_CORRUPT;
+ close();
+ return false;
+ }
+ // in principe both below could fail ...
+ ZSTD_CCtx_setParameter( fd->cctx, ZSTD_c_compressionLevel, 3 );
+ ZSTD_CCtx_setParameter( fd->cctx, ZSTD_c_checksumFlag, 1 );
+ // XML or bzip ...
+ fd->fp = dr_fopen(filename_utf8, "wb");
+ fd->zbuff = xmalloc(LS_BUF_SIZE);
+ // the additional magic for zstd
+ fwrite( "ZD", 1, 2, fd->fp );
+ set_buffered( true );
+#else
+ dbg->fatal( "loadsave_t::rd_open", "Compiled without zstd support!" );
+#endif
+ }
else if( is_bzip2() ) {
// XML or bzip ...
fd->fp = dr_fopen(filename_utf8, "wb");
@@ -448,7 +527,6 @@
if( is_zipped() ? fd->gzfp == NULL : fd->fp == NULL ) {
return false;
}
- saving = true;
// get the right extension
const char *start = pak_extension;
@@ -502,6 +580,28 @@
const char *end = "\n\n";
write( end, strlen(end) );
}
+#ifdef USE_ZSTD
+ if( is_zstd() && fd->fp ) {
+ if( saving ) {
+ // write zero length dummy to indicate end of data
+ fd->zin = { "", 0, 0 };
+ fd->zout = { fd->zbuff, LS_BUF_SIZE, 0 };
+ size_t ret;
+ do {
+ fd->zout.pos = 0;
+ ret = ZSTD_compressStream2( fd->cctx, &(fd->zout), &(fd->zin), ZSTD_e_end );
+ fwrite( fd->zout.dst, 1, fd->zout.pos, fd->fp );
+ } while( ret>0 );
+ ZSTD_freeCCtx( fd->cctx );
+ mode = 0; // let default handle the closing errors ...
+ }
+ else {
+ ZSTD_freeDCtx( fd->dctx );
+ mode = zipped; // let zlib handle the closing errors ...
+ }
+ free( fd->zbuff );
+ }
+#endif
if( is_zipped() && fd->gzfp) {
int err_no;
const char *err_str = gzerror( fd->gzfp, &err_no );
@@ -547,21 +647,21 @@
*/
bool loadsave_t::is_eof()
{
- if( is_bzip2() ) {
- if( buffered ) {
+ if( is_bzip2() ) {
+ if( buffered ) {
bool r;
#ifdef MULTI_THREAD
- pthread_mutex_lock(&loadsave_mutex);
+ pthread_mutex_lock( &loadsave_mutex );
#endif
- r = buf_pos[0]>=buf_len[0] && buf_pos[1]>=buf_len[1] && fd->bse!=BZ_OK;
+ r = buf_pos[ 0 ] >= buf_len[ 0 ] && buf_pos[ 1 ] >= buf_len[ 1 ] && fd->bse != BZ_OK;
#ifdef MULTI_THREAD
- pthread_mutex_unlock(&loadsave_mutex);
+ pthread_mutex_unlock( &loadsave_mutex );
#endif
return r;
}
else {
// any error is EOF ...
- return fd->bse!=BZ_OK;
+ return fd->bse != BZ_OK;
}
}
else {
@@ -663,6 +763,22 @@
BZ2_bzWrite( &bse, fd->bzfp, ls_buf[buf_num], buf_pos[buf_num]);
assert(bse==BZ_OK);
}
+ else if( is_zstd() ) {
+#ifdef USE_ZSTD
+ size_t ret;
+ // first write, whatever remained in buffer
+ gzwrite( fd->gzfp, fd->zout.dst, fd->zout.pos );
+ // then compress the next data
+ fd->zin = { ls_buf[ buf_num ], buf_pos[ buf_num ], 0 };
+ while( fd->zin.pos < fd->zin.size ) {
+ fd->zout.pos = 0;
+ ret = ZSTD_compressStream2( fd->cctx, &(fd->zout), &(fd->zin), ZSTD_e_continue );
+ fwrite( fd->zout.dst, 1, fd->zout.pos, fd->fp );
+ }
+#else
+ dbg->fatal( "loadsave_t::flush_buffer", "Should never happen!" );
+#endif
+ }
else {
fwrite(ls_buf[buf_num], 1, buf_pos[buf_num], fd->fp);
}
@@ -735,23 +851,43 @@
}
-int loadsave_t::fill_buffer(int buf_num)
+int loadsave_t::fill_buffer( int buf_num )
{
int r;
int bse = fd->bse;
- if( is_bzip2() ) {
- if( bse==BZ_OK ) {
- r = BZ2_bzRead( &bse, fd->bzfp, ls_buf[buf_num], LS_BUF_SIZE);
- if ( bse != BZ_OK && bse != BZ_STREAM_END ) {
+ if( is_bzip2() ) {
+ if( bse == BZ_OK ) {
+ r = BZ2_bzRead( &bse, fd->bzfp, ls_buf[ buf_num ], LS_BUF_SIZE );
+ if( bse != BZ_OK && bse != BZ_STREAM_END ) {
r = -1; // an error occurred
}
}
else {
- assert(bse == BZ_STREAM_END);
+ assert( bse == BZ_STREAM_END );
r = 0;
}
}
+ else if( is_zstd() ) {
+ fd->zout = { ls_buf[ buf_num ], LS_BUF_SIZE, 0 };
+ do {
+ // first decompress from remaining input buffer
+ while( fd->zin.pos < fd->zin.size && fd->zout.pos < fd->zout.size ) {
+ size_t ret = ZSTD_decompressStream( fd->dctx, &fd->zout, &fd->zin );
+ if( ret == 0 ) {
+ fd->zout.size = fd->zout.pos;
+ }
+ }
+ // not enough data to fill buffer => read more ...
+ if( fd->zout.pos < fd->zout.size ) {
+ r = fread( (void *)(fd->zin.src), 1, LS_BUF_SIZE, fd->fp );
+ fd->zin.pos = 0;
+ fd->zin.size = r;
+ }
+ }
+ while( fd->zin.pos < fd->zin.size && fd->zout.pos < fd->zout.size );
+ r = fd->zout.pos; // number of bytes decompressed
+ }
else {
r = gzread(fd->gzfp, ls_buf[buf_num], LS_BUF_SIZE);
}
Index: trunk/dataobj/loadsave.h
===================================================================
--- trunk/dataobj/loadsave.h (revision 8887)
+++ trunk/dataobj/loadsave.h (working copy)
@@ -27,8 +27,6 @@
* Input format is automatically detected.
* Output format has a default, changeable with set_savemode, but can be
* overwritten in wr_open.
- *
- * @author V. Meyer, Hj. Malthaner
*/
@@ -41,7 +39,9 @@
zipped=4,
xml_zipped=6,
bzip2=8,
- xml_bzip2=10
+ xml_bzip2=10,
+ zstd=16,
+ xml_zstd=18
};
enum file_error_t {
@@ -131,6 +131,7 @@
bool is_saving() const { return saving; }
bool is_zipped() const { return mode&zipped; }
bool is_bzip2() const { return mode&bzip2; }
+ bool is_zstd() const { return mode&zstd; }
bool is_xml() const { return mode&xml; }
const char *get_pak_extension() const { return pak_extension; }
Index: trunk/gui/loadsave_frame.cc
===================================================================
--- trunk/gui/loadsave_frame.cc (revision 8887)
+++ trunk/gui/loadsave_frame.cc (working copy)
@@ -69,6 +69,7 @@
{
if(do_load) {
welt->switch_server( easy_server.pressed, true );
+ long start_load = dr_time();
if( !welt->load(filename) ) {
welt->switch_server( false, true );
}
@@ -75,6 +76,7 @@
else if( env_t::server ) {
welt->announce_server(0);
}
+ DBG_MESSAGE( "load world", "%li ms", dr_time() - start_load );
}
else {
// saving a game
@@ -89,7 +91,9 @@
// and now we need to copy the servergame to the map ...
#endif
}
+ long start_load = dr_time();
welt->save( filename, loadsave_t::save_mode, env_t::savegame_version_str, false );
+ DBG_MESSAGE( "save world", "%li ms", dr_time() - start_load );
welt->set_dirty();
welt->reset_timer();
}