diff --git a/dataobj/tabfile.cc b/dataobj/tabfile.cc index d352255f4..7698fa85b 100644 --- a/dataobj/tabfile.cc +++ b/dataobj/tabfile.cc @@ -17,7 +17,8 @@ #include "../descriptor/image.h" #include "koord.h" #include "tabfile.h" - +#include "../tpl/vector_tpl.h" +#include "../utils/csv.h" #define LINEBUFFER_SIZE (4096) @@ -856,3 +857,147 @@ void tabfile_t::format_key(char *key) } *t = '\0'; } + +void CSV_file_t::clear(){ + for(auto const & i : header) { + free(const_cast(i.key)); + } + header.clear(); + for(uint16 i = 0; i < data.get_data_count(); i++){ + koord dummypos; + const char *value; + data.get_nonzero(i,dummypos,value); + free(const_cast(value)); + } + data.reset(); +} + +void CSV_file_t::add_obj(const tabfileobj_t &obj){ + uint16 row=data.add_row(10)-1; + uint16 col; + for(auto it = obj.get_begin(); it != obj.get_end(); ++it){ + if(it->value.str==0){ + continue; + } + if(header.is_contained(it->key)){ + col=header.get(it->key); + }else{ + col=data.add_col()-1; + header.put(strdup(it->key),col); + } + data.set(col,row,strdup(it->value.str)); + } +} + +bool CSV_file_t::get_object(tabfileobj_t &obj){ + obj.clear(); + if(current_row >= data.get_size().y){ + return false; + } + for(auto it = header.begin(); it != header.end(); ++it){ + if(data.get(it->value,current_row)){ + obj.put(it->key,data.get(it->value,current_row)); + } + } + + current_row+=1; + return true; +} + +bool CSV_file_t::save_file(const char *filename){ + FILE* file; + if(filename){ + file=dr_fopen(filename,"w"); + }else{ + file=stdout; + } + if(!file) return false; + CSV_t csv; + vector_tpl indexes; + for(auto it = header.begin(); it != header.end(); ++it){ + indexes.append(it->value); + csv.add_field(it->key); + } + csv.new_line(); + for(uint16 j = 0; j < data.get_size().y; j++){ + for(uint16 i = 0; i < indexes.get_count(); i++){ + if(data.get(indexes[i],j)){ + csv.add_field(data.get(indexes[i],j)); + }else{ + csv.add_field(""); + } + } + csv.new_line(); + } + fputs(csv.get_str(),file); + if(filename){ + fclose(file); + } + return true; +} + +bool CSV_file_t::load_file(const char *filename){ + if(!filename) return false; + FILE* file=dr_fopen(filename,"r"); + if(!file) return false; + cbuffer_t filebuff; + const size_t subbuffsize=128; + char subbuff[subbuffsize]; + while(fgets(subbuff,subbuffsize,file) ){ + filebuff.append(subbuff,subbuffsize); + } + fclose(file); + + CSV_t csv(filebuff.get_str()); + + uint16 colcnt=0; + vector_tpl indexes; + for(;;){ + cbuffer_t headerbuffer; + headerbuffer.clear(); + int numsize = csv.get_next_field(headerbuffer); + if(numsize>0){ + if(header.is_contained(headerbuffer.get_str())){ + indexes.append(header.get(headerbuffer.get_str())); + }else{ + uint16 col = data.add_col()-1; + header.put(strdup(headerbuffer.get_str()),col); + indexes.append(col); + } + colcnt+=1; + }else if (numsize==0){ + indexes.append(0xFFFF); + colcnt+=1; + }else if (numsize==-1){ + break; + }else{ + return false; + } + } + + uint16 rowcnt=csv.get_lines()-1; + + for(uint16 j = 0; j < rowcnt; j++){ + if(!csv.next_line()){ + return true; + } + uint16 row=data.add_row(10)-1; + for(uint16 i = 0; i < colcnt; i++){ + cbuffer_t databuffer; + databuffer.clear(); + int status=csv.get_next_field(databuffer); + if(status>=0){ + if(indexes[i]!=0xFFFF){ + if(databuffer.len()){ + data.set(indexes[i],row,strdup(databuffer.get_str())); + } + } + }else if(status==-2){ + data.trim_row(); + return true; + } + } + } + + return true; +} diff --git a/dataobj/tabfile.h b/dataobj/tabfile.h index 9458444ec..e77fb8477 100644 --- a/dataobj/tabfile.h +++ b/dataobj/tabfile.h @@ -11,6 +11,7 @@ #include "../simcolor.h" #include "../tpl/stringhashtable_tpl.h" +#include "../tpl/sparse_tpl.h" class tabfileobj_t; class koord; @@ -27,6 +28,8 @@ public: obj_info_t(bool b, const char *s ) { retrieved=b; str=s; } }; + + /** * This class can be used instead of FILE to read a game definition file, * usually with extension .tab in simutrans. @@ -123,6 +126,14 @@ public: tabfileobj_t() { } ~tabfileobj_t() { clear(); } + stringhashtable_tpl::const_iterator get_begin() const { + return objinfo.begin(); + } + + stringhashtable_tpl::const_iterator get_end() const { + return objinfo.end(); + } + /** * prints all unused options lines in the file which do not start with a character from exclude_start_chars */ @@ -192,4 +203,25 @@ public: sint64 *get_sint64s(const char *key); }; +class CSV_file_t +{ +public: + CSV_file_t() {current_row=0;} + ~CSV_file_t() {clear();} + +public: + void clear(); + void add_obj(const tabfileobj_t& obj); + bool save_file(const char *filename); + + bool load_file(const char *filename); + void reset_current_obj() {current_row=0;} + bool get_object(tabfileobj_t& obj); + +private: + stringhashtable_tpl header; + sparse_tpl data; + uint16 current_row; +}; + #endif diff --git a/descriptor/writer/root_writer.cc b/descriptor/writer/root_writer.cc index ebc640f66..61a93a388 100644 --- a/descriptor/writer/root_writer.cc +++ b/descriptor/writer/root_writer.cc @@ -16,6 +16,8 @@ using std::string; string root_writer_t::inpath; +const koord koord::invalid(-1, -1); //needed for sparce_tpl.h + void root_writer_t::write_header(FILE* fp) { fprintf(fp, @@ -68,46 +70,29 @@ void root_writer_t::write(const char* filename, int argc, char* argv[]) if (infile.open(i)) { tabfileobj_t obj; - if (debuglevel >= log_t::LEVEL_WARN) { - printf(" Reading file %s\n", i); - } - - inpath = arg; - string::size_type n = inpath.rfind('/'); - - if(n!=string::npos) { - inpath = inpath.substr(0, n + 1); - } - else { - inpath = ""; - } + writer_init(i,arg); while(infile.read(obj)) { - if(separate) { - string name(filename); - - name = name + obj.get("obj") + "." + obj.get("name") + ".pak"; + writer_write(separate,filename,outfp,node,obj); + } + } + else { + dbg->warning( "Write pak", "Cannot read %s", i); + } + } + find.search(arg, "csv"); + FOR(searchfolder_t, const& i, find) { + CSV_file_t infile; - outfp = fopen(name.c_str(), "wb"); - if (!outfp) { - dbg->fatal( "Write pak", "Cannot create destination file %s", filename ); - } + if (infile.load_file(i)) { + tabfileobj_t obj; - if (debuglevel >= log_t::LEVEL_WARN) { - printf(" Writing file %s\n", name.c_str()); - } + writer_init(i,arg); - write_header(outfp); - node = new obj_node_t(this, 0, NULL); - } - obj_writer_t::write(outfp, *node, obj); - obj.unused( "#;-/" ); - if(separate) { - node->write(outfp); - delete node; - fclose(outfp); - } + infile.reset_current_obj(); + while(infile.get_object(obj)) { + writer_write(separate,filename,outfp,node,obj); } } else { @@ -122,6 +107,83 @@ void root_writer_t::write(const char* filename, int argc, char* argv[]) } } +void root_writer_t::writer_write(bool separate, const char *filename, FILE *outfp, obj_node_t *node, tabfileobj_t &obj){ + if(separate) { + string name(filename); + + name = name + obj.get("obj") + "." + obj.get("name") + ".pak"; + + outfp = fopen(name.c_str(), "wb"); + if (!outfp) { + dbg->fatal( "Write pak", "Cannot create destination file %s", filename ); + } + + if (debuglevel >= log_t::LEVEL_WARN) { + printf(" Writing file %s\n", name.c_str()); + } + + write_header(outfp); + node = new obj_node_t(this, 0, NULL); + } + obj_writer_t::write(outfp, *node, obj); + obj.unused( "#;-/" ); + + if(separate) { + node->write(outfp); + delete node; + fclose(outfp); + } +} + +void root_writer_t::writer_init(const char *i, const char *arg){ + if (debuglevel >= log_t::LEVEL_WARN) { + printf(" Reading file %s\n", i); + } + + inpath = arg; + string::size_type n = inpath.rfind('/'); + + if(n!=string::npos) { + inpath = inpath.substr(0, n + 1); + } + else { + inpath = ""; + } +} + +void root_writer_t::write_CSV(const char *filename, int argc, char *argv[]){ + searchfolder_t find; + FILE* outfp = NULL; + obj_node_t* node = NULL; + bool separate = false; + + CSV_file_t csv; + + for( int i=0; i==0 || i= log_t::LEVEL_WARN) { + printf(" Reading file %s\n", i); + } + + while(infile.read(obj)) { + csv.add_obj(obj); + } + } + else { + dbg->warning( "Write pak", "Cannot read %s", i); + } + } + } + csv.save_file(filename); +} void root_writer_t::write_obj_node_info_t(FILE* outfp, const obj_node_info_t &root) { diff --git a/descriptor/writer/root_writer.h b/descriptor/writer/root_writer.h index 5b18437f8..cbb987db8 100644 --- a/descriptor/writer/root_writer.h +++ b/descriptor/writer/root_writer.h @@ -27,6 +27,9 @@ class root_writer_t : public obj_writer_t { void write_header(FILE* fp); void write_obj_node_info_t(FILE* outfp, const obj_node_info_t &root); + void writer_init(const char* i, const char* arg); + void writer_write(bool separate, const char* filename, FILE* outfp, obj_node_t *node, tabfileobj_t& obj); + public: void capabilites(); static root_writer_t* instance() { return &the_instance; } @@ -37,6 +40,7 @@ class root_writer_t : public obj_writer_t { void dump(int argc, char* argv[]); void list(int argc, char* argv[]); void copy(const char* name, int argc, char* argv[]); + void write_CSV(const char* name, int argc, char* argv[]); /** * @brief Expands makeobj pre-processor stuff diff --git a/makeobj/Makefile b/makeobj/Makefile index 83de4c97f..e2e93cc34 100644 --- a/makeobj/Makefile +++ b/makeobj/Makefile @@ -128,6 +128,8 @@ SHARED_SOURCES += ../simmem.cc SHARED_SOURCES += ../utils/simstring.cc SHARED_SOURCES += ../utils/searchfolder.cc SHARED_SOURCES += ../utils/float32e8_t.cc +SHARED_SOURCES += ../utils/csv.cc +SHARED_SOURCES += ../utils/cbuffer_t.cc VARIANT_SOURCES += ../utils/simrandom.cc VARIANT_SOURCES += ../dataobj/tabfile.cc VARIANT_SOURCES += ../utils/log.cc diff --git a/makeobj/makeobj.cc b/makeobj/makeobj.cc index 3a85e482a..a5af4e51a 100644 --- a/makeobj/makeobj.cc +++ b/makeobj/makeobj.cc @@ -115,6 +115,27 @@ int main(int argc, char* argv[]) } } + if (argc && !STRICMP(argv[0], "tocsv")) { + argv++; argc--; + + try { + const char* dest; + if (argc) { + dest = argv[0]; + argv++; argc--; + } + else { + dest = 0; + } + root_writer_t::instance()->write_CSV(dest, argc, argv); + } + catch (const obj_pak_exception_t& e) { + dbg->error( e.get_class(), e.get_info() ); + return 1; + } + return 0; + } + if (argc && !STRICMP(argv[0], "expand")) { argv++; argc--; diff --git a/tpl/sparse_tpl.h b/tpl/sparse_tpl.h index 24be62c9c..daa45ea08 100644 --- a/tpl/sparse_tpl.h +++ b/tpl/sparse_tpl.h @@ -34,7 +34,19 @@ class sparse_tpl uint16 data_size; uint16 data_count; + uint16 row_size; + public: + sparse_tpl (){ + size = koord(0,0); + data_size=0; + data_count=0; + data = NULL; + col_ind = NULL; + row_ptr = NULL; + row_size=0; + } + sparse_tpl( koord _size ) { size = _size; data_size = 0; @@ -45,15 +57,52 @@ class sparse_tpl for( uint16 i = 0; i < size.y + 1; i++ ) { row_ptr[i] = 0; } + row_size = size.y + 1; } ~sparse_tpl() { - delete[] data; - data = NULL; - delete[] col_ind; - col_ind = NULL; - delete[] row_ptr; - row_ptr = NULL; + if(data){ + delete[] data; + } + if(col_ind){ + delete[] col_ind; + } + if(row_ptr){ + delete[] row_ptr; + } + } + + uint16 add_col(){ size.x++; return size.x; } + + uint16 add_row(uint16 prefetch=0){ + size.y+=1; + if(row_size < size.y+1){ + if(row_size==0){ + row_size = size.y + 1 + prefetch; + row_ptr = new uint16 [row_size]; + for(uint16 i = 0; i < size.y+1; i++){ + row_ptr[i] = 0; + } + }else{ + row_size = size.y + 1 + prefetch; + uint16* new_rows = new uint16 [row_size]; + for(uint16 i = 0; i< size.y; i++){ + new_rows[i]=row_ptr[i]; + } + new_rows[size.y]=new_rows[size.y-1]; + delete[] row_ptr; + row_ptr=new_rows; + } + }else{ + row_ptr[size.y]=row_ptr[size.y-1]; + } + return size.y; + } + + void trim_row(){ + if(size.y>0 && row_ptr[size.y]==row_ptr[size.y-1]){ + size.y-=1; + } } void clear() { @@ -64,6 +113,13 @@ class sparse_tpl resize_data(0); } + void reset(){ + data_count = 0; + delete [] row_ptr; + row_ptr=NULL; + size = koord(0,0); + } + const koord& get_size() const { return size; }