data.cc

Go to the documentation of this file.
00001 ///
00002 /// \file       data.cc
00003 ///             Classes to help manage pre-determined data files.
00004 ///
00005 
00006 /*
00007     Copyright (C) 2005-2012, Net Direct Inc. (http://www.netdirect.ca/)
00008 
00009     This program is free software; you can redistribute it and/or modify
00010     it under the terms of the GNU General Public License as published by
00011     the Free Software Foundation; either version 2 of the License, or
00012     (at your option) any later version.
00013 
00014     This program is distributed in the hope that it will be useful,
00015     but WITHOUT ANY WARRANTY; without even the implied warranty of
00016     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
00017 
00018     See the GNU General Public License in the COPYING file at the
00019     root directory of this project for more details.
00020 */
00021 
00022 #include "data.h"
00023 #include <fstream>
00024 #include <sstream>
00025 #include <iomanip>
00026 #include <string>
00027 #include <stdexcept>
00028 #include <string.h>
00029 #include <stdlib.h>
00030 #include <locale>
00031 #include "ios_state.h"
00032 
00033 //#define __DEBUG_MODE__
00034 #include "debug.h"
00035 
00036 
00037 using namespace std;
00038 
00039 
00040 namespace Barry {
00041 
00042 inline bool IsHexData(const std::string &s)
00043 {
00044         const char *str = s.c_str();
00045         for( int i = 0; i < 4 && *str; str++, i++ )
00046                 if( *str != ' ' )
00047                         return false;
00048 
00049         for( int i = 0; i < 8 && *str; str++, i++ ) {
00050                 const char *hexchars = "0123456789abcdef";
00051                 if( strchr(hexchars, *str) == NULL )
00052                         return false;
00053         }
00054 
00055         if( *str != ':' )
00056                 return false;
00057 
00058         return true;
00059 }
00060 
00061 
00062 
00063 ///////////////////////////////////////////////////////////////////////////////
00064 // Data class
00065 
00066 bool Data::bPrintAscii = true;
00067 
00068 Data::Data()
00069         : m_memBlock(new unsigned char[0x4100])
00070         , m_blockSize(0x4100)
00071         , m_dataStart(m_memBlock + 0x100)
00072         , m_dataSize(0)
00073         , m_externalData(0)
00074         , m_external(false)
00075         , m_endpoint(-1)
00076 {
00077         memset(m_memBlock, 0, m_blockSize);
00078 }
00079 
00080 Data::Data(int endpoint, size_t startsize, size_t prependsize)
00081         : m_memBlock(new unsigned char[startsize + prependsize])
00082         , m_blockSize(startsize + prependsize)
00083         , m_dataStart(m_memBlock + prependsize)
00084         , m_dataSize(0)
00085         , m_externalData(0)
00086         , m_external(false)
00087         , m_endpoint(endpoint)
00088 {
00089         memset(m_memBlock, 0, m_blockSize);
00090 }
00091 
00092 Data::Data(const void *ValidData, size_t size)
00093         : m_memBlock(0)
00094         , m_blockSize(0)
00095         , m_dataStart(0)
00096         , m_dataSize(size)
00097         , m_externalData((const unsigned char*)ValidData)
00098         , m_external(true)
00099         , m_endpoint(-1)
00100 {
00101 }
00102 
00103 Data::Data(const Data &other)
00104         : m_memBlock(other.m_blockSize ? new unsigned char[other.m_blockSize] : 0)
00105         , m_blockSize(other.m_blockSize)
00106         , m_dataStart(m_memBlock + other.AvailablePrependSpace())
00107         , m_dataSize(other.m_dataSize)
00108         , m_externalData(other.m_externalData)
00109         , m_external(other.m_external)
00110         , m_endpoint(other.m_endpoint)
00111 {
00112         // copy over the raw data
00113         if( !m_external )
00114                 memcpy(m_memBlock, other.m_memBlock, other.m_blockSize);
00115 }
00116 
00117 Data::~Data()
00118 {
00119         delete [] m_memBlock;
00120 }
00121 
00122 //
00123 // MakeSpace
00124 //
00125 /// Reallocates buffers so that it is safe to write desiredsize data
00126 /// to m_dataStart after it returns.  All existing data is preserved.
00127 ///
00128 /// This function also performs any copy on write needed.
00129 ///
00130 /// If desiredprepend is nonzero, then at least desiredprepend bytes
00131 /// of prepend space will exist in the buffer after return.
00132 /// If desiredprepend is zero, defaults will be used if needed.
00133 ///
00134 void Data::MakeSpace(size_t desiredsize, size_t desiredprepend)
00135 {
00136         // use a default prepend size if none currently available
00137         size_t prepend = std::max(AvailablePrependSpace(), desiredprepend);
00138         if( !prepend )
00139                 prepend = 0x100;
00140 
00141         // GetBufSize() returns 0 if m_external is true
00142         if( GetBufSize() < (desiredsize + prepend) ||
00143             (desiredprepend && AvailablePrependSpace() < desiredprepend) )
00144         {
00145                 // get a proper chunk to avoid future resizes
00146                 desiredsize += 1024 + prepend;
00147 
00148                 // desired size must be at least the size of our current
00149                 // data (in case of external data), as well as the size
00150                 // of our desired prepend space
00151                 if( desiredsize < (m_dataSize + prepend) )
00152                         desiredsize = m_dataSize + prepend;
00153 
00154                 // setup new zeroed buffer... reuse m_memBlock if it
00155                 // exists (see operator=())
00156                 unsigned char *newbuf = 0;
00157                 if( m_memBlock && m_blockSize >= desiredsize ) {
00158                         newbuf = m_memBlock;
00159                 }
00160                 else {
00161                         newbuf = new unsigned char[desiredsize];
00162                         memset(newbuf, 0, desiredsize);
00163                 }
00164 
00165                 // copy valid data over
00166                 if( m_external ) {
00167                         memcpy(newbuf + prepend, m_externalData, m_dataSize);
00168 
00169                         // not external anymore
00170                         m_external = false;
00171                 }
00172                 else {
00173                         memcpy(newbuf + prepend, m_dataStart, m_dataSize);
00174                 }
00175 
00176                 // install new buffer if we've allocated a new one
00177                 if( m_memBlock != newbuf ) {
00178                         delete [] m_memBlock;
00179                         m_memBlock = newbuf;
00180                         m_blockSize = desiredsize;
00181                 }
00182 
00183                 // update m_dataStart
00184                 m_dataStart = m_memBlock + prepend;
00185         }
00186 }
00187 
00188 size_t Data::AvailablePrependSpace() const
00189 {
00190         if( m_external )
00191                 return 0;
00192         else
00193                 return m_dataStart - m_memBlock;
00194 }
00195 
00196 void Data::InputHexLine(istream &is)
00197 {
00198         ios_format_state state(is);
00199 
00200         unsigned int values[16];
00201         size_t index = 0;
00202 
00203         size_t address;
00204         is >> setbase(16) >> address;
00205         if( !is )
00206                 return;         // nothing to do
00207 
00208         is.ignore();            // eat the ':'
00209 
00210         while( is && index < 16 ) {
00211                 is >> setbase(16) >> values[index];
00212                 if( is )
00213                         index++;
00214         }
00215 
00216         dout("InputHexLine: read " << index << " bytes");
00217 
00218         MakeSpace(address + index);     // make space for the new
00219         m_dataSize = std::max(address + index, m_dataSize);
00220         while( index-- )
00221                 m_dataStart[address + index] = (unsigned char) values[index];
00222         return;
00223 }
00224 
00225 void Data::DumpHexLine(ostream &os, size_t index, size_t size) const
00226 {
00227         ios_format_state state(os);
00228 
00229         os.setf(ios::right);
00230 
00231         // index
00232         os << "    ";
00233         os << setbase(16) << setfill('0') << setw(8)
00234            << index << ": ";
00235 
00236         // hex byte data
00237         for( size_t i = 0; i < size; i++ ) {
00238                 if( (index+i) < GetSize() ) {
00239                         os << setbase(16) << setfill('0')
00240                            << setw(2) << setprecision(2)
00241                            << (unsigned int) GetData()[index + i] << ' ';
00242                 }
00243                 else {
00244                         os << "   ";
00245                 }
00246         }
00247 
00248         // printable data
00249         if( bPrintAscii ) {
00250                 locale loc = os.getloc();
00251                 os << ' ';
00252                 for( size_t i = 0; i < size && (index+i) < GetSize(); i++ ) {
00253                         ostream::traits_type::char_type c = GetData()[index + i];
00254                         os << setbase(10) << (char) (std::isprint(c, loc) ? c : '.');
00255                 }
00256         }
00257 
00258         os << "\n";
00259 }
00260 
00261 void Data::DumpHex(ostream &os) const
00262 {
00263         for( size_t address = 0; address < GetSize(); address += 16 ) {
00264                 DumpHexLine(os, address, 16);
00265         }
00266 }
00267 
00268 unsigned char * Data::GetBuffer(size_t requiredsize)
00269 {
00270         if( requiredsize == 0 ) {
00271                 // handle default, use data size
00272                 requiredsize = m_dataSize;
00273         }
00274 
00275         MakeSpace(requiredsize);
00276         return m_dataStart;
00277 }
00278 
00279 /// Returns size of buffer returned by GetBuffer().  Note that this does not
00280 /// include available prepend space.
00281 size_t Data::GetBufSize() const
00282 {
00283         if( m_external )
00284                 return 0;
00285         else
00286                 return m_blockSize - (m_dataStart - m_memBlock);
00287 }
00288 
00289 void Data::ReleaseBuffer(int datasize)
00290 {
00291         if( datasize < 0 && datasize != -1)
00292                 throw std::logic_error("Data::ReleaseBuffer() argument must be -1 or >= 0");
00293         if( m_external )
00294                 throw std::logic_error("Data::ReleaseBuffer() must be called after GetBuffer()");
00295         if( !(datasize == -1 || (unsigned int)datasize <= GetBufSize()) )
00296                 throw std::logic_error("Data::ReleaseBuffer() must be called with a size smaller than the original buffer requested");
00297 
00298         if( datasize >= 0 ) {
00299                 m_dataSize = datasize;
00300         }
00301         else {
00302                 // search for last non-zero value in buffer
00303                 m_dataSize = GetBufSize() - 1;
00304                 while( m_dataSize && m_dataStart[m_dataSize] == 0 )
00305                         --m_dataSize;
00306         }
00307 }
00308 
00309 /// Append bytes of data based on str
00310 void Data::AppendHexString(const char *str)
00311 {
00312         MakeSpace(m_dataSize + 512);
00313 
00314         std::istringstream iss(str);
00315         unsigned int byte;
00316         while( iss >> hex >> byte ) {
00317                 MakeSpace(m_dataSize + 1);
00318                 m_dataStart[m_dataSize] = (unsigned char) byte;
00319                 m_dataSize++;
00320         }
00321 }
00322 
00323 /// set buffer to 0 and remove all data
00324 void Data::Zap()
00325 {
00326         if( !m_external )
00327                 memset(m_memBlock, 0, m_blockSize);
00328         m_dataSize = 0;
00329 }
00330 
00331 Data & Data::operator=(const Data &other)
00332 {
00333         if( this == &other )
00334                 return *this;
00335 
00336         if( other.m_external ) {
00337                 // just copy over the pointers
00338                 m_externalData = other.m_externalData;
00339                 m_external = other.m_external;
00340                 m_dataSize = other.m_dataSize;
00341                 m_endpoint = other.m_endpoint;
00342         }
00343         else {
00344                 // don't remove our current buffer, only grow it if needed
00345                 MakeSpace(other.m_dataSize);
00346                 memcpy(m_dataStart, other.m_dataStart, other.m_dataSize);
00347 
00348                 // then copy over the data state
00349                 m_dataSize = other.m_dataSize;
00350                 m_endpoint = other.m_endpoint;
00351         }
00352 
00353         return *this;
00354 }
00355 
00356 void Data::MemCpy(size_t &offset, const void *src, size_t size)
00357 {
00358         unsigned char *pd = GetBuffer(offset + size) + offset;
00359         memcpy(pd, src, size);
00360         offset += size;
00361 
00362         // if the new end of data is larger than m_dataSize, bump it
00363         if( offset > m_dataSize )
00364                 m_dataSize = offset;
00365 }
00366 
00367 void Data::Append(const void *buf, size_t size)
00368 {
00369         // MemCpy updates m_datasize via the offset reference
00370         MemCpy(m_dataSize, buf, size);
00371 }
00372 
00373 void Data::Prepend(const void *buf, size_t size)
00374 {
00375         MakeSpace(0, size);
00376         m_dataStart -= size;
00377         m_dataSize += size;
00378         memcpy(m_dataStart, (const unsigned char*) buf, size);
00379 }
00380 
00381 /// Removes size bytes from the beginning of the buffer.
00382 /// If GetSize() is less than size, then all bytes will be chopped
00383 /// and GetSize() will end up 0.
00384 void Data::Prechop(size_t size)
00385 {
00386         // chopping all the bytes that we have?
00387         if( size >= GetSize() ) {
00388                 QuickZap();
00389                 return;
00390         }
00391 
00392         if( m_external ) {
00393                 m_externalData += size;
00394                 m_dataSize -= size;
00395         }
00396         else {
00397                 m_dataStart += size;
00398                 m_dataSize -= size;
00399         }
00400 }
00401 
00402 istream& operator>> (istream &is, Data &data)
00403 {
00404         data.InputHexLine(is);
00405         return is;
00406 }
00407 
00408 ostream& operator<< (ostream &os, const Data &data)
00409 {
00410         data.DumpHex(os);
00411         return os;
00412 }
00413 
00414 
00415 ///////////////////////////////////////////////////////////////////////////////
00416 // Diff class
00417 
00418 Diff::Diff(const Data &old, const Data &new_)
00419         : m_old(old), m_new(new_)
00420 {
00421 }
00422 
00423 void Diff::Compare(ostream &os, size_t index, size_t size) const
00424 {
00425         ios_format_state state(os);
00426 
00427         size_t min = std::min(m_old.GetSize(), m_new.GetSize());
00428 
00429         // index
00430         os << ">   ";
00431         os << setbase(16) << setfill('0') << setw(8)
00432            << index << ": ";
00433 
00434         // diff data
00435         for( size_t i = 0; i < size; i++ ) {
00436                 size_t address = index + i;
00437 
00438                 // if data is available, print the diff
00439                 if( address < min ) {
00440                         if( m_old.GetData()[address] != m_new.GetData()[address] ) {
00441                                 // differ, print hex
00442                                 os << setbase(16) << setfill('0')
00443                                    << setw(2) << setprecision(2)
00444                                    << (unsigned int) m_new.GetData()[address] << ' ';
00445                         }
00446                         else {
00447                                 // same, just print spaces
00448                                 os << "   ";
00449                         }
00450                 }
00451                 else {
00452                         // one of the buffers is shorter...
00453                         if( address < m_new.GetSize() ) {
00454                                 // new still has data, print it
00455                                 os << setbase(16) << setfill('0')
00456                                    << setw(2) << setprecision(2)
00457                                    << (unsigned int) m_new.GetData()[address]
00458                                    << ' ';
00459                         }
00460                         else if( address < m_old.GetSize() ) {
00461                                 // new is out of data and old still has some
00462                                 os << "XX ";
00463                         }
00464                         else {
00465                                 // no more data, just print spaces
00466                                 os << "   ";
00467                         }
00468                 }
00469         }
00470 
00471         // printable data, just dump new
00472         if( Data::PrintAscii() ) {
00473                 os << ' ';
00474                 for( size_t i = 0; i < size && (index+i) < m_new.GetSize(); i++ ) {
00475                         int c = m_new.GetData()[index + i];
00476                         os << setbase(10) << (char) (isprint(c) ? c : '.');
00477                 }
00478         }
00479 
00480         os << "\n";
00481 }
00482 
00483 void Diff::Dump(std::ostream &os) const
00484 {
00485         ios_format_state state(os);
00486 
00487         if( m_old.GetSize() != m_new.GetSize() )
00488                 os << "sizes differ: "
00489                    << m_old.GetSize() << " != " << m_new.GetSize() << endl;
00490 
00491         size_t max = std::max(m_old.GetSize(), m_new.GetSize());
00492         for( size_t i = 0; i < max; i += 16 ) {
00493                 m_old.DumpHexLine(os, i, 16);
00494                 Compare(os, i, 16);
00495         }
00496 }
00497 
00498 ostream& operator<< (ostream &os, const Diff &diff)
00499 {
00500         diff.Dump(os);
00501         return os;
00502 }
00503 
00504 
00505 ///////////////////////////////////////////////////////////////////////////////
00506 // DBData class
00507 
00508 /// Default constructor, constructs an empty local Data object
00509 DBData::DBData()
00510         : m_version(REC_VERSION_1)  // a reasonable default for now
00511         , m_localData(new Data)
00512         , m_data(*m_localData)
00513 {
00514 }
00515 
00516 /// Copy constructor - always creates an internal Data object, and
00517 /// uses Data object's copy constructor to make it.
00518 /// Copies all meta data as well.
00519 DBData::DBData(const DBData &other)
00520         : m_version(other.m_version)
00521         , m_dbName(other.m_dbName)
00522         , m_recType(other.m_recType)
00523         , m_uniqueId(other.m_uniqueId)
00524         , m_offset(other.m_offset)
00525         , m_localData(new Data(other.m_data))
00526         , m_data(*m_localData)
00527 {
00528 }
00529 
00530 /// Constructs a local Data object that points to external memory
00531 DBData::DBData(const void *ValidData, size_t size)
00532         : m_version(REC_VERSION_1)  // a reasonable default for now
00533         , m_localData(new Data)
00534         , m_data(*m_localData)
00535 {
00536 }
00537 
00538 DBData::DBData(RecordFormatVersion ver,
00539                 const std::string &dbName,
00540                 uint8_t recType,
00541                 uint32_t uniqueId,
00542                 size_t offset,
00543                 const void *ValidData,
00544                 size_t size)
00545         : m_version(ver)
00546         , m_dbName(dbName)
00547         , m_recType(recType)
00548         , m_uniqueId(uniqueId)
00549         , m_offset(offset)
00550         , m_localData(new Data(ValidData, size))
00551         , m_data(*m_localData)
00552 {
00553 }
00554 
00555 /// If copy == false, constructs an external Data object, no local.
00556 /// If copy == true, constructs an internal Data object copy
00557 DBData::DBData(Data &externalData, bool copy)
00558         : m_version(REC_VERSION_1)  // a reasonable default for now
00559         , m_localData(copy ? new Data(externalData) : 0)
00560         , m_data(copy ? *m_localData : externalData)
00561 {
00562 }
00563 
00564 DBData::DBData(RecordFormatVersion ver,
00565                 const std::string &dbName,
00566                 uint8_t recType,
00567                 uint32_t uniqueId,
00568                 size_t offset,
00569                 Data &externalData,
00570                 bool copy)
00571         : m_version(ver)
00572         , m_dbName(dbName)
00573         , m_recType(recType)
00574         , m_uniqueId(uniqueId)
00575         , m_offset(offset)
00576         , m_localData(copy ? new Data(externalData) : 0)
00577         , m_data(copy ? *m_localData : externalData)
00578 {
00579 }
00580 
00581 DBData::~DBData()
00582 {
00583         delete m_localData;
00584 }
00585 
00586 Data& DBData::UseData()
00587 {
00588         // make sure m_data is not external anymore
00589         m_data.GetBuffer();
00590         return m_data;  // return it
00591 }
00592 
00593 // Note: this copy operator does not change what m_data references...
00594 // whatever m_data references in the constructor is what will be changed
00595 // in this copy.
00596 // Note also that the copy *will* involve a memcpy, and maybe a memory
00597 // allocation as well.
00598 DBData& DBData::operator=(const DBData &other)
00599 {
00600         if( this == &other )
00601                 return *this;
00602 
00603         // copy the data block
00604         m_data = other.m_data;
00605 
00606         // copy the metadata
00607         CopyMeta(other);
00608 
00609         return *this;
00610 }
00611 
00612 ///////////////////////////////////////////////////////////////////////////////
00613 // Utility functions
00614 
00615 static bool IsEndpointStart(const std::string &line, int &endpoint)
00616 {
00617         if( strncmp(line.c_str(), "sep: ", 5) == 0 ||
00618             strncmp(line.c_str(), "rep: ", 5) == 0 )
00619         {
00620                 endpoint = atoi(line.c_str() + 5);
00621                 return true;
00622         }
00623         return false;
00624 }
00625 
00626 bool LoadDataArray(const string &filename, std::vector<Data> &array)
00627 {
00628         ifstream in(filename.c_str());
00629         return ReadDataArray(in, array);
00630 }
00631 
00632 bool ReadDataArray(std::istream &is, std::vector<Data> &array)
00633 {
00634         if( !is )
00635                 return false;
00636 
00637         bool bInEndpoint = false;
00638         unsigned int nCurrent = 0;
00639         size_t nLargestSize = 0x100;
00640         while( is ) {
00641                 string line;
00642                 getline(is, line);
00643                 int endpoint;
00644                 if( bInEndpoint ) {
00645                         if( IsHexData(line) ) {
00646                                 istringstream sline(line);
00647                                 sline >> array[nCurrent];
00648                                 continue;
00649                         }
00650                         else {
00651                                 nLargestSize = std::max(nLargestSize,
00652                                         array[nCurrent].GetBufSize());
00653                                 bInEndpoint = false;
00654                         }
00655                 }
00656 
00657                 // check if this line starts a new endpoint
00658                 if( IsEndpointStart(line, endpoint) ) {
00659                         bInEndpoint = true;
00660                         Data chunk(endpoint, nLargestSize);
00661                         array.push_back(chunk);
00662                         nCurrent = array.size() - 1;
00663                 }
00664         }
00665         return true;
00666 }
00667 
00668 } // namespace Barry
00669