From: hama Date: Sun, 1 Mar 2015 23:05:04 +0000 (+0100) Subject: ReDirChecksum X-Git-Url: https://gitweb.hamatoma.de/?a=commitdiff_plain;h=8981926ab5f933de0adcdc586a0c46125e9bb954;p=crepublib ReDirChecksum --- diff --git a/base/ReByteBuffer.cpp b/base/ReByteBuffer.cpp index b08ebb6..9477746 100644 --- a/base/ReByteBuffer.cpp +++ b/base/ReByteBuffer.cpp @@ -231,7 +231,7 @@ ReByteBuffer& ReByteBuffer::appendFix(const char* data, size_t length, ReByteBuffer& ReByteBuffer::appendHexDump(const char* data, size_t length, int offset, int bytesPerLine, const char* offsetFormat, bool withAscii, int groupWidth, int gapBehind, const char* separator) { - if (length == -1) + if (length == (size_t) -1) length = strlen(data); if (gapBehind < 0) gapBehind = bytesPerLine / 2; diff --git a/base/ReByteBuffer.hpp b/base/ReByteBuffer.hpp index 6914d70..d18ff71 100644 --- a/base/ReByteBuffer.hpp +++ b/base/ReByteBuffer.hpp @@ -131,8 +131,8 @@ public: if (length == (size_t) -1) length = strlen(data); bool rc = length == m_length - && (!ignoreCase && _memcmp(data, m_buffer, length) == 0 - || ignoreCase && _strnicmp(data, m_buffer, length) == 0); + && ((!ignoreCase && _memcmp(data, m_buffer, length) == 0) + || (ignoreCase && _strnicmp(data, m_buffer, length) == 0)); return rc; } /** @brief Tests whether another instance is equal to this instance. @@ -143,9 +143,9 @@ public: inline bool equals(const ReByteBuffer& buffer, bool ignoreCase = false) const { bool rc = buffer.length() == m_length - && (!ignoreCase && _memcmp(buffer.str(), m_buffer, m_length) == 0 - || ignoreCase - && _strnicmp(buffer.str(), m_buffer, m_length) == 0); + && ((!ignoreCase && _memcmp(buffer.str(), m_buffer, m_length) == 0) + || (ignoreCase + && _strnicmp(buffer.str(), m_buffer, m_length) == 0)); return rc; } ReByteBuffer& fill(Byte filler = 0, int start = 0, int end = -1); diff --git a/base/ReDirectory.cpp b/base/ReDirectory.cpp index 5c7afde..83411c2 100644 --- a/base/ReDirectory.cpp +++ b/base/ReDirectory.cpp @@ -104,8 +104,9 @@ void ReDirectory::deleteTree(const char* base, bool deleteBaseToo) { do { struct stat info; const char* node = dir.currentFile(); - if (!(node[0] == '.' - && (node[1] == '\0' || node[1] == '.' && node[2] == '\0')) + if ((!(node[0] == '.' + && (node[1] == '\0' + || (node[1] == '.' && node[2] == '\0')))) && stat(dir.fullpath(full).str(), &info) == 0) { if (S_ISDIR(info.st_mode)) deleteTree(full.str(), true); diff --git a/base/ReHashList.cpp b/base/ReHashList.cpp index 5d8d18c..95be346 100644 --- a/base/ReHashList.cpp +++ b/base/ReHashList.cpp @@ -71,7 +71,7 @@ void ReHashList::dump(FILE* stream, const char* prefix) const { ReArrayPosition position; ReByteBuffer key, value; while (next(position, &key, &value)) { - fprintf(stream, "%s : [%3d] %s\n", key.str(), value.length(), + fprintf(stream, "%s : [%3d] %s\n", key.str(), (int) value.length(), value.str()); } } diff --git a/base/ReProgramArgs.cpp b/base/ReProgramArgs.cpp index 9280034..637caf9 100644 --- a/base/ReProgramArgs.cpp +++ b/base/ReProgramArgs.cpp @@ -338,7 +338,6 @@ void ReProgramArgs::search(char shortName, const char* longName, lengthLongName = strlen(longName); } while (!found && m_properties.next(position, &name, &properties)) { - const char* ptr = properties.str(); list.split(properties.str(), '\1'); if (longName == NULL && list.count() > IxShort && shortName == list.strOf(IxShort)[0]) diff --git a/base/ReStringList.cpp b/base/ReStringList.cpp index 89271a3..e08a453 100644 --- a/base/ReStringList.cpp +++ b/base/ReStringList.cpp @@ -42,7 +42,7 @@ ReStringList& ReStringList::append(const char* source, Tag tagOf) { * @return the instance itself (for chaining) */ ReStringList& ReStringList::append(const ReByteBuffer& source, Tag tag) { - int ix = add(-1, source.str(), source.length() + 1, tag); + add(-1, source.str(), source.length() + 1, tag); return *this; } diff --git a/base/ReTestUnit.cpp b/base/ReTestUnit.cpp index 542f558..27bac11 100644 --- a/base/ReTestUnit.cpp +++ b/base/ReTestUnit.cpp @@ -259,6 +259,8 @@ void ReTestUnit::createTestDir() { if (_mkdir(m_tempDir.str(), ALLPERMS) != 0) { error = getLastOSError(); } + if (error != 0) + logF(true, "cannot create directory: %s", m_tempDir.str()); ReDirectory::deleteTree(m_tempDir.str(), false); m_tempDir.ensureLastChar(OS_SEPARATOR_CHAR); } diff --git a/base/ReVarArgs.hpp b/base/ReVarArgs.hpp index c662585..8658ff0 100644 --- a/base/ReVarArgs.hpp +++ b/base/ReVarArgs.hpp @@ -46,8 +46,18 @@ public: ReVarArgs& arg(int64_t value, const char* format = "%lld"); ReVarArgs& arg(unsigned int arg, const char* format = "%u"); ReVarArgs& arg(uint64_t value, const char* format = "%llu"); - ReVarArgs& arg(const char* arg, int minWidth = 0, int maxWidth = 1024, + ReVarArgs& arg(const char* value, int minWidth = 0, int maxWidth = 1024, bool alignRight = false); + /** Appends an argument of the type ReByteBuffer. + * @param value argument value + * @param minWidth minimum width of the appended string. Padding with ' ' + * @param maxWidth maximum width of the string + * @param alignRight true: padding is done at the start + */ + inline ReVarArgs& arg(const ReByteBuffer& value, int minWidth = 0, + int maxWidth = 1024, bool alignRight = false){ + return arg(value.str(), minWidth, maxWidth, alignRight); + } ReVarArgs& arg(double arg, const char* format = "%f"); const char* asCString(); virtual void end(void); diff --git a/cunit/cuReDirTools.cpp b/cunit/cuReDirTools.cpp index b7e089a..2ee6c15 100644 --- a/cunit/cuReDirTools.cpp +++ b/cunit/cuReDirTools.cpp @@ -74,6 +74,7 @@ private: void run() { initTree(); if (m_testAll) { + testChecksumBuild(); testList2(); testToolStatistic(); testBasic(); @@ -86,6 +87,22 @@ private: testBatch(); } } + void testChecksumBuild(){ + ReByteBuffer nameCurrent; + buildFilename("current", nameCurrent); + ReByteBuffer optOutput("--output-file="); + optOutput.append(nameCurrent); + const char* argv[] = { "dt", "checksum", "-P;*;-cache", "-tr", "-p;*.txt", + "-s4711", "-arpd64", "-cBuild", + optOutput.str(), m_base.str(), NULL }; + ReDirTools tools; + tools.main(-1, (char**) argv); + // checkFile(nameCurrent, ""); + const char* argv2[] = { "dt", "checksum", "-P;*;-cache", "-tr", "-p;*.txt", + "-s4711", "-arpd64", "-cCompare", + optOutput.str(), m_base.str(), NULL }; + tools.main(-1, (char**) argv2); + } // F:\temp\retestunit\dirtool void testList2() { const char* argv[] = { "dt", "list", "-r", m_base.str(), NULL }; diff --git a/cunit/cuReRPD64.cpp b/cunit/cuReRPD64.cpp index 0c2e4b3..41676ac 100644 --- a/cunit/cuReRPD64.cpp +++ b/cunit/cuReRPD64.cpp @@ -28,14 +28,14 @@ private: const char* text = "01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVEXYZ01234567890\n"; RPD64.update((const uint8_t*) text, strlen(text)); - checkEqu("148517865cc2ca2c4ea2c2130a485769", RPD64.hexDigest()); + checkEqu("096a35ff313aeeace1de611a4c391eb5", RPD64.hexDigest()); RPD64.reset(); text = "The quick brown fox jumps over the lazy dog"; RPD64.update((const uint8_t*) text, -1); - checkEqu("064f9242cfaa1e270f9c28d7a90a6bcc", RPD64.hexDigest()); + checkEqu("a978d3ccb8c3e37ec715f30a3ea4f979", RPD64.hexDigest()); RPD64.reset(); RPD64.update((const uint8_t*) "", 0); - checkEqu("e602b42cc6d1e20b6b56f2b2f7542f38", RPD64.hexDigest()); + checkEqu("3c7b4e20d8ee3cf7716c2dacdddd9e30", RPD64.hexDigest()); } int testOneLong(ReByteBuffer& text, int seed2 = 0) { ReRPD64 RPD64; diff --git a/cunit/cuReTraverser.cpp b/cunit/cuReTraverser.cpp index fd94366..01e2c24 100644 --- a/cunit/cuReTraverser.cpp +++ b/cunit/cuReTraverser.cpp @@ -10,8 +10,6 @@ #include "base/rebase.hpp" #include "os/reos.hpp" -static const char* s_empty[] = { NULL }; - class TestReTraverser: public ReTestUnit { public: TestReTraverser() : diff --git a/cunit/testall.cpp b/cunit/testall.cpp index ddc73ae..f853161 100644 --- a/cunit/testall.cpp +++ b/cunit/testall.cpp @@ -78,7 +78,7 @@ void testMath() { } void testAll() { try { - testMath(); + testOs(); if (s_testAll) { testBase(); testString(); diff --git a/math/ReMD5.cpp b/math/ReMD5.cpp index 000252d..9c05b7c 100644 --- a/math/ReMD5.cpp +++ b/math/ReMD5.cpp @@ -34,14 +34,21 @@ const uint32_t ReMD5::m_K[RE_DIGEST_CHUNK_SIZE] = { * * @param digest the buffer for the binary checksum * @param digestSize the length of digest + * @param waiting a buffer for a chunk. Must have space for at least 2*chunksize bytes + * NULL: an intrinsic buffer will be used + * @param chunkSize the length of one full input block */ -ReDigest::ReDigest(uint8_t* digest, size_t digestSize) : +ReDigest::ReDigest(uint8_t* digest, size_t digestSize, uint8_t* waiting, + size_t chunkSize) : m_digest(digest), m_digestSize(digestSize), - // m_waiting[RE_DIGEST_CHUNK_SIZE]; + // m_waitingBuffer[RE_DIGEST_CHUNK_SIZE]; + m_waiting(waiting != NULL ? waiting : m_waitingBuffer), m_lengthWaiting(0), + m_chunkSize(chunkSize), m_length(0), - m_finalized(false) { + m_finalized(false), + m_salt(0) { } /** * Destructor. @@ -87,7 +94,7 @@ void ReDigest::update(const uint8_t* block, int blockLength) { // process the "waiting" input (incomplete chunk): m_length += blockLength; if (m_lengthWaiting > 0) { - int rest = m_digestSize - m_lengthWaiting; + int rest = m_chunkSize - m_lengthWaiting; if (rest > blockLength) rest = blockLength; memcpy(m_waiting + m_lengthWaiting, block, rest); @@ -95,17 +102,17 @@ void ReDigest::update(const uint8_t* block, int blockLength) { block += rest; m_lengthWaiting += rest; // Is the chunk complete? - if (m_lengthWaiting == RE_DIGEST_CHUNK_SIZE) { + if (m_lengthWaiting == m_chunkSize) { processChunk(m_waiting); m_lengthWaiting = 0; } } // process full 512 bit chunks (64 byte blocks): - for (int ix = blockLength / RE_DIGEST_CHUNK_SIZE; ix > 0; ix--) { + for (int ix = blockLength / m_chunkSize; ix > 0; ix--) { processChunk(block); - block += RE_DIGEST_CHUNK_SIZE; + block += m_chunkSize; } - blockLength %= RE_DIGEST_CHUNK_SIZE; + blockLength %= m_chunkSize; if (blockLength != 0) { assert(m_lengthWaiting == 0); memcpy(m_waiting, block, blockLength); @@ -113,6 +120,19 @@ void ReDigest::update(const uint8_t* block, int blockLength) { } } +/** + * Sets the salt of the checksum algorithm. + * + * Important: set the salt before you make the first update()! + * + * The salt makes that the checksum of the same input is (extremly) different + * to another salt. + * + * @param salt the salt to set + */ +void ReDigest::setSalt(uint64_t salt){ + m_salt = salt; +} /** * Constructor. */ @@ -157,7 +177,7 @@ void ReMD5::finalize() { memset(block + blockLength, 0, zeros); blockLength += zeros; //append original length in bits mod (2 pow 64) to message - uint64_t lengthBits = 8LL * m_length; + uint64_t lengthBits = 8LL * m_length + m_salt; #if defined __LITTLE_ENDIAN__ memcpy(block + blockLength, &lengthBits, 8); blockLength += 8; @@ -488,7 +508,7 @@ void ReMD5::reset() { m_c0 = 0x98badcfe; m_d0 = 0x10325476; memset(m_digest, 0, sizeof m_digest); - memset(m_waiting, 0, sizeof m_waiting); + memset(m_waiting, 0, m_chunkSize); m_lengthWaiting = 0; m_length = 0; m_hexDigest.setLength(0); diff --git a/math/ReMD5.hpp b/math/ReMD5.hpp index 427db8c..1a3695e 100644 --- a/math/ReMD5.hpp +++ b/math/ReMD5.hpp @@ -13,14 +13,23 @@ typedef uint8_t ReMD5Digest_t[16]; class ReDigest { public: - ReDigest(uint8_t* buffer, size_t bufferSize); + ReDigest(uint8_t* buffer, size_t bufferSize, uint8_t* waiting = NULL, + size_t blocksize = RE_DIGEST_CHUNK_SIZE); virtual ~ReDigest(); public: const uint8_t* digest(); const ReByteBuffer& hexDigest(); virtual void update(const uint8_t* block, int blockLength); + /** Updates the digest with data of any length. + * @param block the data to process + */ + inline void update(const ReByteBuffer& block) { + update(reinterpret_cast(block.str()), + (int) block.length()); + } virtual void processChunk(const uint8_t* block) = 0; virtual void reset() = 0; + virtual void setSalt(uint64_t salt); protected: virtual void finalize() = 0; protected: @@ -29,11 +38,14 @@ protected: ReByteBuffer m_hexDigest; // normally only the first chunk is used (64 byte), but while finalization // a 2nd chunk may be needed. - uint8_t m_waiting[RE_DIGEST_CHUNK_SIZE + RE_DIGEST_CHUNK_SIZE]; + uint8_t m_waitingBuffer[2 *RE_DIGEST_CHUNK_SIZE]; + uint8_t* m_waiting; int m_lengthWaiting; + int m_chunkSize; // total count of input bytes: uint32_t m_length; bool m_finalized; + uint64_t m_salt; }; /** diff --git a/math/ReRPD64.cpp b/math/ReRPD64.cpp index 5fa6a26..19598b0 100644 --- a/math/ReRPD64.cpp +++ b/math/ReRPD64.cpp @@ -12,17 +12,12 @@ * Constructor. */ ReRPD64::ReRPD64() : + ReDigest(m_digest, sizeof m_digest), //m_digest - // m_hexDigest; - // m_waiting[64]; - m_lengthWaiting(0), - m_length(0), m_a0(0), m_b0(0), m_c0(0), - m_d0(0), - m_finalized(false), - m_salt(1) { + m_d0(0) { reset(); } @@ -32,18 +27,6 @@ ReRPD64::ReRPD64() : ReRPD64::~ReRPD64() { } -/** - * Returns the binary digest value. - * - * @return the binary digest (16 byte array) - */ -const uint8_t* ReRPD64::digest() { - if (!m_finalized) { - finalize(); - } - return m_digest; -} - /** * Finalizes the digest. * @@ -111,20 +94,6 @@ void ReRPD64::finalize() { #endif } -/** - * Returns the binary digest value. - * - * @return the binary digest (16 byte array) - */ -const ReByteBuffer& ReRPD64::hexDigest() { - if (m_hexDigest.length() == 0) { - digest(); - for (int ix = 0; ix < 16; ix++) { - m_hexDigest.appendInt(m_digest[ix], "%02x"); - } - } - return m_hexDigest; -} #define F1(B, C, D) ((B & C) | (~ B & D)) #define F2(B, C, D) ((D & B) | (C & ~ D)) #define F3(B, C, D) ((B ^ C) ^ D) @@ -257,41 +226,4 @@ void ReRPD64::reset() { m_hexDigest.setLength(0); m_finalized = false; } -/** - * Processes a 64 byte block. - * - * @param block a block which should be added to the digest - * @param blockLength the length of block - */ -void ReRPD64::update(const uint8_t* block, int blockLength) { - if (blockLength == -1) - blockLength = strlen((const char*) block); - // process the "waiting" input (incomplete chunk): - m_length += blockLength; - if (m_lengthWaiting > 0) { - int rest = 64 - m_lengthWaiting; - if (rest > blockLength) - rest = blockLength; - memcpy(m_waiting + m_lengthWaiting, block, rest); - blockLength -= rest; - block += rest; - m_lengthWaiting += rest; - // Is the chunk complete? - if (m_lengthWaiting == 64) { - processChunk (m_waiting); - m_lengthWaiting = 0; - } - } - // process full 512 bit chunks (64 byte blocks): - for (int ix = blockLength / 64; ix > 0; ix--) { - processChunk(block); - block += 64; - } - blockLength %= 64; - if (blockLength != 0) { - assert(m_lengthWaiting == 0); - memcpy(m_waiting, block, blockLength); - m_lengthWaiting = blockLength; - } -} diff --git a/math/ReRPD64.hpp b/math/ReRPD64.hpp index 21ba5fa..8a4fa09 100644 --- a/math/ReRPD64.hpp +++ b/math/ReRPD64.hpp @@ -14,34 +14,22 @@ typedef uint8_t ReRPD64Digest[16]; * * The algorithm is similar to MD5, but calculation is done in 64-bit. */ -class ReRPD64 { +class ReRPD64 : public ReDigest { public: ReRPD64(); virtual ~ReRPD64(); public: - const uint8_t* digest(); - const ReByteBuffer& hexDigest(); - void update(const uint8_t* block, int blockLength); - void processChunk(const uint8_t block[64]); - void reset(); - void setSalt(uint64_t salt); + virtual void processChunk(const uint8_t*_block); + virtual void reset(); private: - void finalize(); + virtual void finalize(); private: ReRPD64Digest m_digest; - ReByteBuffer m_hexDigest; - // normally only the first chunk is used (64 byte), but while finalization - // a 2nd chunk may be needed. - uint8_t m_waiting[64 + 64]; - int m_lengthWaiting; - // total count of input bytes: - uint64_t m_length; uint64_t m_a0; uint64_t m_b0; uint64_t m_c0; uint64_t m_d0; - bool m_finalized; uint64_t m_salt; }; diff --git a/os/ReDirTools.cpp b/os/ReDirTools.cpp index 5d2825e..ca9c86e 100644 --- a/os/ReDirTools.cpp +++ b/os/ReDirTools.cpp @@ -8,8 +8,8 @@ */ #include "base/rebase.hpp" -#include "os/reos.hpp" #include "math/remath.hpp" +#include "os/reos.hpp" enum LOCATION_DIRTOOL { LC_COPY_FILE_1 = LC_DIRTOOLS + 1, // 50101 @@ -24,6 +24,16 @@ enum LOCATION_DIRTOOL { LC_SET_PROPERTIES_2, // 50110 LC_SET_PROPERTIES_3, // 50111 LC_TOUCH_1, // 50112 + LC_CALCULATE_CHECKSUM_1, // 50113 + LC_BUILD_DIRECTORY_1, // 50114 + LC_COMPARE_STORAGE_1, // 50115 + LC_COMPARE_STORAGE_2, // 50116 + LC_COMPARE_STORAGE_3, // 50117 + LC_COMPARE_STORAGE_4, // 50118 + LC_COMPARE_STORAGE_5, // 50119 + LC_COMPARE_STORAGE_6, // 50120 + LC_COMPARE_STORAGE_7, // 50121 + LC_COMPARE_DIR_1, // 50122 }; const char* ReDirTools::m_version = "2015.02.25"; ReLogger* ReDirTools::m_logger = NULL; @@ -32,9 +42,10 @@ static const char* s_helpSummary[] = { "dirtool or dt ", " Useful commands around directory trees.", " Type 'dirtool help ' for more help.", ":", "batch produce output to handle the found files with a script", + "checksum shows the checksum (MD5 or other) of the selected files", + "delete delete the selected files", "help shows info about the arguments/options", "list shows the meta data of the selected files", - "md5 shows the MD5 checksum of the selected files", "statistic shows statistics about a direcctory tree", "synchronize copies only modified or new files from", " from a source directory tre to a target", @@ -64,12 +75,15 @@ const char* s_listExamples[] = "dirtool li --type=f -y7d --size=10M -p;*.cpp;*.hpp;Makefile*;-*~ /home/data", NULL }; -const char* s_md5Usage[] = { - ": m(d5) [] [ ...]", - " shows the MD5 check sum of the given files", +const char* s_checksumUsage[] = { + ": c(hecksum) [] [ ...]", + " shows a check sum of the given files or manages a checksum storage file in each directory for observing", NULL }; -const char* s_md5Examples[] = { "dirtool md5 --buffer-size=512 e:\\data", - "dirtool m --type=f --size=10M -p;*.iso /home/iso /down/debian.iso", +const char* s_checksumExamples[] = { "dirtool ch --buffer-size=512 e:\\data", + "dirtool check -cList -p;*.iso /home/iso /down/debian.iso", + "dirtool checksum -cBuild -aMD5 /home", + "dirtool checksum --command=Compare /home", + "dirtool checksum -cUpdate -O/var/log/checksum.log /home", NULL }; static const char* s_statisticUsage[] = @@ -521,7 +535,7 @@ void ReDirOptions::checkStandardFilterOptions() { if (m_programArgs.getString("verbose", buffer)[0] != '\0') { unsigned level = V_NORMAL; if (ReStringUtils::lengthOfUnsigned(buffer.str(), -1, &level) - != buffer.length()) + != (int) buffer.length()) help(i18n("verbose level is not a number (or '')"), buffer.str()); else m_verboseLevel = VerboseLevel(level); @@ -1062,6 +1076,334 @@ void ReDirBatch::processFile(ReDirStatus_t* entry) { fprintf(m_output, "%s\n", line.str()); } +/** + * Constructor. + * + * @param logger logger for error handling + */ +ReDirChecksum::ReDirChecksum(ReLogger* logger) : + ReTool(s_checksumUsage, s_checksumExamples, 0, 0, 0, true, logger), + m_command(CMD_LIST), + m_digest(NULL), + m_buffer() { + // standard short options: D d O o P p T t v y Z z + m_programArgs.addString("algorithm", + i18n("algorithm: 'MD5' or 'RPD64'"), 'a', "--algorithm", false, "MD5"); + m_programArgs.addInt("buffersize", + i18n("buffer size for file reading (in KiByte)"), 'b', "--buffer-size", + 4 * 1024); + m_programArgs.addString("command", + i18n("'build': builds in each directory a file '.dt.chksum' with sum and name\n" + "'compare': shows differences between the content of '.dt.chksum' and the current files\n" + "'list': shows checksum and filename\n" + "'update': like compare, but update the content of '.dt.chksum'"), + 'c', "--command", false, "list"); + m_programArgs.addInt("salt", + i18n("a number which change the checksum"), 's', "--salt", 0); + addStandardFilterOptions(); +} + +/** + * Destructor. + */ +ReDirChecksum::~ReDirChecksum(){ + delete m_digest; + m_digest = NULL; +} +/** + * Builds a checksum storage for a given directory. + * + * @param path the directory to process + */ +void ReDirChecksum::buildStorage(const char* path, const char* storageFile){ + ReTraverser traverser(path); + traverser.setMaxLevel(0); + ReDirStatus_t* entry; + ReRPD64 digest2; + digest2.setSalt(0x2004199111121989ll); + ReByteBuffer line; + FILE* fp = fopen(storageFile, "w"); + if (fp == NULL){ + m_logger->sayF(LOG_ERROR | CAT_FILE, LC_BUILD_DIRECTORY_1, + i18n("cannot open file: $1 (errno: $2)")).arg(storageFile).arg( + errno).end(); + + } else { + int level; + while ((entry = traverser.nextFile(level, &m_filter)) != NULL) { + line.setLength(0); + if (!entry->isDirectory()){ + calculateChecksum(entry->fullName(), *m_digest, m_buffer, + m_logger); + line.append(m_digest->hexDigest()); + } + line.appendChar('\t').append(entry->node()); + digest2.update(line); + fprintf(fp, "%s\n", line.str()); + if (m_verboseLevel >= V_NORMAL) + fprintf(m_output, "%16s\t\%s", + entry->isDirectory() ? "" : m_digest->hexDigest().str(), + entry->fullName()); + } + fprintf(fp, "%s\n", digest2.hexDigest().str()); + fclose(fp); + } +} + +/** + * Compares the files of a directory with the files in the storage file + * + * @param path the directory + * @param storageFile the file with the checksums + */ +void ReDirChecksum::compareDir(const char* path, ReStringList& names, + const char* storageFile){ + // forget the last line (checksum): + int count = names.count() - 1; + names.remove(count - 1); + for (int ii = 0; ii < count; ii++){ + const char* name = names.strOf(ii); + const char* ptr = strchr(name, '\t'); + if (ptr != NULL) + names.replaceString(ii, ptr + 1); + } + names.sort(); + names.setSorted(true); + ReTraverser traverser(path); + traverser.setMaxLevel(0); + ReDirStatus_t* entry; + int level; + while ((entry = traverser.nextFile(level, &m_filter)) != NULL) { + ReSeqArray::Index index; + const char* node = entry->node(); + // the stringlist stores the string with trailing '\0' + if (! names.binarySearch(node, strlen(node) + 1, index)){ + m_logger->sayF(LOG_ERROR | CAT_SECURITY, LC_COMPARE_DIR_1, + i18n("missing file $1 in storage $2")).arg(entry->fullName()) + .arg(storageFile).end(); + } + } +} + +/** + * Compares a storage file with the files of a directory. + * + * @param path the directory + * @param storageFile the file with the checksums + */ +void ReDirChecksum::compareStorage(const char* path, const char* storageFile){ + ReStringList storage; + storage.readFromFile(storageFile); + if (storage.count() <= 0){ + m_logger->sayF(LOG_ERROR | CAT_SECURITY, LC_COMPARE_STORAGE_1, + i18n("empty storage file: $1")).arg(storageFile).end(); + } else if (! isValid(storage)){ + m_logger->sayF(LOG_ERROR | CAT_SECURITY, LC_COMPARE_STORAGE_2, + i18n("corrupted storage file: $1")).arg(storageFile).end(); + } else { + ReStringList cols; + ReByteBuffer fullname(path); + fullname.ensureLastChar(OS_SEPARATOR_CHAR); + int pathLength = fullname.length(); + struct stat info; + // test all files of the storage: + for (size_t ii = 0; ii < storage.count() - 1; ii++){ + cols.split(storage.strOf(ii), '\t'); + int colCount = cols.count(); + if (colCount != 2){ + m_logger->sayF(LOG_ERROR | CAT_SECURITY, LC_COMPARE_STORAGE_3, + i18n("wrong format ($1) in storage file $2-$3")) + .arg(colCount).arg(storageFile).arg(ii+1).end(); + break; + } else { + fullname.setLength(pathLength).append(cols.strOf(1)); + if (cols.strLengthOf(0) == 0){ + // a directory: + if (stat(fullname.str(), &info) != 0){ + m_logger->sayF(LOG_ERROR | CAT_SECURITY, + LC_COMPARE_STORAGE_4, + i18n("missing directory $1 ($2-$3)")) + .arg(fullname).arg(storageFile).arg(ii+1).end(); + } else if (! S_ISDIR(info.st_mode)){ + m_logger->sayF(LOG_ERROR | CAT_SECURITY, + LC_COMPARE_STORAGE_5, + i18n("directory is now a file: $1 ($2-$3)")) + .arg(fullname).arg(storageFile).arg(ii+1).end(); + } + } else { + // a directory: + if (stat(fullname.str(), &info) != 0){ + m_logger->sayF(LOG_ERROR | CAT_SECURITY, + LC_COMPARE_STORAGE_6, + i18n("missing file $1 ($2-$3)")) + .arg(fullname).arg(storageFile).arg(ii+1).end(); + } else if (S_ISDIR(info.st_mode)){ + m_logger->sayF(LOG_ERROR | CAT_SECURITY, + LC_COMPARE_STORAGE_7, + i18n("file is now a directory: $1 ($2-$3)")) + .arg(fullname).arg(storageFile).arg(ii+1).end(); + } else { + calculateChecksum(fullname.str(), *m_digest, m_buffer, + m_logger); + if (! m_digest->hexDigest().equals(cols.strOf(0))){ + m_logger->sayF(LOG_ERROR | CAT_SECURITY, + LC_COMPARE_STORAGE_7, + i18n("checksum different: $1 ($2-$3)")) + .arg(fullname).arg(storageFile).arg(ii+1).end(); + } + } + } + } + } + compareDir(path, storage, storageFile); + } +} +/** + * Checks the validity of a storage file. + * + * The storage file is protected with a checksum. + * The checksum has another salt as the checksums of the files. This makes it + * much harder to manipulate the storage file. + * + * @param storage the storage file as string list. + */ +bool ReDirChecksum::isValid(const ReStringList& storage){ + int count = storage.count(); + ReRPD64 digest2; + digest2.setSalt(0x2004199111121989ll); + + for (int ii = 0; ii < count - 1; ii++){ + const char* line = storage.strOf(ii); + digest2.update(line); + } + bool rc = true; + const char* hex = storage.strOf(count - 1); + if (! digest2.hexDigest().equals(hex)) + rc = false; + return rc; +} +/** + * Lists the metadata of the specified files. + */ +void ReDirChecksum::doIt() { + int size = m_programArgs.getInt("buffersize") * 1024; + m_buffer.setLength(size); + ReByteBuffer value; + m_programArgs.getString("command", value); + if (value.equals("list", -1, true)) + m_command = CMD_LIST; + else if (value.equals("build", -1, true)) + m_command = CMD_BUILD; + else if (value.equals("compare", -1, true)) + m_command = CMD_COMPARE; + else if (value.equals("update", -1, true)) + m_command = CMD_UPDATE; + else + help(i18n("unknown command (expected: build compare list update): "), + value.str()); + + m_programArgs.getString("algorithm", value); + if (value.equals("md5", -1, true)) + m_digest = new ReMD5(); + else if (value.equals("rpd64", -1, true)) + m_digest = new ReRPD64(); + else + help(i18n("unknown algorithm (expected: MD5 RPD64): "), + value.str()); + int salt = m_programArgs.getInt("salt"); + if (salt != 0) + m_digest->setSalt(salt); + m_filter.m_allDirectories = true; + processFileArguments(); + printSummary(); +} + +/** + * Processes one directory. + * + * @param entry the properties of the directory to process + */ +void ReDirChecksum::processDir(ReDirStatus_t* entry) { + ReByteBuffer storageFile; + if (m_command != CMD_LIST) { + storageFile.append(entry->fullName()); + storageFile.ensureLastChar(OS_SEPARATOR_CHAR); + storageFile.append(".dt.chksum"); + } + switch(m_command){ + case CMD_BUILD: + buildStorage(entry->fullName(), storageFile.str()); + break; + case CMD_COMPARE: + compareStorage(entry->fullName(), storageFile.str()); + break; + case CMD_UPDATE: + printf("update not implemented"); + break; + default: + break; + } +} +/** + * Processes one file. + * + * @param entry the properties of the file to process + */ +void ReDirChecksum::processFile(ReDirStatus_t* entry) { + const char* name = entry->fullName(); + switch(m_command){ + case CMD_LIST: + calculateChecksum(name, *m_digest, m_buffer, m_logger); + fprintf(m_output, "%s %s\n", m_digest->hexDigest().str(), name); + break; + default: + break; + } +} +/** + * Compares a storage file with the files of a directory. + * + * @param path the directory + * @param storageFile the file with the checksums + */ +void ReDirChecksum::updateStorage(const char* path, const char* storageFile){ + +} + +/** + * Calculates the checksum of a given file. + * + * @param name the full filename + * @param digest IN: defines the kind of checksum
+ * OUT: contains the calculated checksum + * @param buffer IN/OUT: a buffer for file reading + * @param logger logger for error processing + * @return digest (for chaining) + */ +ReDigest& ReDirChecksum::calculateChecksum(const char* name, + ReDigest& digest, ReByteBuffer& buffer, ReLogger* logger){ + FILE* fp = fopen(name, "rb"); + if (fp == NULL) { + if (logger != NULL) + logger->sayF(LOG_ERROR | CAT_FILE, LC_CALCULATE_CHECKSUM_1, + i18n("cannot open file: $1 (errno: $2)")).arg(name).arg( + errno).end(); + } else { + ReMD5 digest; + size_t readBytes; + uint8_t* buf = reinterpret_cast(buffer.buffer()); + size_t blockSize = buffer.length(); + + while ((readBytes = fread(buf, 1, blockSize, fp)) > 0) { + digest.update(buf, readBytes); + } + fclose(fp); + } + return digest; +} + + + /** * Constructor. * @@ -1126,60 +1468,6 @@ void ReDirList::processFile(ReDirStatus_t* entry) { } } -/** - * Constructor. - * - * @param logger logger for error handling - */ -ReDirMD5::ReDirMD5(ReLogger* logger) : - ReTool(s_md5Usage, s_md5Examples, 0, 0, 0, true, logger), - m_buffer() { - // standard short options: D d O o P p T t v y Z z - m_programArgs.addInt("buffersize", - i18n("buffer size for file reading (in KiByte)"), 'b', "--buffer-size", - 4 * 1024); - addStandardFilterOptions(); -} - -/** - * Lists the metadata of the specified files. - */ -void ReDirMD5::doIt() { - int size = m_programArgs.getInt("buffersize") * 1024; - m_buffer.setLength(size); - processFileArguments(); - printSummary(); -} - -/** - * Processes one directory. - * - * @param entry the properties of the directory to process - */ -void ReDirMD5::processDir(ReDirStatus_t* entry) { -} -/** - * Processes one file. - * - * @param entry the properties of the file to process - */ -void ReDirMD5::processFile(ReDirStatus_t* entry) { - const char* name = entry->fullName(); - FILE* fp = fopen(name, "rb"); - if (fp != NULL) { - ReMD5 digest; - size_t readBytes; - uint8_t* buffer = reinterpret_cast(m_buffer.buffer()); - size_t blockSize = m_buffer.length(); - - while ((readBytes = fread(buffer, 1, blockSize, fp)) > 0) { - digest.update(buffer, readBytes); - } - fclose(fp); - fprintf(m_output, "%s %s\n", digest.hexDigest().str(), name); - } -} - /** * Calculates the statistic of a directory tree. * @@ -1475,7 +1763,10 @@ void ReDirTouch::processFile(ReDirStatus_t* entry) { ReDirStatus_t::filetimeToString( filetimeIsUndefined(m_modified) ? &m_accessed : &m_modified, bufferTime); - fprintf(m_output, "%s -> %s %s\n", bufferTime.str(), name); + fprintf(m_output, "%s -> %s %s\n", + ReDirStatus_t::filetimeToString(entry->modified(), + bufferTime2), + bufferTime.str(), (char*) name); } } } @@ -1882,9 +2173,7 @@ void ReDirSync::doIt() { m_statInfo.st_mtime - entry->filetimeToTime(entry->modified())); if (!ignoreDate - && m_statInfo.st_mtime - - entry->filetimeToTime(entry->modified()) - <= maxFileTimeDiff) { + && diff <= maxFileTimeDiff) { if (m_verboseLevel >= V_CHATTER) fprintf(m_output, "=ignored: %s same time\n", targetRelativePath); @@ -2018,14 +2307,14 @@ void ReDirTools::help(int argc, const char* argv[]) { if (isArg("batch", arg0)) { ReDirBatch batch(m_logger); batch.help(NULL); + } else if (isArg("checksum", arg0)) { + ReDirChecksum sum(m_logger); + sum.help(NULL); } else if (isArg("list", arg0)) { ReDirList list(m_logger); list.help(NULL); } else if (isArg("help", arg0)) { printField(s_helpSummary); - } else if (isArg("md5", arg0)) { - ReDirMD5 md5(m_logger); - md5.help(NULL); } else if (isArg("statistic", arg0)) { ReDirStatistic stat(m_logger); stat.help(NULL); @@ -2061,6 +2350,11 @@ void ReDirTools::run(int argc, const char* argv[], ReTool& tool) { */ int ReDirTools::main(int argc, char* orgArgv[]) { ReDirTools tools; + if (argc < 0){ + argc = 0; + for (int ix = 0; orgArgv[ix] != NULL; ix++) + argc++; + } const char** argv = (const char**) orgArgv; if (argc < 2) { tools.help(0, argv); @@ -2072,12 +2366,12 @@ int ReDirTools::main(int argc, char* orgArgv[]) { const char* arg0 = argv[0]; if (isArg("batch", arg0)) ReDirBatch(m_logger).run(argc, argv); + else if (isArg("checksum", arg0)) + ReDirChecksum(m_logger).run(argc, argv); else if (isArg("help", arg0)) tools.help(argc, argv); else if (isArg("list", arg0)) ReDirList(m_logger).run(argc, argv); - else if (isArg("md5", arg0)) - ReDirMD5(m_logger).run(argc, argv); else if (isArg("statistic", arg0)) ReDirStatistic(m_logger).run(argc, argv); else if (isArg("synchronize", arg0)) diff --git a/os/ReDirTools.hpp b/os/ReDirTools.hpp index 9e7cfe8..3777e5c 100644 --- a/os/ReDirTools.hpp +++ b/os/ReDirTools.hpp @@ -162,18 +162,42 @@ protected: bool m_numerical; }; - +class ReDigest; /** - * Calculates the MD5 checksum of files. + * Calculates / compares the checksum of files. + * + * MD5 or a RPD64 (similar to MD5, but calculated in 64 bit) can be used + * as algorithms. */ -class ReDirMD5: public ReTool { +class ReDirChecksum: public ReTool { +public: + enum Command { + CMD_UNDEF, + CMD_BUILD, + CMD_COMPARE, + CMD_LIST, + CMD_UPDATE + }; public: - ReDirMD5(ReLogger* logger); + ReDirChecksum(ReLogger* logger); + ~ReDirChecksum(); protected: virtual void doIt(); virtual void processDir(ReDirStatus_t* entry); virtual void processFile(ReDirStatus_t* entry); +private: + void buildStorage(const char* path, const char* storageFile); + void compareDir(const char* path, ReStringList& names, + const char* storageFile); + void compareStorage(const char* path, const char* storageFile); + bool isValid(const ReStringList& storage); + void updateStorage(const char* path, const char* storageFile); +public: + static ReDigest& calculateChecksum(const char* name, + ReDigest& digest, ReByteBuffer& buffer, ReLogger* logger); protected: + Command m_command; + ReDigest* m_digest; ReByteBuffer m_buffer; }; diff --git a/os/ReTraverser.cpp b/os/ReTraverser.cpp index 344da80..ada7afa 100644 --- a/os/ReTraverser.cpp +++ b/os/ReTraverser.cpp @@ -578,7 +578,8 @@ ReDirEntryFilter_t::ReDirEntryFilter_t() : //m_minAge(0), //m_maxAge(0), m_minDepth(0), - m_maxDepth(512) { + m_maxDepth(512), + m_allDirectories(false){ setFiletimeUndef(m_minAge); setFiletimeUndef(m_maxAge); } @@ -594,6 +595,10 @@ ReDirEntryFilter_t::~ReDirEntryFilter_t() { bool ReDirEntryFilter_t::match(ReDirStatus_t& entry) { bool rc = false; do { + if (m_allDirectories && entry.isDirectory()){ + rc = true; + break; + } if (0 == (entry.type() & m_types)) break; int64_t size = entry.fileSize(); @@ -691,7 +696,7 @@ ReTraceUnit::~ReTraceUnit() { * @return true (for chaining) */ bool ReTraceUnit::trace(const char* message) { - printf("%d\n", message); + printf("%s\n", message); return true; } diff --git a/os/ReTraverser.hpp b/os/ReTraverser.hpp index 75ddb62..d97add5 100644 --- a/os/ReTraverser.hpp +++ b/os/ReTraverser.hpp @@ -122,6 +122,7 @@ public: ReFileTime_t m_maxAge; int m_minDepth; int m_maxDepth; + bool m_allDirectories; }; class ReTraceUnit { public: diff --git a/os/reos.hpp b/os/reos.hpp index 68f63c7..ccc7b8a 100644 --- a/os/reos.hpp +++ b/os/reos.hpp @@ -30,11 +30,12 @@ */ inline bool operator >(const ReFileTime_t& time1, const ReFileTime_t& time2) { #if defined __linux__ - return time1.tv_sec > time2.tv_sec || time1.tv_sec == time2.tv_sec && time1.tv_nsec > time2.tv_nsec; + return time1.tv_sec > time2.tv_sec + || (time1.tv_sec == time2.tv_sec && time1.tv_nsec > time2.tv_nsec); #else return time1.dwHighDateTime > time2.dwHighDateTime - || time1.dwHighDateTime == time2.dwHighDateTime - && time1.dwLowDateTime > time2.dwLowDateTime; + || (time1.dwHighDateTime == time2.dwHighDateTime + && time1.dwLowDateTime > time2.dwLowDateTime); #endif } #include "os/ReTraverser.hpp"