From 5960c5f0487e96799214b63d3ed177c5c95c6830 Mon Sep 17 00:00:00 2001 From: hama Date: Tue, 6 Jan 2015 23:08:28 +0100 Subject: [PATCH] dirtools sync implemented --- base/ReConfigFile.cpp | 6 ++ base/baselocations.hpp | 7 +- os/ReDirTools.cpp | 170 ++++++++++++++++++++++++++++++++++++----- os/ReDirTools.hpp | 7 ++ os/ReTraverser.cpp | 22 ++++-- os/ReTraverser.hpp | 3 +- 6 files changed, 184 insertions(+), 31 deletions(-) diff --git a/base/ReConfigFile.cpp b/base/ReConfigFile.cpp index 9f0db53..ec56b06 100644 --- a/base/ReConfigFile.cpp +++ b/base/ReConfigFile.cpp @@ -6,6 +6,12 @@ #include "base/rebase.hpp" +enum RELOC_RECONFIGFILE { + LC_CONFIGFILE_DOUBLE = LC_CONFIGFILE + 1, // 50001 + LC_CONFIGFILE_NO_INT, // 50002 + LC_CONFIGFILE_NO_BOOL, // 50003 +}; + const char* ReConfigFile::m_trueValues = i18nm(";true;t;1;j;ja;yes;"); const char* ReConfigFile::m_falseValues = i18nm(";false;f;0;n;nein;no;"); diff --git a/base/baselocations.hpp b/base/baselocations.hpp index c67c0ee..83b39ad 100644 --- a/base/baselocations.hpp +++ b/base/baselocations.hpp @@ -10,10 +10,9 @@ /** The files of the REal PUBlic LIB use the range from 50000 until 99999. */ -enum RELOC_RECONFIGFILE { - LC_CONFIGFILE_DOUBLE = 50001, - LC_CONFIGFILE_NO_INT = 50002, - LC_CONFIGFILE_NO_BOOL = 50003, +enum RELOC_LIB { + LC_CONFIGFILE = 50000, + LC_DIRTOOLS = 50100, }; enum RELOC_UDPCONNECTION { LC_UDPCONNECTION_CONSTRUCT = 50101, diff --git a/os/ReDirTools.cpp b/os/ReDirTools.cpp index ed3223d..c4be6ef 100644 --- a/os/ReDirTools.cpp +++ b/os/ReDirTools.cpp @@ -8,6 +8,14 @@ #include "base/rebase.hpp" #include "os/reos.hpp" +enum LOCATION_DIRTOOL { + LC_COPY_FILE_1 = LC_DIRTOOLS + 1, // 50101 + LC_COPY_FILE_2, // 50102 + LC_COPY_FILE_3, // 50103 + LC_COPY_FILE_4, // 50104 + LC_COPY_FILE_5, // 50105 + LC_COPY_FILE_6, // 50106 +}; const char* ReDirTools::m_version = "2015.01.05"; static const char* s_helpSummary[] = { @@ -994,14 +1002,116 @@ void makeDirWithParents(ReByteBuffer& path, int minWidth){ * Constructor. */ ReDirSync::ReDirSync() : - ReDirOptions(s_syncUsage, s_syncExamples) + ReDirOptions(s_syncUsage, s_syncExamples), + m_buffer() { + m_buffer.ensureSize(4u*1024*1024*1024); + m_programArgs.addBool("add", + i18n("copies only files which does not exist on the target"), + 'a', "add", false); + m_programArgs.addBool("dry", + i18n("does nothing, but says what should be done"), + 'd', "dry", false); + m_programArgs.addInt("timediff", + i18n("filetime difference is considered to be equal\n" + "if the difference is less than this value (in seconds)"), + 'D', "time-delta", 2); + m_programArgs.addBool("ignoredate", + i18n("the modification is recognized only by the different size (not time)"), + 'i', "ignore-time", false); + m_programArgs.addBool("chatter", + i18n("comments the action of each file"), + 'h', "chatter", false); m_programArgs.addBool("mustexist", i18n("files which don't exist on the target will not be copied"), 'm', "must-exist", false); addStandardFilterOptions(); } +/** + * Copies a file. + * + * @param entry the source file info + * @param target the name of the target file + */ +void ReDirSync::copyFile(ReDirStatus_t* entry, const char* target){ + copyFile(entry->fullName(), entry->filetimeToTime(entry->modified()), + entry->fileSize(), target, m_buffer, ReLogger::globalLogger()); +} +/** + * Copies a file. + * + * @param source the source file name + * @param modified 0 or modification time + * @param size -1 or filesize + * @param target the name of the target file + * @param buffer OUT: the reading uses this buffer
+ * Set the capacity to make it more efficient + * @param logger NULL or the logger for error messages + */ +bool ReDirSync::copyFile(const char* source, time_t modified, int64_t size, + const char* target, ReByteBuffer& buffer, ReLogger* logger){ + bool rc = false; + if (size < 0ll){ + struct stat info; + if (stat(source, &info) == 0) + size = info.st_size; + else if (logger != NULL) + logger->sayF(LOG_ERROR, LC_COPY_FILE_1, + i18n("could not find: $ (errno: $2)")).arg(source).arg(errno).end(); + } + if (size >= 0){ + FILE* fpSource = fopen(source, "rb"); + if (fpSource == NULL){ + if (logger != NULL) + logger->sayF(LOG_ERROR, LC_COPY_FILE_2, + i18n("cannot open $1 (errno: $2)")) + .arg(source).arg(errno).end(); + } else { + FILE* fpTarget = fopen(target, "w"); + if (fpTarget == NULL){ + if (logger != NULL) + logger->sayF(LOG_ERROR, LC_COPY_FILE_3, + i18n("cannot open $1 (errno: $2)")) + .arg(target).arg(errno).end(); + } else{ + // Reserve the space: + if (fseek(fpTarget, size, SEEK_SET) != size){ + if (logger != NULL) + logger->sayF(LOG_ERROR, LC_COPY_FILE_4, + i18n("cannot reserve space for $1 (errno: $2)")) + .arg(target).arg(errno).end(); + } else { + fseek(fpTarget, 0, SEEK_SET); + while(size > 0){ + size_t blockSize = buffer.capacity(); + if (blockSize > size) + blockSize = size; + if (fread(buffer.buffer(), blockSize, 1, fpSource) != 1){ + if (logger != NULL) + logger->sayF(LOG_ERROR, LC_COPY_FILE_5, + i18n("cannot read $1 (errno: $2)")) + .arg(source).arg(errno).end(); + break; + } + if (fwrite(buffer.buffer(), 1, blockSize, fpSource) != 1){ + if (logger != NULL) + logger->sayF(LOG_ERROR, LC_COPY_FILE_6, + i18n("cannot write $1 (errno: $2)")) + .arg(target).arg(errno).end(); + break; + } + size -= blockSize; + } + } + rc = size == 0ll; + fclose(fpTarget); + } + fclose(fpSource); + } + } + return rc; +} /** * Gets the arguments for the "list" command and execute this. * @@ -1026,16 +1136,19 @@ void ReDirSync::synchronize(int argc, const char* argv[]){ else if (! S_ISDIR(info.st_mode)) help(i18n("target is not a directory: $1"), target.str()); size_t lengthTargetBase = target.length(); + bool addOnly = m_programArgs.getBool("add"); + bool chatterMode = m_programArgs.getBool("chatter"); + int maxFileTimeDiff = m_programArgs.getInt("timediff"); + bool dry = m_programArgs.getBool("dry"); + bool ignoreDate = m_programArgs.getBool("ignoredate"); + bool mustExist = m_programArgs.getBool("mustexist"); bool verbose = ! m_programArgs.getBool("quiet"); - bool chatterMode = true; - bool copyExistingOnly = false; setFilterFromProgramArgs(filter); int64_t sumSizes = 0; int files = 0; int dirs = 0; ReByteBuffer source; ReByteBuffer targetFile; - int maxFileTimeDiff = 2; for (int ix = 0; ix < m_programArgs.getArgCount() - 1; ix++){ source.set(m_programArgs.getArg(ix), -1); target.setLength(lengthTargetBase); @@ -1061,30 +1174,46 @@ void ReDirSync::synchronize(int argc, const char* argv[]){ while( (entry = traverser.nextFile(level, &filter)) != NULL){ // append the new relative path from source to target: target.setLength(ixTargetRelative); + const char* targetRelativePath = target.str() + ixTargetRelative; target.append(entry->m_path.str() + ixSourceRelative, -1); if (stat(target.str(), &info) != 0) makeDirWithParents(target, ixTargetRelative); targetFile.set(target).append(entry->node(), -1); bool exists = stat(targetFile.str(), &info) == 0; - if ( (! exists && ! copyExistingOnly) - || entry->filetimeToTime(entry->modified()) - info.st_mtime + if (! exists && ! mustExist){ + if (chatterMode) + printf("-ignored: %s does not exist\n", targetRelativePath); + continue; + } + if (exists && addOnly){ + if (chatterMode) + printf("~ignored: %s exists\n", targetRelativePath); + continue; + } + if (ignoreDate && entry->fileSize() == info.st_size){ + if (chatterMode) + printf("_ignored: %s same size\n", targetRelativePath); + continue; + } + if (! ignoreDate && entry->filetimeToTime(entry->modified()) - info.st_mtime > maxFileTimeDiff) { if (chatterMode) - printf("%s%s same time\n", entry->m_path.str(), entry->node()); + printf("=ignored: %s same time\n", targetRelativePath); continue; } else { - - } - - if (entry->isDirectory()) - dirs++; - else{ - files++; - sumSizes += entry->fileSize(); - } - if (verbose) - printf("%s\n", line.str()); - } + if (entry->isDirectory()) + dirs++; + else{ + files++; + sumSizes += entry->fileSize(); + } + if (verbose || chatterMode || dry) + printf("%c%s%s\n", exists ? '!' : '+', targetRelativePath, + dry ? " would be copied" : ""); + if (! dry) + copyFile(entry, target.str()); + } + } } if (verbose){ int duration = int(time(NULL) - start); @@ -1200,7 +1329,6 @@ int ReDirTools::main(int argc, char* orgArgv[]){ testAll(); }else tools.usage("unknown command: ", argv[1]); - //testOs(); - ReLogger::freeGlobalLogger(); + ReLogger::freeGlobalLogger(); return 0; } diff --git a/os/ReDirTools.hpp b/os/ReDirTools.hpp index 5929171..6b6fe66 100644 --- a/os/ReDirTools.hpp +++ b/os/ReDirTools.hpp @@ -49,6 +49,13 @@ public: ReDirSync(); public: void synchronize(int argc, const char* argv[]); +protected: + void copyFile(ReDirStatus_t* entry, const char* target); +public: + static bool copyFile(const char* source, time_t modified, int64_t size, + const char* target, ReByteBuffer& buffer, ReLogger* logger = NULL); +protected: + ReByteBuffer m_buffer; }; class ReDirList : public ReDirOptions { diff --git a/os/ReTraverser.cpp b/os/ReTraverser.cpp index 3d4d6d1..a7def56 100644 --- a/os/ReTraverser.cpp +++ b/os/ReTraverser.cpp @@ -21,12 +21,12 @@ const char* const ReTraverser::m_separatorStr = "\\"; */ ReDirStatus_t::ReDirStatus_t() : m_path(), + m_fullName(), m_passNo(0), #ifdef __linux__ m_handle(NULL), - m_data(NULL), + m_data(NULL) //m_status; - m_fullName() #elif defined WIN32 m_handle(INVALID_HANDLE_VALUE) //m_data; @@ -55,6 +55,7 @@ void ReDirStatus_t::freeEntry(){ } #endif m_path.setLength(0); + m_fullName.setLength(0); } /** @@ -135,6 +136,7 @@ bool ReDirStatus_t::findFirst(){ m_handle = FindFirstFileA(thePath.str(), &m_data); rc = m_handle != INVALID_HANDLE_VALUE; #endif + m_fullName.setLength(0); return rc; } @@ -150,6 +152,7 @@ bool ReDirStatus_t::findNext(){ #elif defined __WIN32__ bool rc = m_handle != INVALID_HANDLE_VALUE && FindNextFileA(m_handle, &m_data); #endif + m_fullName.setLength(0); return rc; } @@ -186,6 +189,16 @@ ReDirStatus_t::Type_t ReDirStatus_t::type(){ return rc; } +/** + * Returns the full filename (with path). + * + * @return the filename with path + */ +const char* ReDirStatus_t::fullName(){ + if (m_fullName.length() == 0) + m_fullName.set(m_path).append(node(), -1); + return m_fullName.str(); +} /** * Tests whether the instance is a directory. * @@ -353,9 +366,8 @@ bool ReDirEntryFilter_t::match(ReDirStatus_t& entry){ */ struct stat* ReDirStatus_t::getStatus() { if (m_status.st_ino == 0){ - m_fullName.set(m_path).append(m_data->d_name, -1); - if (stat(m_fullName.str(), &m_status) != 0) - memset((void*) &m_status, 0, sizeof m_status); + if (stat(fullName(), &m_status) != 0) + m_status.st_ino = 0; } return &m_status; } diff --git a/os/ReTraverser.hpp b/os/ReTraverser.hpp index be66239..c751c48 100644 --- a/os/ReTraverser.hpp +++ b/os/ReTraverser.hpp @@ -45,6 +45,7 @@ public: public: void freeEntry(); const char* node() const; + const char* fullName(); bool findFirst(); bool findNext(); bool hasData() const; @@ -59,12 +60,12 @@ public: Type_t type(); public: ReByteBuffer m_path; + ReByteBuffer m_fullName; int m_passNo; #ifdef __linux__ DIR* m_handle; struct dirent* m_data; struct stat m_status; - ReByteBuffer m_fullName; public: struct stat* getStatus(); #elif defined WIN32 -- 2.39.5