*/\r
\r
#include "base/rebase.hpp"\r
-#include "os/reos.hpp"\r
#include "math/remath.hpp"\r
+#include "os/reos.hpp"\r
\r
enum LOCATION_DIRTOOL {\r
LC_COPY_FILE_1 = LC_DIRTOOLS + 1, // 50101\r
LC_SET_PROPERTIES_2, // 50110\r
LC_SET_PROPERTIES_3, // 50111\r
LC_TOUCH_1, // 50112\r
+ LC_CALCULATE_CHECKSUM_1, // 50113\r
+ LC_BUILD_DIRECTORY_1, // 50114\r
+ LC_COMPARE_STORAGE_1, // 50115\r
+ LC_COMPARE_STORAGE_2, // 50116\r
+ LC_COMPARE_STORAGE_3, // 50117\r
+ LC_COMPARE_STORAGE_4, // 50118\r
+ LC_COMPARE_STORAGE_5, // 50119\r
+ LC_COMPARE_STORAGE_6, // 50120\r
+ LC_COMPARE_STORAGE_7, // 50121\r
+ LC_COMPARE_DIR_1, // 50122\r
};\r
const char* ReDirTools::m_version = "2015.02.25";\r
ReLogger* ReDirTools::m_logger = NULL;\r
" Useful commands around directory trees.",\r
" Type 'dirtool help <command>' for more help.", "<command>:",\r
"batch produce output to handle the found files with a script",\r
+ "checksum shows the checksum (MD5 or other) of the selected files",\r
+ "delete delete the selected files",\r
"help shows info about the arguments/options",\r
"list shows the meta data of the selected files",\r
- "md5 shows the MD5 checksum of the selected files",\r
"statistic shows statistics about a direcctory tree",\r
"synchronize copies only modified or new files from",\r
" from a source directory tre to a target",\r
"dirtool li --type=f -y7d --size=10M -p;*.cpp;*.hpp;Makefile*;-*~ /home/data",\r
NULL };\r
\r
-const char* s_md5Usage[] = {\r
- "<command>: m(d5) [<opts>] <dir_or_file1> [<dir_or_file2> ...]",\r
- " shows the MD5 check sum of the given files",\r
+const char* s_checksumUsage[] = {\r
+ "<command>: c(hecksum) [<opts>] <dir_or_file1> [<dir_or_file2> ...]",\r
+ " shows a check sum of the given files or manages a checksum storage file in each directory for observing",\r
NULL };\r
-const char* s_md5Examples[] = { "dirtool md5 --buffer-size=512 e:\\data",\r
- "dirtool m --type=f --size=10M -p;*.iso /home/iso /down/debian.iso",\r
+const char* s_checksumExamples[] = { "dirtool ch --buffer-size=512 e:\\data",\r
+ "dirtool check -cList -p;*.iso /home/iso /down/debian.iso",\r
+ "dirtool checksum -cBuild -aMD5 /home",\r
+ "dirtool checksum --command=Compare /home",\r
+ "dirtool checksum -cUpdate -O/var/log/checksum.log /home",\r
NULL };\r
\r
static const char* s_statisticUsage[] =\r
if (m_programArgs.getString("verbose", buffer)[0] != '\0') {\r
unsigned level = V_NORMAL;\r
if (ReStringUtils::lengthOfUnsigned(buffer.str(), -1, &level)\r
- != buffer.length())\r
+ != (int) buffer.length())\r
help(i18n("verbose level is not a number (or '')"), buffer.str());\r
else\r
m_verboseLevel = VerboseLevel(level);\r
fprintf(m_output, "%s\n", line.str());\r
}\r
\r
+/**\r
+ * Constructor.\r
+ *\r
+ * @param logger logger for error handling\r
+ */\r
+ReDirChecksum::ReDirChecksum(ReLogger* logger) :\r
+ ReTool(s_checksumUsage, s_checksumExamples, 0, 0, 0, true, logger),\r
+ m_command(CMD_LIST),\r
+ m_digest(NULL),\r
+ m_buffer() {\r
+ // standard short options: D d O o P p T t v y Z z\r
+ m_programArgs.addString("algorithm",\r
+ i18n("algorithm: 'MD5' or 'RPD64'"), 'a', "--algorithm", false, "MD5");\r
+ m_programArgs.addInt("buffersize",\r
+ i18n("buffer size for file reading (in KiByte)"), 'b', "--buffer-size",\r
+ 4 * 1024);\r
+ m_programArgs.addString("command",\r
+ i18n("'build': builds in each directory a file '.dt.chksum' with sum and name\n"\r
+ "'compare': shows differences between the content of '.dt.chksum' and the current files\n"\r
+ "'list': shows checksum and filename\n"\r
+ "'update': like compare, but update the content of '.dt.chksum'"),\r
+ 'c', "--command", false, "list");\r
+ m_programArgs.addInt("salt",\r
+ i18n("a number which change the checksum"), 's', "--salt", 0);\r
+ addStandardFilterOptions();\r
+}\r
+\r
+/**\r
+ * Destructor.\r
+ */\r
+ReDirChecksum::~ReDirChecksum(){\r
+ delete m_digest;\r
+ m_digest = NULL;\r
+}\r
+/**\r
+ * Builds a checksum storage for a given directory.\r
+ *\r
+ * @param path the directory to process\r
+ */\r
+void ReDirChecksum::buildStorage(const char* path, const char* storageFile){\r
+ ReTraverser traverser(path);\r
+ traverser.setMaxLevel(0);\r
+ ReDirStatus_t* entry;\r
+ ReRPD64 digest2;\r
+ digest2.setSalt(0x2004199111121989ll);\r
+ ReByteBuffer line;\r
+ FILE* fp = fopen(storageFile, "w");\r
+ if (fp == NULL){\r
+ m_logger->sayF(LOG_ERROR | CAT_FILE, LC_BUILD_DIRECTORY_1,\r
+ i18n("cannot open file: $1 (errno: $2)")).arg(storageFile).arg(\r
+ errno).end();\r
+\r
+ } else {\r
+ int level;\r
+ while ((entry = traverser.nextFile(level, &m_filter)) != NULL) {\r
+ line.setLength(0);\r
+ if (!entry->isDirectory()){\r
+ calculateChecksum(entry->fullName(), *m_digest, m_buffer,\r
+ m_logger);\r
+ line.append(m_digest->hexDigest());\r
+ }\r
+ line.appendChar('\t').append(entry->node());\r
+ digest2.update(line);\r
+ fprintf(fp, "%s\n", line.str());\r
+ if (m_verboseLevel >= V_NORMAL)\r
+ fprintf(m_output, "%16s\t\%s",\r
+ entry->isDirectory() ? "" : m_digest->hexDigest().str(),\r
+ entry->fullName());\r
+ }\r
+ fprintf(fp, "%s\n", digest2.hexDigest().str());\r
+ fclose(fp);\r
+ }\r
+}\r
+\r
+/**\r
+ * Compares the files of a directory with the files in the storage file\r
+ *\r
+ * @param path the directory\r
+ * @param storageFile the file with the checksums\r
+ */\r
+void ReDirChecksum::compareDir(const char* path, ReStringList& names,\r
+ const char* storageFile){\r
+ // forget the last line (checksum):\r
+ int count = names.count() - 1;\r
+ names.remove(count - 1);\r
+ for (int ii = 0; ii < count; ii++){\r
+ const char* name = names.strOf(ii);\r
+ const char* ptr = strchr(name, '\t');\r
+ if (ptr != NULL)\r
+ names.replaceString(ii, ptr + 1);\r
+ }\r
+ names.sort();\r
+ names.setSorted(true);\r
+ ReTraverser traverser(path);\r
+ traverser.setMaxLevel(0);\r
+ ReDirStatus_t* entry;\r
+ int level;\r
+ while ((entry = traverser.nextFile(level, &m_filter)) != NULL) {\r
+ ReSeqArray::Index index;\r
+ const char* node = entry->node();\r
+ // the stringlist stores the string with trailing '\0'\r
+ if (! names.binarySearch(node, strlen(node) + 1, index)){\r
+ m_logger->sayF(LOG_ERROR | CAT_SECURITY, LC_COMPARE_DIR_1,\r
+ i18n("missing file $1 in storage $2")).arg(entry->fullName())\r
+ .arg(storageFile).end();\r
+ }\r
+ }\r
+}\r
+\r
+/**\r
+ * Compares a storage file with the files of a directory.\r
+ *\r
+ * @param path the directory\r
+ * @param storageFile the file with the checksums\r
+ */\r
+void ReDirChecksum::compareStorage(const char* path, const char* storageFile){\r
+ ReStringList storage;\r
+ storage.readFromFile(storageFile);\r
+ if (storage.count() <= 0){\r
+ m_logger->sayF(LOG_ERROR | CAT_SECURITY, LC_COMPARE_STORAGE_1,\r
+ i18n("empty storage file: $1")).arg(storageFile).end();\r
+ } else if (! isValid(storage)){\r
+ m_logger->sayF(LOG_ERROR | CAT_SECURITY, LC_COMPARE_STORAGE_2,\r
+ i18n("corrupted storage file: $1")).arg(storageFile).end();\r
+ } else {\r
+ ReStringList cols;\r
+ ReByteBuffer fullname(path);\r
+ fullname.ensureLastChar(OS_SEPARATOR_CHAR);\r
+ int pathLength = fullname.length();\r
+ struct stat info;\r
+ // test all files of the storage:\r
+ for (size_t ii = 0; ii < storage.count() - 1; ii++){\r
+ cols.split(storage.strOf(ii), '\t');\r
+ int colCount = cols.count();\r
+ if (colCount != 2){\r
+ m_logger->sayF(LOG_ERROR | CAT_SECURITY, LC_COMPARE_STORAGE_3,\r
+ i18n("wrong format ($1) in storage file $2-$3"))\r
+ .arg(colCount).arg(storageFile).arg(ii+1).end();\r
+ break;\r
+ } else {\r
+ fullname.setLength(pathLength).append(cols.strOf(1));\r
+ if (cols.strLengthOf(0) == 0){\r
+ // a directory:\r
+ if (stat(fullname.str(), &info) != 0){\r
+ m_logger->sayF(LOG_ERROR | CAT_SECURITY,\r
+ LC_COMPARE_STORAGE_4,\r
+ i18n("missing directory $1 ($2-$3)"))\r
+ .arg(fullname).arg(storageFile).arg(ii+1).end();\r
+ } else if (! S_ISDIR(info.st_mode)){\r
+ m_logger->sayF(LOG_ERROR | CAT_SECURITY,\r
+ LC_COMPARE_STORAGE_5,\r
+ i18n("directory is now a file: $1 ($2-$3)"))\r
+ .arg(fullname).arg(storageFile).arg(ii+1).end();\r
+ }\r
+ } else {\r
+ // a directory:\r
+ if (stat(fullname.str(), &info) != 0){\r
+ m_logger->sayF(LOG_ERROR | CAT_SECURITY,\r
+ LC_COMPARE_STORAGE_6,\r
+ i18n("missing file $1 ($2-$3)"))\r
+ .arg(fullname).arg(storageFile).arg(ii+1).end();\r
+ } else if (S_ISDIR(info.st_mode)){\r
+ m_logger->sayF(LOG_ERROR | CAT_SECURITY,\r
+ LC_COMPARE_STORAGE_7,\r
+ i18n("file is now a directory: $1 ($2-$3)"))\r
+ .arg(fullname).arg(storageFile).arg(ii+1).end();\r
+ } else {\r
+ calculateChecksum(fullname.str(), *m_digest, m_buffer,\r
+ m_logger);\r
+ if (! m_digest->hexDigest().equals(cols.strOf(0))){\r
+ m_logger->sayF(LOG_ERROR | CAT_SECURITY,\r
+ LC_COMPARE_STORAGE_7,\r
+ i18n("checksum different: $1 ($2-$3)"))\r
+ .arg(fullname).arg(storageFile).arg(ii+1).end();\r
+ }\r
+ }\r
+ }\r
+ }\r
+ }\r
+ compareDir(path, storage, storageFile);\r
+ }\r
+}\r
+/**\r
+ * Checks the validity of a storage file.\r
+ *\r
+ * The storage file is protected with a checksum.\r
+ * The checksum has another salt as the checksums of the files. This makes it\r
+ * much harder to manipulate the storage file.\r
+ *\r
+ * @param storage the storage file as string list.\r
+ */\r
+bool ReDirChecksum::isValid(const ReStringList& storage){\r
+ int count = storage.count();\r
+ ReRPD64 digest2;\r
+ digest2.setSalt(0x2004199111121989ll);\r
+\r
+ for (int ii = 0; ii < count - 1; ii++){\r
+ const char* line = storage.strOf(ii);\r
+ digest2.update(line);\r
+ }\r
+ bool rc = true;\r
+ const char* hex = storage.strOf(count - 1);\r
+ if (! digest2.hexDigest().equals(hex))\r
+ rc = false;\r
+ return rc;\r
+}\r
+/**\r
+ * Lists the metadata of the specified files.\r
+ */\r
+void ReDirChecksum::doIt() {\r
+ int size = m_programArgs.getInt("buffersize") * 1024;\r
+ m_buffer.setLength(size);\r
+ ReByteBuffer value;\r
+ m_programArgs.getString("command", value);\r
+ if (value.equals("list", -1, true))\r
+ m_command = CMD_LIST;\r
+ else if (value.equals("build", -1, true))\r
+ m_command = CMD_BUILD;\r
+ else if (value.equals("compare", -1, true))\r
+ m_command = CMD_COMPARE;\r
+ else if (value.equals("update", -1, true))\r
+ m_command = CMD_UPDATE;\r
+ else\r
+ help(i18n("unknown command (expected: build compare list update): "),\r
+ value.str());\r
+\r
+ m_programArgs.getString("algorithm", value);\r
+ if (value.equals("md5", -1, true))\r
+ m_digest = new ReMD5();\r
+ else if (value.equals("rpd64", -1, true))\r
+ m_digest = new ReRPD64();\r
+ else\r
+ help(i18n("unknown algorithm (expected: MD5 RPD64): "),\r
+ value.str());\r
+ int salt = m_programArgs.getInt("salt");\r
+ if (salt != 0)\r
+ m_digest->setSalt(salt);\r
+ m_filter.m_allDirectories = true;\r
+ processFileArguments();\r
+ printSummary();\r
+}\r
+\r
+/**\r
+ * Processes one directory.\r
+ *\r
+ * @param entry the properties of the directory to process\r
+ */\r
+void ReDirChecksum::processDir(ReDirStatus_t* entry) {\r
+ ReByteBuffer storageFile;\r
+ if (m_command != CMD_LIST) {\r
+ storageFile.append(entry->fullName());\r
+ storageFile.ensureLastChar(OS_SEPARATOR_CHAR);\r
+ storageFile.append(".dt.chksum");\r
+ }\r
+ switch(m_command){\r
+ case CMD_BUILD:\r
+ buildStorage(entry->fullName(), storageFile.str());\r
+ break;\r
+ case CMD_COMPARE:\r
+ compareStorage(entry->fullName(), storageFile.str());\r
+ break;\r
+ case CMD_UPDATE:\r
+ printf("update not implemented");\r
+ break;\r
+ default:\r
+ break;\r
+ }\r
+}\r
+/**\r
+ * Processes one file.\r
+ *\r
+ * @param entry the properties of the file to process\r
+ */\r
+void ReDirChecksum::processFile(ReDirStatus_t* entry) {\r
+ const char* name = entry->fullName();\r
+ switch(m_command){\r
+ case CMD_LIST:\r
+ calculateChecksum(name, *m_digest, m_buffer, m_logger);\r
+ fprintf(m_output, "%s %s\n", m_digest->hexDigest().str(), name);\r
+ break;\r
+ default:\r
+ break;\r
+ }\r
+}\r
+/**\r
+ * Compares a storage file with the files of a directory.\r
+ *\r
+ * @param path the directory\r
+ * @param storageFile the file with the checksums\r
+ */\r
+void ReDirChecksum::updateStorage(const char* path, const char* storageFile){\r
+\r
+}\r
+\r
+/**\r
+ * Calculates the checksum of a given file.\r
+ *\r
+ * @param name the full filename\r
+ * @param digest IN: defines the kind of checksum<br>\r
+ * OUT: contains the calculated checksum\r
+ * @param buffer IN/OUT: a buffer for file reading\r
+ * @param logger logger for error processing\r
+ * @return <code>digest</code> (for chaining)\r
+ */\r
+ReDigest& ReDirChecksum::calculateChecksum(const char* name,\r
+ ReDigest& digest, ReByteBuffer& buffer, ReLogger* logger){\r
+ FILE* fp = fopen(name, "rb");\r
+ if (fp == NULL) {\r
+ if (logger != NULL)\r
+ logger->sayF(LOG_ERROR | CAT_FILE, LC_CALCULATE_CHECKSUM_1,\r
+ i18n("cannot open file: $1 (errno: $2)")).arg(name).arg(\r
+ errno).end();\r
+ } else {\r
+ ReMD5 digest;\r
+ size_t readBytes;\r
+ uint8_t* buf = reinterpret_cast<uint8_t*>(buffer.buffer());\r
+ size_t blockSize = buffer.length();\r
+\r
+ while ((readBytes = fread(buf, 1, blockSize, fp)) > 0) {\r
+ digest.update(buf, readBytes);\r
+ }\r
+ fclose(fp);\r
+ }\r
+ return digest;\r
+}\r
+\r
+\r
+\r
/**\r
* Constructor.\r
*\r
}\r
}\r
\r
-/**\r
- * Constructor.\r
- *\r
- * @param logger logger for error handling\r
- */\r
-ReDirMD5::ReDirMD5(ReLogger* logger) :\r
- ReTool(s_md5Usage, s_md5Examples, 0, 0, 0, true, logger),\r
- m_buffer() {\r
- // standard short options: D d O o P p T t v y Z z\r
- m_programArgs.addInt("buffersize",\r
- i18n("buffer size for file reading (in KiByte)"), 'b', "--buffer-size",\r
- 4 * 1024);\r
- addStandardFilterOptions();\r
-}\r
-\r
-/**\r
- * Lists the metadata of the specified files.\r
- */\r
-void ReDirMD5::doIt() {\r
- int size = m_programArgs.getInt("buffersize") * 1024;\r
- m_buffer.setLength(size);\r
- processFileArguments();\r
- printSummary();\r
-}\r
-\r
-/**\r
- * Processes one directory.\r
- *\r
- * @param entry the properties of the directory to process\r
- */\r
-void ReDirMD5::processDir(ReDirStatus_t* entry) {\r
-}\r
-/**\r
- * Processes one file.\r
- *\r
- * @param entry the properties of the file to process\r
- */\r
-void ReDirMD5::processFile(ReDirStatus_t* entry) {\r
- const char* name = entry->fullName();\r
- FILE* fp = fopen(name, "rb");\r
- if (fp != NULL) {\r
- ReMD5 digest;\r
- size_t readBytes;\r
- uint8_t* buffer = reinterpret_cast<uint8_t*>(m_buffer.buffer());\r
- size_t blockSize = m_buffer.length();\r
-\r
- while ((readBytes = fread(buffer, 1, blockSize, fp)) > 0) {\r
- digest.update(buffer, readBytes);\r
- }\r
- fclose(fp);\r
- fprintf(m_output, "%s %s\n", digest.hexDigest().str(), name);\r
- }\r
-}\r
-\r
/**\r
* Calculates the statistic of a directory tree.\r
*\r
ReDirStatus_t::filetimeToString(\r
filetimeIsUndefined(m_modified) ? &m_accessed : &m_modified,\r
bufferTime);\r
- fprintf(m_output, "%s -> %s %s\n", bufferTime.str(), name);\r
+ fprintf(m_output, "%s -> %s %s\n",\r
+ ReDirStatus_t::filetimeToString(entry->modified(),\r
+ bufferTime2),\r
+ bufferTime.str(), (char*) name);\r
}\r
}\r
}\r
m_statInfo.st_mtime\r
- entry->filetimeToTime(entry->modified()));\r
if (!ignoreDate\r
- && m_statInfo.st_mtime\r
- - entry->filetimeToTime(entry->modified())\r
- <= maxFileTimeDiff) {\r
+ && diff <= maxFileTimeDiff) {\r
if (m_verboseLevel >= V_CHATTER)\r
fprintf(m_output, "=ignored: %s same time\n",\r
targetRelativePath);\r
if (isArg("batch", arg0)) {\r
ReDirBatch batch(m_logger);\r
batch.help(NULL);\r
+ } else if (isArg("checksum", arg0)) {\r
+ ReDirChecksum sum(m_logger);\r
+ sum.help(NULL);\r
} else if (isArg("list", arg0)) {\r
ReDirList list(m_logger);\r
list.help(NULL);\r
} else if (isArg("help", arg0)) {\r
printField(s_helpSummary);\r
- } else if (isArg("md5", arg0)) {\r
- ReDirMD5 md5(m_logger);\r
- md5.help(NULL);\r
} else if (isArg("statistic", arg0)) {\r
ReDirStatistic stat(m_logger);\r
stat.help(NULL);\r
*/\r
int ReDirTools::main(int argc, char* orgArgv[]) {\r
ReDirTools tools;\r
+ if (argc < 0){\r
+ argc = 0;\r
+ for (int ix = 0; orgArgv[ix] != NULL; ix++)\r
+ argc++;\r
+ }\r
const char** argv = (const char**) orgArgv;\r
if (argc < 2) {\r
tools.help(0, argv);\r
const char* arg0 = argv[0];\r
if (isArg("batch", arg0))\r
ReDirBatch(m_logger).run(argc, argv);\r
+ else if (isArg("checksum", arg0))\r
+ ReDirChecksum(m_logger).run(argc, argv);\r
else if (isArg("help", arg0))\r
tools.help(argc, argv);\r
else if (isArg("list", arg0))\r
ReDirList(m_logger).run(argc, argv);\r
- else if (isArg("md5", arg0))\r
- ReDirMD5(m_logger).run(argc, argv);\r
else if (isArg("statistic", arg0))\r
ReDirStatistic(m_logger).run(argc, argv);\r
else if (isArg("synchronize", arg0))\r