QMutex BackupEngine::m_mutex;
QChar BackupEngine::m_separator = '\t';
QString BackupEngine::m_separatorString = "\t";
+
QStringList ChecksumTask::m_checksumInfo;
bool ChecksumTask::m_sourceProcessingReady = false;
m_sourceDirs(sourceDirs),
m_targetBaseDir(targetDir),
m_targetDirs(),
+ m_shadowBaseDir(),
+ m_shadowDirs(),
m_mainWindow(mainWindow),
m_name(name)
{
return false;
}
+/**
+ * Initializes the shadow directories.
+ *
+ * All not too old superflous files will be moved to the shadow directory.
+ * Note: the length of the shadow directory is equal to
+ * the length of the target dir: Otherwise full path length could be too long.
+ */
+void BackupEngine::initializeShadowDir(){
+ if (m_shadowBaseDir.isEmpty()){
+ int fullLength = m_targetBaseDir.length();
+ QString node = ReFileUtils::nodeOf(m_targetBaseDir);
+ int nodeLength = node.length();
+ if (nodeLength <= 2)
+ m_shadowBaseDir = ReFileUtils::parentOf(m_targetBaseDir) + "$";
+ else {
+ int middle = nodeLength / 2;
+ m_shadowBaseDir = "." + m_targetBaseDir;
+ m_shadowBaseDir.remove(fullLength - middle, 1);
+ }
+ }
+ m_shadowDirs.clear();
+ for (int ix = 0; ix < m_sourceDirs.size(); ix++){
+ m_shadowDirs.append(m_shadowBaseDir + OS_SEPARATOR_STR
+ + ReFileUtils::nodeOf(m_sourceDirs.at(ix)));
+ }
+}
+
/**
* @brief Logs a message.
*
}
/**
- * Constructor.
+ * Inserts a remove command into queue for too old files in the shadow.
*
- * @param compareWithTarget <code>false</code>: all files will be found<br>
- * otherwise: only not existing or newer files will
- * be found
- * @param name name of the task
- * @param filePatterns only files which match the patterns will be found
- * @param dirPatterns only subdirectories which matches the patterns will be entered
- * @param source the list of base directories to search
- * @param targetDir the base target directory
- * @param mainWindow the GUI module, the parent
- */
-SearchTask::SearchTask(bool compareWithTarget, const QString& name,
- const QString& filePatterns, const QString& dirPatterns,
- const QStringList& sourceDirs, const QString& targetDir,
- MainWindow* mainWindow) :
- BackupEngine(name, sourceDirs, targetDir, mainWindow),
- m_fileMatcher(filePatterns),
- m_dirMatcher(dirPatterns),
- m_compareWithTarget(compareWithTarget)
-{
-}
-
-/**
- * Runs the task.
- */
-void SearchTask::run()
-{
- qint64 start = QDateTime::currentMSecsSinceEpoch();
- m_searchReady = false;
- QString targetDir, sourceDir;
- for (int ix = 0; ix < m_sourceDirs.size(); ix++){
- sourceDir = m_sourceDirs.at(ix);
- if (m_compareWithTarget)
- targetDir = m_targetDirs.at(ix) + ReFileUtils::nodeOf(sourceDir);
- searchOneDirectory(sourceDir, targetDir, ix);
- }
- m_searchReady = true;
- m_mainWindow->externalLog(tr(
- "Search finished: to process: %1 with %2 matching: %3 total: %4 "
- "subdirs: %5 runtime: %6")
- .arg(m_hotFiles)
- .arg(ReQStringUtils::readableSize(m_hotBytes))
- .arg(m_matchedFiles).arg(m_totalFiles)
- .arg(m_totalDirs)
- .arg(ReQStringUtils::readableDuration(
- QDateTime::currentMSecsSinceEpoch() - start)));
-}
-/**
- * Search the files to backup and write it to a list.
+ * Note: this method is recursive.
*
- * This method is recursive.
- *
- * @param source the directory to inspect
- * @param target "": no target directory exists<br>
- * otherwise: only files will be written if the file found in
- * the source directory does not exists in this target or
- * it is newer or has another size
- * @param index the index of the base directory in m_sourceDirs
+ * @param directory directory to inspect
+ * @param maxAge all files older than this time point will be deleted
+ * @param index the index in the shadow directories
*/
-void SearchTask::searchOneDirectory(const QString& source,
- const QString& target, int index){
- QDirIterator it(source);
+void BackupEngine::removeOlder(const QString& directory, const QDateTime& time,
+ int index){
+ QDirIterator it(directory);
QString info, node;
- int lengthBase = m_sourceDirs.at(index).length();
+ int lengthBase = m_shadowDirs.at(index).length();
assert(index < 0x7fff);
QString prefix;
m_mutex.lock();
m_totalDirs++;
m_mutex.unlock();
- if (source.length() > lengthBase){
- prefix = QChar(1 + index)
- + ReFileUtils::nativePath(source.mid(lengthBase)) + OS_SEPARATOR_STR
+ if (directory.length() > lengthBase){
+ prefix = QChar(1 + index)
+ + ReFileUtils::nativePath(directory.mid(lengthBase)) + OS_SEPARATOR_STR
+ m_separator;
} else {
prefix = QChar(1 + index) + m_separatorString;
}
+ bool isEmpty = true;
while (it.hasNext()){
if (m_shouldStop){
break;
it.next();
node = it.fileName();
if (it.fileInfo().isDir()){
- // nothing to do
- } else if (! m_fileMatcher.matches(node)){
- m_mutex.lock();
- m_totalFiles++;
- m_mutex.unlock();
- } else {
- qint64 diff = 0;
- bool doCopy = false;
- if (! m_compareWithTarget)
- doCopy = true;
- else if (target.isEmpty())
- doCopy = true;
- else {
- QFileInfo trg(target + it.fileName());
- if (! trg.exists())
- doCopy = true;
- else {
- const QFileInfo src = it.fileInfo();
- if (trg.size() != src.size())
- doCopy = true;
- else if ((diff = src.lastModified().toMSecsSinceEpoch()
- - trg.lastModified().toMSecsSinceEpoch()) > 2000)
- doCopy = true;
- }
- }
- if (doCopy){
- info = prefix + it.fileName();
+ if (node != "." && node != ".."){
+ removeOlder(it.filePath(), time, index);
+ isEmpty = false;
}
+ } else if (it.fileInfo().lastModified() < time){
+ isEmpty = false;
+ info = prefix + QChar(CmdRemove) + it.fileName();
m_mutex.lock();
- if (doCopy){
- m_files.append(info);
- m_hotFiles++;
- m_hotBytes += it.fileInfo().size();
- }
+ m_files.append(info);
+ m_hotFiles++;
+ m_hotBytes += it.fileInfo().size();
m_totalFiles++;
- m_matchedFiles++;
m_mutex.unlock();
}
}
- if (! m_shouldStop){
- QDirIterator it2(source);
- QString node;
- QString subTarget;
- while (it2.hasNext()){
- if (m_shouldStop){
- break;
- }
- it2.next();
-
- if (it2.fileInfo().isDir() && (node = it2.fileName()) != "." && node != ".."
- && m_dirMatcher.matches(node)){
- if (target.isEmpty())
- subTarget.clear();
- else{
- subTarget = target + it2.fileName();
- if (! ReFileUtils::isDirectory(subTarget))
- subTarget.clear();
- else
- subTarget += OS_SEPARATOR_STR;
- }
- searchOneDirectory(it2.filePath(), subTarget, index);
- }
- }
+ if (isEmpty){
+ info = prefix + QChar(CmdRemoveDir) + it.fileName();
+ m_mutex.lock();
+ m_files.append(info);
+ m_matchedFiles++;
+ m_mutex.unlock();
}
+
}
/**
node = info.mid(pos + 1);
copyFile(index, relPath, node);
qint64 now = QDateTime::currentMSecsSinceEpoch();
- qint64 estimated = qint64(double(now - start) / max(1LL, m_processedBytes) * m_hotBytes);
+ qint64 estimated = qint64(double(now - start) / max(1LL, m_processedBytes) * m_hotBytes);
m_mainWindow->externalAppend(ReGuiQueueItem::StatusLine, NULL,
tr("%1 of %2 (%3 of %4) %5 MB/sec Remaining: %6 of %7")
.arg(m_processedFiles).arg(m_hotFiles)
error(QObject::tr("checksum differs: ") + relPath + node
+ " [" + sourceChecksum + "/" + checksum + "]");
}
- qint64 processedBytes, hotBytes;
- int hotFiles, processedFiles;
- m_mutex.lock();
- hotFiles = m_hotFiles;
- hotBytes = m_hotBytes;
- processedBytes = m_processedBytes;
- processedFiles = m_processedFiles;
- m_mutex.unlock();
- now = QDateTime::currentMSecsSinceEpoch();
- qint64 duration = (now - start);
- qint64 estimated = estimated = qint64(double(now - start) / max(1LL, m_processedBytes) * m_hotBytes * 2);
- qint64 rest = estimated - duration;
+ qint64 processedBytes, hotBytes;
+ int hotFiles, processedFiles;
+ m_mutex.lock();
+ hotFiles = m_hotFiles;
+ hotBytes = m_hotBytes;
+ processedBytes = m_processedBytes;
+ processedFiles = m_processedFiles;
+ m_mutex.unlock();
+ now = QDateTime::currentMSecsSinceEpoch();
+ double duration = (now - start);
+ double estimated = qint64(double(now - start));
+ estimated /= max(1LL, m_processedBytes);
+ estimated *= m_hotBytes * 2;
+ qint64 rest = qint64(estimated - duration);
m_mainWindow->externalAppend(ReGuiQueueItem::StatusLine, NULL,
tr("%1 of %2 (%3 of %4) %5 MB/sec Remaining: %6 of %7")
- .arg(m_processedFiles).arg(hotFiles * 2)
- .arg(ReQStringUtils::readableSize(processedBytes))
- .arg(ReQStringUtils::readableSize(hotBytes * 2))
- .arg(processedBytes / 1024.0 / 1024 / duration * 1000.0, 0, 'f', 3)
- .arg(ReQStringUtils::readableDuration(rest))
- .arg(ReQStringUtils::readableDuration(estimated)));
+ .arg(processedFiles).arg(hotFiles * 2)
+ .arg(ReQStringUtils::readableSize(processedBytes))
+ .arg(ReQStringUtils::readableSize(hotBytes * 2))
+ .arg(processedBytes / 1024.0 / 1024 / duration * 1000.0, 0, 'f', 3)
+ .arg(ReQStringUtils::readableDuration(rest))
+ .arg(ReQStringUtils::readableDuration(qint64(estimated))));
}
}
.arg(m_mainWindow->errors()));
}
+/**
+ Constructor.
+
+ * @param name name of the task
+ * @param sourceDirs the list of source directories to inspect
+ * @param targetDir the base of the target directories
+ * @param maxAgeSec superflous file which is older will be moved to the
+ * shadow, the files which are younger will be deleted
+ * @param mainWindow the GUI module
+ */
+
+CleanTask::CleanTask(const QString& name, const QStringList& sourceDirs,
+ const QString& targetDir, MainWindow* mainWindow) :
+ BackupEngine(name, sourceDirs, targetDir, mainWindow)
+{
+
+}
+
+/**
+ * Runs the task.
+ */
+void CleanTask::run()
+{
+ QString relPath, node;
+ QString info;
+ qint64 start = QDateTime::currentMSecsSinceEpoch();
+ while (true){
+ m_mutex.lock();
+ if (m_files.size() == 0)
+ info.clear();
+ else{
+ info = m_files.first();
+ m_files.removeFirst();
+ }
+ m_processedFiles++;
+ m_mutex.unlock();
+ if (info.isEmpty()){
+ if (m_searchReady)
+ break;
+ else
+ QThread::msleep(50);
+ } else {
+ int index = int(info.at(0).unicode()) - 1;
+ int pos = info.indexOf(m_separator, 1);
+ if (pos == 1)
+ relPath.clear();
+ else
+ relPath = info.mid(1, pos - 1);
+ Command command = (Command) info.at(pos + 1).unicode();
+ node = info.mid(pos + 2);
+ switch(command){
+ case CmdRemove:
+ m_mainWindow->addToFileList("-" + relPath);
+ break;
+ case CmdRemoveDir:
+ m_mainWindow->addToFileList("/" + relPath);
+ break;
+ case CmdMove:
+ {
+ QString target = m_targetDirs.at(index) + relPath + node;
+ QString shadowDir = m_shadowDirs.at(index) + relPath;
+ QString shadow = shadowDir + node;
+ if (! ReFileUtils::makeDirWithParents(shadowDir))
+ error(QObject::tr("cannot create the shadow directory: %1")
+ .arg(shadowDir));
+ m_mainWindow->addToFileList(">" + target + " -> " + shadow);
+ break;
+ }
+ default:
+ error("unknown command: " + QString::number(command));
+ break;
+ }
+ qint64 now = QDateTime::currentMSecsSinceEpoch();
+ qint64 estimated = qint64(double(now - start) / max(1, m_processedFiles) * m_hotFiles);
+ m_mainWindow->externalAppend(ReGuiQueueItem::StatusLine, NULL,
+ tr("%1 of %2 (%3 of %4) %5 MB/sec Remaining: %6 of %7")
+ .arg(m_processedFiles).arg(m_hotFiles)
+ .arg(ReQStringUtils::readableSize(m_processedBytes))
+ .arg(ReQStringUtils::readableSize(m_hotBytes))
+ .arg(m_processedBytes / 1024.0 / 1024 / (now - start) * 1000, 0, 'f', 3)
+ .arg(ReQStringUtils::readableDuration(estimated - (now - start)))
+ .arg(ReQStringUtils::readableDuration(estimated)));
+ }
+ }
+ m_mainWindow->externalTaskFinished(tr("backup complete after %1. Errors: %2")
+ .arg(ReQStringUtils::readableDuration(
+ QDateTime::currentMSecsSinceEpoch() - start))
+ .arg(m_mainWindow->errors()));
+}
+
+
+/**
+ * Constructor.
+ *
+ * @param compareWithTarget <code>false</code>: all files will be found<br>
+ * otherwise: only not existing or newer files will
+ * be found
+ * @param name name of the task
+ * @param filePatterns only files which match the patterns will be found
+ * @param dirPatterns only subdirectories which matches the patterns will be entered
+ * @param source the list of base directories to search
+ * @param targetDir the base target directory
+ * @param mainWindow the GUI module, the parent
+ */
+SearchTask::SearchTask(bool compareWithTarget, const QString& name,
+ const QString& filePatterns, const QString& dirPatterns,
+ const QStringList& sourceDirs, const QString& targetDir,
+ MainWindow* mainWindow) :
+ BackupEngine(name, sourceDirs, targetDir, mainWindow),
+ m_fileMatcher(filePatterns),
+ m_dirMatcher(dirPatterns),
+ m_compareWithTarget(compareWithTarget)
+{
+}
+
+/**
+ * Runs the task.
+ */
+void SearchTask::run()
+{
+ qint64 start = QDateTime::currentMSecsSinceEpoch();
+ m_searchReady = false;
+ QString targetDir, sourceDir;
+ for (int ix = 0; ix < m_sourceDirs.size(); ix++){
+ sourceDir = m_sourceDirs.at(ix);
+ if (m_compareWithTarget)
+ targetDir = m_targetDirs.at(ix) + ReFileUtils::nodeOf(sourceDir);
+ searchOneDirectory(sourceDir, targetDir, ix);
+ }
+ m_searchReady = true;
+ m_mainWindow->externalLog(tr(
+ "Search finished: to process: %1 with %2 matching: %3 total: %4 "
+ "subdirs: %5 runtime: %6")
+ .arg(m_hotFiles)
+ .arg(ReQStringUtils::readableSize(m_hotBytes))
+ .arg(m_matchedFiles).arg(m_totalFiles)
+ .arg(m_totalDirs)
+ .arg(ReQStringUtils::readableDuration(
+ QDateTime::currentMSecsSinceEpoch() - start)));
+}
+/**
+ * Search the files to backup and write it to a list.
+ *
+ * This method is recursive.
+ *
+ * @param source the directory to inspect
+ * @param target "": no target directory exists<br>
+ * otherwise: only files will be written if the file found in
+ * the source directory does not exists in this target or
+ * it is newer or has another size
+ * @param index the index of the base directory in m_sourceDirs
+ */
+void SearchTask::searchOneDirectory(const QString& source,
+ const QString& target, int index){
+ QDirIterator it(source);
+ QString info, node;
+ int lengthBase = m_sourceDirs.at(index).length();
+ assert(index < 0x7fff);
+ QString prefix;
+ m_mutex.lock();
+ m_totalDirs++;
+ m_mutex.unlock();
+ if (source.length() > lengthBase){
+ prefix = QChar(1 + index)
+ + ReFileUtils::nativePath(source.mid(lengthBase)) + OS_SEPARATOR_STR
+ + m_separator;
+ } else {
+ prefix = QChar(1 + index) + m_separatorString;
+ }
+ while (it.hasNext()){
+ if (m_shouldStop){
+ break;
+ }
+ it.next();
+ node = it.fileName();
+ if (it.fileInfo().isDir()){
+ // nothing to do
+ } else if (! m_fileMatcher.matches(node)){
+ m_mutex.lock();
+ m_totalFiles++;
+ m_mutex.unlock();
+ } else {
+ qint64 diff = 0;
+ bool doCopy = false;
+ if (! m_compareWithTarget)
+ doCopy = true;
+ else if (target.isEmpty())
+ doCopy = true;
+ else {
+ QFileInfo trg(target + it.fileName());
+ if (! trg.exists())
+ doCopy = true;
+ else {
+ const QFileInfo src = it.fileInfo();
+ if (trg.size() != src.size())
+ doCopy = true;
+ else if ((diff = src.lastModified().toMSecsSinceEpoch()
+ - trg.lastModified().toMSecsSinceEpoch()) > 2000)
+ doCopy = true;
+ }
+ }
+ if (doCopy){
+ info = prefix + it.fileName();
+ }
+ m_mutex.lock();
+ if (doCopy){
+ m_files.append(info);
+ m_hotFiles++;
+ m_hotBytes += it.fileInfo().size();
+ }
+ m_totalFiles++;
+ m_matchedFiles++;
+ m_mutex.unlock();
+ }
+ }
+ if (! m_shouldStop){
+ QDirIterator it2(source);
+ QString node;
+ QString subTarget;
+ while (it2.hasNext()){
+ if (m_shouldStop){
+ break;
+ }
+ it2.next();
+
+ if (it2.fileInfo().isDir() && (node = it2.fileName()) != "." && node != ".."
+ && m_dirMatcher.matches(node)){
+ if (target.isEmpty())
+ subTarget.clear();
+ else{
+ subTarget = target + it2.fileName();
+ if (! ReFileUtils::isDirectory(subTarget))
+ subTarget.clear();
+ else
+ subTarget += OS_SEPARATOR_STR;
+ }
+ searchOneDirectory(it2.filePath(), subTarget, index);
+ }
+ }
+ }
+}
+
+/**
+ Constructor.
+
+ * @param name name of the task
+ * @param sourceDirs the list of source directories to inspect
+ * @param targetDir the base of the target directories
+ * @param maxAgeSec superflous file which is older will be moved to the
+ * shadow, the files which are younger will be deleted
+ * @param mainWindow the GUI module
+ */
+
+SearchTargetTask::SearchTargetTask(const QString& name, const QStringList& sourceDirs,
+ const QString& targetDir, qint64 maxAgeSec, MainWindow* mainWindow) :
+ BackupEngine(name, sourceDirs, targetDir, mainWindow),
+ m_maxAge()
+{
+ m_maxAge.setMSecsSinceEpoch(maxAgeSec * 1000);
+ initializeShadowDir();
+}
+
+/**
+ * Search the files to clean.
+ *
+ * This method is recursive.
+ *
+ * @param target the directory to inspect
+ * @param source the source directory to compare
+ * @param index the index of the base directory in m_targetDirs
+ */
+void SearchTargetTask::searchOneDirectory(const QString& target,
+ const QString& source, int index){
+ QDirIterator it(target);
+ QString info, node;
+ int lengthBase = m_targetDirs.at(index).length();
+ assert(index < 0x7fff);
+ QString prefix;
+ m_mutex.lock();
+ m_totalDirs++;
+ m_mutex.unlock();
+ if (source.length() > lengthBase){
+ prefix = QChar(1 + index)
+ + ReFileUtils::nativePath(source.mid(lengthBase)) + OS_SEPARATOR_STR
+ + m_separator;
+ } else {
+ prefix = QChar(1 + index) + m_separatorString;
+ }
+ while (it.hasNext()){
+ if (m_shouldStop){
+ break;
+ }
+ it.next();
+ node = it.fileName();
+ if (it.fileInfo().isDir()){
+ // nothing to do
+ } else{
+ Command command = CmdUndef;
+ QFileInfo src(source + it.fileName());
+ if (! src.exists()){
+ const QFileInfo trg = it.fileInfo();
+ command = trg.lastModified() < m_maxAge ? CmdRemove : CmdMove;
+ info = prefix + QChar(command) + it.fileName();
+ }
+ m_mutex.lock();
+ if (command != CmdUndef){
+ m_files.append(info);
+ m_hotFiles++;
+ m_hotBytes += it.fileInfo().size();
+ }
+ m_totalFiles++;
+ m_mutex.unlock();
+ }
+ }
+ if (! m_shouldStop){
+ QDirIterator it2(target);
+ QString node;
+ QString source;
+ while (it2.hasNext()){
+ if (m_shouldStop){
+ break;
+ }
+ it2.next();
+
+ if (it2.fileInfo().isDir() && (node = it2.fileName()) != "." && node != ".."){
+ QString newSource = source + it2.fileName();
+ QFileInfo src(newSource);
+ if (src.exists() && src.isDir())
+ searchOneDirectory(it2.filePath(), newSource + OS_SEPARATOR_STR, index);
+ else{
+ moveToShadow(it2.filePath(), prefix, index);
+ }
+ }
+ }
+ }
+}
+
+/**
+ * Moves a target directory to the shadow directory and delegate the removing files.
+ *
+ * @param target the target directory
+ * @param relPath the relative path of the parent of <code>target</code>
+ * @param index the index of the target in <code>m_targetDirs</code>
+ */
+void SearchTargetTask::moveToShadow(const QString& target, const QString& relPath, int index){
+ QString shadowDir = m_shadowDirs.at(index) + relPath;
+ if (! ReFileUtils::makeDirWithParents(shadowDir))
+ error(QObject::tr("cannot create shadow directory (%1): %2")
+ .arg(errno).arg(shadowDir));
+ else {
+ QString shadow = shadowDir + ReFileUtils::nodeOf(target);
+ if (rename(I18N::s2b(target).constData(), I18N::s2b(shadow).constData()) != 0){
+ error(QObject::tr("cannot move to shadow directory (%1): %2 -> %3")
+ .arg(errno).arg(target).arg(shadow));
+ } else {
+ removeOlder(shadowDir, m_maxAge, index);
+ }
+ }
+}
+
+/**
+ * Executes the task "search superflous files/dirs in the target directory".
+ */
+void SearchTargetTask::run()
+{
+ qint64 start = QDateTime::currentMSecsSinceEpoch();
+ m_searchReady = false;
+ QString targetDir, sourceDir;
+ for (int ix = 0; ix < m_sourceDirs.size(); ix++){
+ sourceDir = m_sourceDirs.at(ix);
+ targetDir = m_targetDirs.at(ix);
+ searchOneDirectory(targetDir, sourceDir, ix);
+ }
+ m_searchReady = true;
+ m_mainWindow->externalLog(tr(
+ "Search in target finished: to process: %1 with %2 dirs to delete: %3 total: %4 "
+ "subdirs: %5 runtime: %6")
+ .arg(m_hotFiles)
+ .arg(ReQStringUtils::readableSize(m_hotBytes))
+ .arg(m_matchedFiles).arg(m_totalFiles)
+ .arg(m_totalDirs)
+ .arg(ReQStringUtils::readableDuration(
+ QDateTime::currentMSecsSinceEpoch() - start)));
+}
projects / reqt / commitdiff