From 6f8f00817ce1f581bb587edb12d50bcd80387925 Mon Sep 17 00:00:00 2001 From: Hamatoma <author@hamatoma.de> Date: Mon, 19 Feb 2024 22:03:37 +0100 Subject: [PATCH 1/1] Initial state --- .gitignore | 10 + All.sh | 14 + Env.sh | 6 + InstallAll | 25 + MakeCloud.py | 135 + MkTar.sh | 6 + appl/.gitignore | 1 + appl/BackupBase.py | 306 +++ appl/BackupTool.py | 979 ++++++++ appl/BenchTool.py | 243 ++ appl/DbTool.py | 941 +++++++ appl/DirTool.py | 1206 +++++++++ appl/FtpTool.py | 176 ++ appl/Monitor.py | 1184 +++++++++ appl/RestoreTool.py | 2174 +++++++++++++++++ appl/SvgTool.py | 741 ++++++ appl/TextTool.py | 1313 ++++++++++ appl/UrlChecker.py | 357 +++ appl/WebDashClient.py | 426 ++++ appl/WebDashServer.py | 336 +++ appl/Webserver.py | 107 + appl/WikiTool.py | 205 ++ appl/ZipTool.py | 181 ++ appl/__init__.py | 0 base/BaseLogger.py | 98 + base/BaseTool.py | 1454 +++++++++++ base/CryptoEngine.py | 400 +++ base/FileHelper.py | 678 +++++ base/JavaConfig.py | 92 + base/LinuxUtils.py | 234 ++ base/Logger.py | 49 + base/MemoryLogger.py | 64 + base/ProcessHelper.py | 224 ++ base/PythonToTypeScript.py | 652 +++++ base/StringUtils.py | 436 ++++ base/TextProcessor.py | 1776 ++++++++++++++ base/ThreadLogger.py | 38 + base/Zipper.py | 468 ++++ base/__init__.py | 0 doc/mysql.txt | 1 + examples/safe/php.ini | 562 +++++ net/EMail.py | 130 + net/FileTcpTaskHandler.py | 242 ++ net/FtpEngine.py | 295 +++ net/HttpClient.py | 154 ++ net/LinuxTcpTaskHandler.py | 176 ++ net/TcpClient.py | 82 + net/TcpServer.py | 151 ++ net/TcpTaskHandler.py | 66 + net/__init__.py | 5 + unittest/PackageTest.py | 25 + unittest/UnitTestCase.py | 402 +++ unittest/UnitTestSuite.py | 116 + unittest/__init__.py | 0 unittest/appl/BackupToolTest.py | 508 ++++ unittest/appl/BaseToolTest.py | 191 ++ unittest/appl/BenchToolTest.py | 48 + unittest/appl/DbToolTest.py | 390 +++ unittest/appl/DirToolTest.py | 418 ++++ unittest/appl/FtpToolTest.py | 154 ++ unittest/appl/MonitorTest.py | 276 +++ unittest/appl/RestoreToolTest.py | 694 ++++++ unittest/appl/SvgToolTest.py | 176 ++ unittest/appl/TextToolTest.py | 812 ++++++ unittest/appl/WikiToolTest.py | 101 + unittest/appl/__init__.py | 0 unittest/base/BaseTester.py | 36 + unittest/base/CryptoEngineTest.py | 174 ++ unittest/base/FileHelperTest.py | 371 +++ unittest/base/JavaConfigTest.py | 68 + unittest/base/LinuxUtilsTest.py | 96 + unittest/base/LoggerTest.py | 92 + unittest/base/MemoryLoggerTest.py | 27 + unittest/base/ProcessHelperTest.py | 70 + unittest/base/PythonToTypeScriptTest.py | 230 ++ unittest/base/StringUtilsTest.py | 301 +++ unittest/base/TextProcessorTest.py | 481 ++++ unittest/base/ThreadLoggerTest.py | 30 + unittest/base/ZipperTest.py | 274 +++ unittest/base/__init__.py | 0 unittest/base/sample.env | 6 + unittest/configuration/default.conf | 33 + unittest/configuration/error/default.conf | 32 + unittest/configuration/localhost.conf | 23 + unittest/configuration/observed/default.conf | 23 + .../configuration/observed/localhost.conf | 29 + unittest/configuration/sites/otherport | 5 + unittest/configuration/sites/wiki.hamatoma.de | 20 + unittest/data/etc.tgz | Bin 0 -> 25579 bytes unittest/data/etc.work.tgz | Bin 0 -> 785 bytes unittest/data/example.html | 1 + unittest/data/example.sh | 2 + unittest/data/example.tar | Bin 0 -> 20480 bytes unittest/data/example.tbz | Bin 0 -> 1025 bytes unittest/data/example.tgz | Bin 0 -> 1044 bytes unittest/data/example.txt | 35 + unittest/data/example.zip | Bin 0 -> 3680 bytes unittest/data/examples2.tgz | Bin 0 -> 1004 bytes unittest/data/monitor/localhost.dump.txt | 68 + unittest/img/landscape.jpg | Bin 0 -> 798 bytes unittest/img/portrait.jpg | Bin 0 -> 798 bytes unittest/net/FileTcpTaskHandlerTest.py | 165 ++ unittest/net/HttpClientTest.py | 30 + unittest/net/LinuxTcpTaskHandlerTest.py | 50 + unittest/net/NetTester.py | 20 + unittest/net/__init__.py | 0 106 files changed, 26702 insertions(+) create mode 100644 .gitignore create mode 100755 All.sh create mode 100755 Env.sh create mode 100755 InstallAll create mode 100644 MakeCloud.py create mode 100755 MkTar.sh create mode 100644 appl/.gitignore create mode 100755 appl/BackupBase.py create mode 100755 appl/BackupTool.py create mode 100755 appl/BenchTool.py create mode 100755 appl/DbTool.py create mode 100755 appl/DirTool.py create mode 100755 appl/FtpTool.py create mode 100755 appl/Monitor.py create mode 100755 appl/RestoreTool.py create mode 100755 appl/SvgTool.py create mode 100755 appl/TextTool.py create mode 100755 appl/UrlChecker.py create mode 100755 appl/WebDashClient.py create mode 100755 appl/WebDashServer.py create mode 100755 appl/Webserver.py create mode 100755 appl/WikiTool.py create mode 100755 appl/ZipTool.py create mode 100644 appl/__init__.py create mode 100644 base/BaseLogger.py create mode 100644 base/BaseTool.py create mode 100644 base/CryptoEngine.py create mode 100644 base/FileHelper.py create mode 100644 base/JavaConfig.py create mode 100644 base/LinuxUtils.py create mode 100644 base/Logger.py create mode 100644 base/MemoryLogger.py create mode 100644 base/ProcessHelper.py create mode 100644 base/PythonToTypeScript.py create mode 100644 base/StringUtils.py create mode 100644 base/TextProcessor.py create mode 100644 base/ThreadLogger.py create mode 100644 base/Zipper.py create mode 100644 base/__init__.py create mode 100644 doc/mysql.txt create mode 100644 examples/safe/php.ini create mode 100644 net/EMail.py create mode 100644 net/FileTcpTaskHandler.py create mode 100644 net/FtpEngine.py create mode 100644 net/HttpClient.py create mode 100644 net/LinuxTcpTaskHandler.py create mode 100644 net/TcpClient.py create mode 100644 net/TcpServer.py create mode 100644 net/TcpTaskHandler.py create mode 100644 net/__init__.py create mode 100644 unittest/PackageTest.py create mode 100644 unittest/UnitTestCase.py create mode 100644 unittest/UnitTestSuite.py create mode 100644 unittest/__init__.py create mode 100644 unittest/appl/BackupToolTest.py create mode 100644 unittest/appl/BaseToolTest.py create mode 100644 unittest/appl/BenchToolTest.py create mode 100644 unittest/appl/DbToolTest.py create mode 100644 unittest/appl/DirToolTest.py create mode 100644 unittest/appl/FtpToolTest.py create mode 100644 unittest/appl/MonitorTest.py create mode 100644 unittest/appl/RestoreToolTest.py create mode 100644 unittest/appl/SvgToolTest.py create mode 100644 unittest/appl/TextToolTest.py create mode 100644 unittest/appl/WikiToolTest.py create mode 100644 unittest/appl/__init__.py create mode 100644 unittest/base/BaseTester.py create mode 100644 unittest/base/CryptoEngineTest.py create mode 100644 unittest/base/FileHelperTest.py create mode 100644 unittest/base/JavaConfigTest.py create mode 100644 unittest/base/LinuxUtilsTest.py create mode 100644 unittest/base/LoggerTest.py create mode 100644 unittest/base/MemoryLoggerTest.py create mode 100644 unittest/base/ProcessHelperTest.py create mode 100644 unittest/base/PythonToTypeScriptTest.py create mode 100644 unittest/base/StringUtilsTest.py create mode 100644 unittest/base/TextProcessorTest.py create mode 100644 unittest/base/ThreadLoggerTest.py create mode 100644 unittest/base/ZipperTest.py create mode 100644 unittest/base/__init__.py create mode 100644 unittest/base/sample.env create mode 100644 unittest/configuration/default.conf create mode 100644 unittest/configuration/error/default.conf create mode 100644 unittest/configuration/localhost.conf create mode 100644 unittest/configuration/observed/default.conf create mode 100644 unittest/configuration/observed/localhost.conf create mode 100644 unittest/configuration/sites/otherport create mode 100644 unittest/configuration/sites/wiki.hamatoma.de create mode 100644 unittest/data/etc.tgz create mode 100644 unittest/data/etc.work.tgz create mode 100644 unittest/data/example.html create mode 100644 unittest/data/example.sh create mode 100644 unittest/data/example.tar create mode 100644 unittest/data/example.tbz create mode 100644 unittest/data/example.tgz create mode 100644 unittest/data/example.txt create mode 100644 unittest/data/example.zip create mode 100644 unittest/data/examples2.tgz create mode 100644 unittest/data/monitor/localhost.dump.txt create mode 100644 unittest/img/landscape.jpg create mode 100644 unittest/img/portrait.jpg create mode 100644 unittest/net/FileTcpTaskHandlerTest.py create mode 100644 unittest/net/HttpClientTest.py create mode 100644 unittest/net/LinuxTcpTaskHandlerTest.py create mode 100644 unittest/net/NetTester.py create mode 100644 unittest/net/__init__.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8407395 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.project +.pydevproject +.settings +*.pyc +2*.sh +/unpackShell.sh +examples/data/ +examples/config +/upd-monitor.sh +/2dragon diff --git a/All.sh b/All.sh new file mode 100755 index 0000000..fa3ef01 --- /dev/null +++ b/All.sh @@ -0,0 +1,14 @@ +#! /bin/bash +TAR=/tmp/rsh.tgz +FN=unpackShell.sh +./MkTar.sh +All2Tmp all $TAR +rsync -av $TAR hm@dromedar:/home/www/public +cat <<EOS >$FN +#! /bin/bash +cd /usr/share/pyrshell +./upd.sh +EOS +chmod +x $FN +Cmd2All all $FN + diff --git a/Env.sh b/Env.sh new file mode 100755 index 0000000..ce97eb1 --- /dev/null +++ b/Env.sh @@ -0,0 +1,6 @@ +#! /bin/bash +export PYTHONPATH=/home/ws/py/pyrshell:/home/ws/py/pyrshell/base:/usr/lib/python35.zip:/usr/lib/python3.5:/usr/lib/python3.5/plat-x86_64-linux-gnu:/usr/lib/python3.5/lib-dynload:/usr/local/lib/python3.5/dist-packages:/usr/lib/python3/dist-packages:/home/ws/py/pyrshell +export MONITOR_HOST=hawk +export MONITOR_CONFIG=/etc/pymonitor/config.d +export MONITOR_APPL=pymonitor +export MONITOR_LOGFILE=/var/log/local/pymonitor.log diff --git a/InstallAll b/InstallAll new file mode 100755 index 0000000..a355a76 --- /dev/null +++ b/InstallAll @@ -0,0 +1,25 @@ +#! /bin/bash +MODE=$1 +if [ "$MODE" != 'min' -a "$MODE" != 'all' -a "$MODE" != 'std' ]; then + echo "Usage: InstallAll [MODE]" + echo "Installs all applications" + echo "MODE: min std all" +else + appl/DbTool.py -v3 install + appl/DirTool.py -v3 install + appl/RestoreTool.py -v3 install + appl/SvgTool.py -v3 install + appl/TextTool.py -v3 install + appl/ZipTool.py -v3 install + if [ "$MODE" = 'std' -o "$MODE" = 'all' ]; then + appl/BackupTool.py -v3 install + appl/Monitor.py -v3 install + fi + if [ "$MODE" = 'all' ]; then + appl/UrlChecker.py -v3 install + appl/WebDashClient.py -v3 install + appl/WebDashServer.py -v3 install + appl/WikiTool.py -v3 install + appl/FtpTool.py -v3 install + fi +fi diff --git a/MakeCloud.py b/MakeCloud.py new file mode 100644 index 0000000..f3fc182 --- /dev/null +++ b/MakeCloud.py @@ -0,0 +1,135 @@ +#!/usr/local/bin/python2.7 +# encoding: utf-8 +''' +MakeCloud -- shortdesc + +MakeCloud is a description + +It defines classes_and_methods + +@author: user_name + +@copyright: 2018 organization_name. All rights reserved. + +@license: license + +@contact: user_email +@deffield updated: Updated +''' + +import sys +import os + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2018-03-13' +__updated__ = '2018-03-13' + +DEBUG = 1 +TESTRUN = 0 +PROFILE = 0 + +class CLIError(Exception): + '''Generic exception to raise and log different fatal errors.''' + def __init__(self, msg): + super(CLIError).__init__(type(self)) + self.msg = "E: %s" % msg + def __str__(self): + return self.msg + def __unicode__(self): + return self.msg + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by user_name on %s. + Copyright 2018 organization_name. All rights reserved. + + Licensed under the Apache License 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-r", "--recursive", dest="recurse", action="store_true", help="recurse into subfolders [default: %(default)s]") + parser.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]") + parser.add_argument("-i", "--include", dest="include", help="only include paths matching this regex pattern. Note: exclude is given preference over include. [default: %(default)s]", metavar="RE" ) + parser.add_argument("-e", "--exclude", dest="exclude", help="exclude paths matching this regex pattern. [default: %(default)s]", metavar="RE" ) + parser.add_argument('-V', '--version', action='version', version=program_version_message) + parser.add_argument(dest="paths", help="paths to folder(s) with source file(s) [default: %(default)s]", metavar="path", nargs='+') + + # Process arguments + args = parser.parse_args() + + paths = args.paths + verbose = args.verbose + recurse = args.recurse + inpat = args.include + expat = args.exclude + + if verbose > 0: + print("Verbose mode on") + if recurse: + print("Recursive mode on") + else: + print("Recursive mode off") + + if inpat and expat and inpat == expat: + raise CLIError("include and exclude pattern are equal! Nothing will be processed.") + + for inpath in paths: + ### do something with inpath ### + print(inpath) + return 0 + except KeyboardInterrupt: + ### handle keyboard interrupt ### + return 0 + except Exception, e: + if DEBUG or TESTRUN: + raise(e) + indent = len(program_name) * " " + sys.stderr.write(program_name + ": " + repr(e) + "\n") + sys.stderr.write(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + if DEBUG: + sys.argv.append("-h") + sys.argv.append("-v") + sys.argv.append("-r") + if TESTRUN: + import doctest + doctest.testmod() + if PROFILE: + import cProfile + import pstats + profile_filename = 'MakeCloud_profile.txt' + cProfile.run('main()', profile_filename) + statsfile = open("profile_stats.txt", "wb") + p = pstats.Stats(profile_filename, stream=statsfile) + stats = p.strip_dirs().sort_stats('cumulative') + stats.print_stats() + statsfile.close() + sys.exit(0) + sys.exit(main()) \ No newline at end of file diff --git a/MkTar.sh b/MkTar.sh new file mode 100755 index 0000000..1740d7c --- /dev/null +++ b/MkTar.sh @@ -0,0 +1,6 @@ +#! /bin/bash +TAR=/tmp/rsh.tgz +test -f $TAR && rm -f $TAR +tar czf $TAR appl base net unittest doc examples InstallAll +ls -ld $TAR + diff --git a/appl/.gitignore b/appl/.gitignore new file mode 100644 index 0000000..ce97416 --- /dev/null +++ b/appl/.gitignore @@ -0,0 +1 @@ +/DoIt.py diff --git a/appl/BackupBase.py b/appl/BackupBase.py new file mode 100755 index 0000000..9a00eae --- /dev/null +++ b/appl/BackupBase.py @@ -0,0 +1,306 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' + +import time +import datetime +import sys +import os.path +import stat +import subprocess +import tempfile +import gzip +import fnmatch +import re +import traceback + +from platform import node +import shutil +sys.path.insert(0, '/usr/share/pyrshell') +import base.Logger +import base.MemoryLogger +import base.StringUtils +import base.JavaConfig +import base.BaseTool +import net.EMail + +class BackupBase (base.BaseTool.BaseTool): + + def __init__(self, globalOptions): + '''Constructor. + @param logger: the logger + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'backup.conf') + self._lastTargetDir = None + + def allDatabases(self, metadataToo=False): + '''Returns all mysql databases. + @param metadataToo: True: mysql is returned too + @param user: a user having the rights to read the database mysql + @param password: the user's password + @return: array with databases + ''' + if '_mysql' in sys.modules: + mode = 'intern' + import _mysql + else: + mode = self._configuration.getString('mysql.mode', 'file') + + rc = [] + user = self._configuration.getString('mysql.user') + password = self._configuration.getString('mysql.code') + if user == None or password == None: + self._logger.error('missing mysql.user and/or mysql.code in configuration (backup.conf)') + elif mode == 'file': + path = '/var/lib/mysql' + nodes = os.listdir(path) + for node in nodes: + full = path + os.sep + node + if node != 'sys' and os.path.isdir(full): + if not metadataToo and (node == 'mysql' or node == 'information_schema' or node == 'performance_schema'): + continue + rc.append(node) + elif mode == 'extern': + argv = ['/usr/bin/mysql', '-u', user, '-p' + password, 'mysql'] + errorFile = tempfile.gettempdir() + os.sep + 'backuptool.err.txt' + with open(errorFile, 'w') as fpError: + proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=fpError.fileno()) + proc.communicate(b'show databases;\nquit;\n') + lineNo = 0 + while True: + lineNo += 1 + line = proc.stdout.readline().decode() + if line == '': + break + if lineNo < 4: + continue + if line.startswith('|'): + db = line.strip('| \n') + if not metadataToo and (db == 'mysql' or db == 'information_schema' or db == 'performance_schema'): + continue + rc.append(db) + self.errorFileToLog(errorFile, 'database mysql') + os.unlink(errorFile) + else: + db =_mysql.connect('localhost', user, password, 'mysql') + db.query('show databases;') + dbs = db.store_result() + results = dbs.fetch_row(0) + for item in results: + value = item[0] + if not metadataToo and (value == b'mysql' or value == b'information_schema' or value == b'performance_schema'): + continue + rc.append(value.decode()) + return rc + + def findConfig(self, variable, value, subdir): + '''Finds a configuration file given by a "variable" and a value. + @param variable: a configuration variable + @param value: the value to search + @param subdir: the subdirectory inside the backup configuration, e.g. 'webapps.d' + @return: None: not found otherwise: the JavaConfig instance of the found configuration + ''' + rc = None + configDir = self._configDir + os.sep + subdir + if not os.path.isdir(configDir): + self._logger.error('missing {:s} in {:s}'.format(subdir, self._configDir)) + else: + nodes = os.listdir(configDir) + for node in nodes: + if node.endswith('.conf'): + full = configDir + os.sep + node + config = base.JavaConfig.JavaConfig(full, self._logger) + if config.getString(variable) == value: + rc = config + break + return rc + + def findDbInfo(self, path, defaultUser, defaultPassword): + '''Searches the info about the database in a given directory. + @param path: the directory to search + @return: a tuple (db, user, password) + ''' + configWordPress = path + os.sep + 'wp-config.php' + configShop = path + os.sep + 'config.php' + configTimetracking = path + '/config/sys/main.php' + configNextCloud = path + '/config/config.php' + configMediaWiki = path + '/LocalSettings.php' + configWinfothek = path + '/resources/winfothek.conf' + if os.path.exists(configWinfothek): + # .dburl=mysql:host=localhost;dbname=winfothek + # .dbuser=winfothek + db = base.BaseTool.BasicStatics.grep('.dburl=mysql:', configWinfothek)[0].split('dbname=')[1].strip() + user = base.BaseTool.BasicStatics.grep('.dbuser=', configWinfothek)[0].split('.dbuser=')[1].strip() + password = base.BaseTool.BasicStatics.grep('.dbcode=', configWinfothek)[0].split('.dbcode=')[1].strip() + elif os.path.exists(configWordPress): + # define('DB_NAME', 'testdb'); // Der Name der Datenbank, die du benutzt. + db = base.BaseTool.BasicStatics.grep('DB_NAME', configWordPress)[0].split(')')[0].split(',')[1].strip().strip(",');").strip('"') + user = base.BaseTool.BasicStatics.grep('DB_USER', configWordPress)[0].split(')')[0].split(',')[1].strip().strip(",');").strip('"') + password = base.BaseTool.BasicStatics.grep('DB_PASSWORD', configWordPress)[0].split(');')[0].split("DB_PASSWORD',")[1].strip().strip(",');").strip('"') + elif os.path.exists(configNextCloud): + db = base.BaseTool.BasicStatics.grep('dbname', configNextCloud)[0].split('=>')[1].strip().strip(",');").strip('"') + user = base.BaseTool.BasicStatics.grep('dbuser', configNextCloud)[0].split('=>')[1].strip().strip(",');").strip('"') + password = base.BaseTool.BasicStatics.grep('dbpassword', configNextCloud)[0].split('=>')[1].strip().strip(",');").strip('"') + elif os.path.exists(configShop): + db = base.BaseTool.BasicStatics.grep('dbname', configShop)[0].split('=>')[1].strip().strip(',').strip("'").strip('"') + user = base.BaseTool.BasicStatics.grep('username', configShop)[0].split('=>')[1].strip().strip(',').strip("'").strip('"') + password = base.BaseTool.BasicStatics.grep('password', configShop)[0].split('=>')[1].strip().strip(',').strip("'").strip('"') + elif os.path.exists(configTimetracking): + db = base.BaseTool.BasicStatics.grep("'dsn'", configTimetracking)[0].split('dbname=')[1].strip().strip(',').strip("'").strip('"').strip(',') + user = base.BaseTool.BasicStatics.grep("'user'", configTimetracking)[0].split('=>')[1].strip().strip(',').strip("'").strip('"') + password = base.BaseTool.BasicStatics.grep("'pwd'", configTimetracking)[0].split('=>')[1].strip().strip(',').strip("'").strip('"') + elif os.path.exists(configMediaWiki): + # $wgDBuser = "forum"; + db = base.BaseTool.BasicStatics.grep("wgDBname", configMediaWiki)[0].split('wgDBname =')[1].strip().strip(';').strip('"') + user = base.BaseTool.BasicStatics.grep("wgDBuser", configMediaWiki)[0].split('wgDBuser =')[1].strip().strip(';').strip('"') + password = base.BaseTool.BasicStatics.grep("wgDBpassword", configMediaWiki)[0].split('wgDBpassword =')[1].strip().strip(';').strip('"') + else: + db = 'unknowndb' + user = defaultUser + password = defaultPassword + return (db, user, password) + + def relativeBackupPath(self, mode, timestamp=None): + '''Returns the (date based) relative backup path, e.g. '/dayly/Sun' + ''' + relPath = os.sep + mode + os.sep + if mode == 'dayly': + relPath += ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][time.localtime(timestamp).tm_wday] + elif mode == 'weekly': + aTuple = time.localtime(timestamp) + # %W: week number with Mon as first day of the week + weekNo = int(time.strftime('%W', aTuple)) + no = int(weekNo) % 4 + relPath += str(no) + elif mode == 'monthly': + relPath += str(time.localtime(timestamp).tm_mon) + else: + relPath = relPath[0:-1] + return relPath + + def replaceMacros(self, text): + '''Replaces macros (written as '%<name>%') with its values + @param text: a text with macros, e.g. '%etc%/dirs.d/*.conf' + @return: the text with replaced macros, e.g. '/etc/pyrshell/dirs.d/*.conf' + ''' + rc = text.replace('%etc%', self._configDir) + if text.find('%backup') >= 0: + rc = rc.replace('%backup.base%', self._baseTargetDir) + rc = rc.replace('%backup.current%', self._targetDir) + if rc.find('%backup.last%') >= 0: + if self._lastTargetDir == None: + self._logger.error('%backup.last% is undefined. we replace it with %backup.current%: ' + self._targetDir) + rc = rc.replace('%backup.last%', self._targetDir) + else: + rc = rc.replace('%backup.last%', self._lastTargetDir) + node = os.path.basename(self._targetDir) + if node == 'often': + minipath = node + else: + minipath = os.path.basename(os.path.dirname(self._targetDir)) + os.sep + node + rc = rc.replace('%backup.current.minipath%', minipath) + rc = rc.replace('%backup.current.node%', node) + return rc + + def synchronize(self, source, target, deleteToo): + '''Synchronizes a directory from another. + Copies the newer or not existing files from the source to the target, recursive + @param source: the source directory + @param target: the target directory + @param deleteToo: files in target but not existing in source will be deleted + ''' + def logError(arg): + self._logger.error('error while removing directory {:s}: {:s}'.format(fullTarget, arg)) + sourceNodes = os.listdir(source) + targetNodes = os.listdir(target) + dirs = [] + for node in sourceNodes: + fullSource = source + os.sep + node + fullTarget = target + os.sep + node + doCopy = False + infoSource = os.lstat(fullSource) + if stat.S_ISDIR(infoSource.st_mode): + dirs.append(node) + else: + try: + infoTarget = os.lstat(fullTarget) + doCopy = infoTarget.st_mtime > infoSource.st_mtime + except OSError: + doCopy = True + if doCopy: + self._logger.log('copying {:s}'.format(fullSource), 3) + shutil.copy(fullSource, fullTarget) + if deleteToo: + if node in targetNodes: + targetNodes.remove(node) + if deleteToo: + for node in targetNodes: + fullTarget = target + os.sep + node + self._logger.log('deleting {:s}'.format(fullTarget), 3) + if os.path.isdir(fullTarget): + shutil.rmtree(fullTarget) + else: + os.unlink(fullTarget) + for node in dirs: + trgDir = target + os.sep + node + if os.path.exists(trgDir): + if not os.path.isdir(trgDir): + os.unlink(trgDir) + os.mkdir(trgDir) + else: + os.mkdir(trgDir) + self.synchronize(source + os.sep + node, trgDir, deleteToo) + +class ConfigurationSet(): + '''Administrates a set of configuration files stored in a given directory. + ''' + def __init__(self, pattern, logger): + '''Constructor. + @param pattern: a file pattern to identify the configuration files. + @param logger: the logger + ''' + self._logger = logger + parts = pattern.split(':') + self._pattern = parts[0] + self._variables = parts[1:] + self._configurations = [] + self._path = os.path.dirname(self._pattern) + self._namePattern = os.path.basename(self._pattern) + self._lastHandledFile = None + if not os.path.isdir(self._path): + self._logger.error('not a directory: ' + self._path) + else: + files = os.listdir(self._path) + for node in files: + if fnmatch.fnmatch(node, self._namePattern): + self._configurations.append(node) + + def nextFile(self): + '''Generator method returning the next configuration file of the set. + @yields: the next file + ''' + for node in self._configurations: + yield self._path + os.sep + node + + def nextParameters(self): + '''Generator method returning the parameters stored in the next configuration file of the set. + @yields: the parameters of the next file + ''' + for name in self.nextFile(): + self._lastHandledFile = name + config = base.JavaConfig.JavaConfig(name, self._logger) + rc = [] + if len(self._variables) == 0: + self._logger.error('no variables defined in ' + self._pattern) + for variable in self._variables: + value = config.getString(variable) + if value == None: + self._logger.error('variable {:s} not defined in {:s}'.format(variable, name)) + rc.append(config.getString(variable)) + yield rc + +if __name__ == '__main__': + pass diff --git a/appl/BackupTool.py b/appl/BackupTool.py new file mode 100755 index 0000000..94da96d --- /dev/null +++ b/appl/BackupTool.py @@ -0,0 +1,979 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' + +import time +import datetime +import sys +import os.path +import gzip +import re +import traceback +import subprocess +import shutil +from base.ProcessHelper import ProcessHelper +from base.MemoryLogger import MemoryLogger + +sys.path.insert(0, '/usr/share/pyrshell') +import base.Logger +import base.StringUtils +import base.BaseTool +import net.EMail +import appl.BackupBase +import appl.DbTool + +class BackupTool (appl.BackupBase.BackupBase): + + def __init__(self, globalOptions): + '''Constructor. + @param logger: the logger + ''' + appl.BackupBase.BackupBase.__init__(self, globalOptions) + self._logger.setErrorFilter(re.compile(r'/bin/tar: .*(file changed as we read it|tar: Removing leading|mysqldump:.*Warning.*Using a password)'), True) + + def btrfsInfo(self): + rc = '' + lines = self._processHelper.executeInputOutput(['/usr/sbin/blkid']) + devs = [] + for line in lines: + if line.find('TYPE="btrfs"') >= 0: + # remove leading ':' + devs.append(line.split(' ')[0][0:-1]) + mountDir = '/media/tmp' + self.ensureDirectory(mountDir) + files = os.listdir(mountDir); + if len(files) > 0: + self._processHelper.execute(['/bin/umount', mountDir], False) + for dev in devs: + self._processHelper.execute(['mount', '-o', 'ro', dev, mountDir], False) + lines = self._processHelper.executeInputOutput(['/usr/bin/btrfs', 'subvol', 'list', mountDir]) + if rc == '': + rc = '\n=== btrfs-Info\n' + rc += '= btrfs: ' + dev + '\n' + '\n'.join(lines) + lines = self._processHelper.executeInputOutput(['/usr/bin/btrfs', 'device', 'usage', mountDir]) + rc += '= device usage: ' + dev + '\n' + '\n'.join(lines) + lines = self._processHelper.executeInputOutput(['/usr/bin/btrfs', 'device', 'stats', mountDir]) + rc += '= device stats: ' + dev + '\n' + '\n'.join(lines) + lines = self._processHelper.executeInputOutput(['/usr/bin/btrfs', 'filesystem', 'df', mountDir]) + rc += '= filesytem df: ' + dev + '\n' + '\n'.join(lines) + self._processHelper.execute(['/bin/umount', mountDir], False) + self._logger.log('devs: {}'.format(len(devs)), 2) + return rc + + def createConfigExample(self): + self.ensureDirectory(self._configDir) + dirWebApps = self._configDir + os.sep + 'webapps.d' + self.ensureDirectory(dirWebApps) + dirDirs = self._configDir + os.sep + 'dirs.d' + self.ensureDirectory(dirDirs) + filename = self._configDir + os.sep + 'backup.conf' + if not base.StringUtils.hasContent(filename, '#'): + filename = self._configDir + os.sep + 'backup.example' + base.StringUtils.toFile(filename, '''# backup example configuration +target.path=/media/backup +log.file=/var/log/local/backup.log +log.mirror=/media/backup/log/backup.log +zip.volume.size=10g +# jobs to do: Fix is job.<mode> (job.dayly...) +job.often=job.often.dirs job.chown job.rsync +job.often.dirs=&saveDirByZipLatest 1 @%etc%/dirs.d/minimal.*.conf:directory +job.dayly=job.sysinfo job.save.webapps job.sys job.minimal.1 job.chown job.once +job.weekly=job.sysinfo job.save.webapps job.sys job.full job.db.all job.chown +job.sys=&saveDirByTar @%etc%/dirs.d/sys.*.conf:directory:excluded +job.full=&saveDirByTar @%etc%/dirs.d/std.*.conf:directory:excluded +job.minimal.1=&saveDirByZipLatest 7 @%etc%/dirs.d/std.*.conf:directory:excluded +job.minimal.2=&saveDirByZipLatest 7 @%etc%/webapps.d/*.conf:directory +job.db=&saveMysql @%etc%/webapps.d/*.conf:db:user:password:sql.file +job.db.all=&saveAllDatabases +job.sysinfo=&systemInfo +job.rsync=&rsync setDromedarDayly %backup.current% +job.once=job.once.zip job.once.rsync job.chown +job.once.rsync=&rsync setOnce %backup.last%/ +job.once.zip=&saveDirByZip @%etc%/webapps.d/*.conf:directory:excluded +job.save.webapps=&saveWebapps @%etc%/webapps.d/*.conf:db:user:password:directory +# job.gitlab.backup=&gitlabBackup +job.chown=&chown bupsrv.bupsrv %backup.current% +# Reading metadata from mysql: +mysql.user=backup +mysql.code=Secret.Phrase +mysql.mode=file +# rsync set for dayly synchronisation +setDromedarDayly.user=extbup +setDromedarDayly.target=bupcaribou@dromedar:/opt/extbup/caribou/often/%factor% +setDromedarDayly.factor=4 + +setOnce.user=root +setOnce.target=%backup.base%/once +setOnce.factor=1 + +# Email data for error report: +# Receiver of the error messages: may be a blank separated list of email addresses +send.always=False +location={:s} +admin.email=hm.neutral@gmx.de +smtp.host=smtp.gmx.de +smtp.port=587 +smtp.sender=hm.neutral@gmx.de +smtp.user=hm.neutral@gmx.de +smtp.code=sEcReT +smtp.tls=True +'''.format(self.hostname(True))) + self._logger.log('created: ' + filename) + filenameWebApp = dirWebApps + os.sep + 'testdb.example' + base.StringUtils.toFile(filenameWebApp, '''# Example created by backuptool +db=testdb +user=testuser +password=NotVerySecret +sql.file=testdb +directory=/opt/webapp/testapp +''') + self._logger.log('created: ' + filenameWebApp) + BackupTool.createDirsD(dirDirs, self._logger, True) + + @staticmethod + def createDirsD(dirDirs, logger, varLogToo): + '''Creates the standard files in dirs.d. + @param dirDirs: the full path of the directory dirs.d + @param logger: None or the logger + @param varLogToo: True: the configuration file to store /var/log will be created + Note: the files can be read only by root + ''' + def createConfig(name, content): + if not os.path.exists(name): + base.StringUtils.toFile(name, content) + if logger != None: + logger.log('created: ' + name) + createConfig(dirDirs + os.sep + 'sys.etc.conf', '''# Example created by backuptool +directory=/etc +excluded= +''') + createConfig(dirDirs + os.sep + 'sys.home.bin.conf', '''# Example created by backuptool +directory=/home/bin +excluded= +''') + createConfig(dirDirs + os.sep + 'sys.usr.local.conf', '''# Example created by backuptool +directory=/usr/local +excluded= +''') + if varLogToo: + createConfig(dirDirs + os.sep + 'std.var.log.conf', '''# Example created by backuptool +directory=/var/log +excluded= +''') + createConfig(dirDirs + os.sep + 'std.srv.conf', '''# Example created by backuptool +directory=/srv +excluded= +''') + + def createLockAndReadyFile(self, mode): + '''Creates the names of the lock file and the ready file. + @param mode: 'dayly', 'weekly', 'monthly' + @return: an array [lockFile, readyFile] + ''' + lockFile = "{}{}.{}.lck".format(self._baseTargetDir, os.sep, mode) + readyFile = "{}{}.{}.finished".format(self._baseTargetDir, os.sep, mode) + return [lockFile, readyFile] + + def createWebapp(self, password, patterns): + '''Creates the webapps.d files given by the nginx configurations. + @param patterns: a list of patterns to select the nginx configurations, e.g. ["*.de", "*.com"] + ''' + nodes = [] + sourceDir = '/etc/nginx/sites-enabled' + trgDir = self._configDir + os.sep + 'webapps.d/' + for pattern in patterns: + nodes += base.BaseTool.BasicStatics.findFiles(sourceDir, pattern, False) + for node in nodes: + if self._verboseLevel >= 3: + self._logger.log('working on ' + node) + root = base.BaseTool.BasicStatics.grep('root', sourceDir + os.sep + node)[0].strip() + root = root.split()[1].replace('/;', '').replace(';', '') + self._logger.log('root: ' + root) + (db, user, password) = self.findDbInfo(root, 'backup', password) + if db == 'unknowndb': + dbFile = root + os.sep + '.dbname' + if os.path.exists(dbFile): + db = base.StringUtils.fromFile(dbFile).strip() + self._logger.log('db: {:s} user: {:s} pw: {:s}'.format(db, user, password)) + base.StringUtils.toFile(trgDir + node + '.conf', '''db={:s} +user={:s} +password={:s} +sql.file={:s} +directory={:s} +'''.format(db, user, password, node + '_' + db, root)) + + def doJob(self, config, statement): + '''Interpretes one statement. + @param config: the configuration file + @param statement: the statement to execute + ''' + statement = statement.strip() + tokens = statement.split(' ') + if statement.startswith('&'): + if tokens[0] == '&saveDirByTar': + self.jobSaveDirByTar(tokens[1:]) + elif tokens[0] == '&saveDirByZip': + self.jobSaveDirByZip(tokens[1:]) + elif tokens[0] == '&saveDirByZipLatest': + self.jobSaveDirByZipLatest(tokens[1:]) + elif tokens[0] == '&deleteFile': + self.jobDeleteFile(tokens[1:]) + elif tokens[0] == '&saveMysql': + self.jobSaveMysql(tokens[1:]) + elif tokens[0] == '&saveAllDatabases': + self.jobSaveAllDatabases() + elif tokens[0] == '&systemInfo': + self.systemInfo(self._targetDir) + elif tokens[0] == '&synchronize': + self.jobSynchronize(tokens[1:]) + elif tokens[0] == '&gitlabBackup': + self.jobGitlabBackup() + elif tokens[0] == '&saveWebApps': + self.jobSaveWebApps(tokens[1:]) + elif tokens[0] == '&chown': + self.jobChown(tokens[1:]) + elif tokens[0] == '&rsync': + self.jobRSynchronize(tokens[1:]) + else: + self._logger.error('unknown command: ' + tokens[0]) + else: + for token in tokens: + savePath = None + if token == 'job.once': + savePath = self._lastTargetDir = self._targetDir + self._targetDir = self._baseTargetDir + os.sep + 'once' + self.ensureDirectory(self._targetDir) + statement2 = config.getString(token) + if statement2 == None: + self._logger.error('unknown variable {:s} in {:s} while working on "{:s}"'.format( + token, config._filename, statement)) + else: + self.doJob(config, statement2) + if savePath != None: + self._targetDir = savePath + + def doBackup(self, mode): + '''Does the backup process controlled by configuration files. + @param mode: 'dayly', 'weekly' ... + ''' + logFile = self._configuration.getString('log.file') + if logFile != None and (not hasattr(self._logger, '_logfile') or os.path.abspath(logFile) != os.path.abspath(self._logger._logfile)): + logger2 = base.Logger.Logger(logFile, self._logger._verbose) + logger2.transferErrors(self._logger) + self._logger = logger2 + logFile2 = self._configuration.getString('log.mirror') + if logFile2 != None: + logger3 = base.Logger.Logger(logFile, False) + self._logger.setMirror(logger3) + self._mode = mode + self._baseTargetDir = self._configuration.getString('target.path') + self._targetDir = self._baseTargetDir + try: + if self._configuration.getString('job.' + mode) == None: + self._logger.error('missing job.{:s} in {:s} Wrong mode?'.format(mode, self._configuration._filename)) + elif self._targetDir == None: + self._logger.error('missing "{:s}" in {:s}'.format(self._targetDir, self._configuration._filename)) + elif not os.path.isdir(self._targetDir): + self._logger.error('{:s} is not a directory. See "target.path" in '.format(self._targetDir, self._configuration._filename)) + else: + relPath = self.relativeBackupPath(mode) + self._targetDir += relPath + if self.ensureDirectory(self._targetDir) != None: + if mode == 'dayly' or mode == 'weekly' or mode == 'monthly': + linkTarget = os.path.dirname(self._targetDir) + os.sep + 'current' + if os.path.exists(linkTarget): + os.unlink(linkTarget) + os.symlink(os.path.basename(self._targetDir), linkTarget) + statement = self._configuration.getString('job.' + mode ); + if statement == None: + self._logger.error('missing job.' + mode + ' in ' + self._configuration._filename); + else: + [lockFile, readyFile] = self.createLockAndReadyFile(mode) + if os.path.exists(lockFile): + self._logger.error('backup [{}] is already running: {} exists'.format(mode, lockFile)) + base.StringUtils.toFile(lockFile, '') + base.BaseTool.BasicStatics.ensureFileDoesNotExist(readyFile, None, self._logger) + self.doJob(self._configuration, statement) + base.StringUtils.toFile(readyFile, '') + base.BaseTool.BasicStatics.ensureFileDoesNotExist(lockFile, None, self._logger) + except Exception as exc: + self._logger.error('backup aborted with exception: ' + str(exc)) + traceback.print_exc() + subject = None + if self._logger._errors > 0: + subject = 'Backup {:s} failed on {:s}'.format(mode, self._configuration.getString('location', base.BaseTool.BasicStatics.hostname(True))) + elif self._configuration.getString('send.always', 'False').startswith('T'): + subject = 'Backup {:s} finished on {:s}'.format(mode, self._configuration.getString('location', base.BaseTool.BasicStatics.hostname(True))) + if subject != None: + now = datetime.datetime.now() + body = now.strftime('%Y.%m.%d %H:%M:%S') + ' Backup {:s} finished with {:d} error(s)\n\n'.format(mode, self._logger._errors) + body += '\n'.join(self._logger._firstErrors) + self.sendStatusEmail(subject, body) + + def example(self): + example = '''# backup example configuration +target.path=/media/backup +log.file=/var/log/local/backup.log +log.mirror=/media/backup/log/backup.log +zip.volume.size=10g +# jobs to do: Fix is job.<mode> (job.dayly...) +job.often=job.often.dirs job.chown job.rsync +job.often.dirs=&saveDirByZipLatest 1 @%etc%/dirs.d/minimal.*.conf:directory +job.dayly=job.sysinfo job.sys job.minimal.1 job.chown +job.weekly=job.sysinfo job.sys job.full job.db.all job.chown +job.sys=&saveDirByTar @%etc%/dirs.d/sys.*.conf:directory:excluded +job.full=&saveDirByTar @%etc%/dirs.d/std.*.conf:directory:excluded +job.minimal.1=&saveDirByZipLatest 7 @%etc%/dirs.d/std.*.conf:directory:excluded +job.minimal.2=&saveDirByZipLatest 7 @%etc%/webapps.d/*.conf:directory +job.db=&saveMysql @%etc%/webapps.d/*.conf:db:user:password:sql.file +job.db.all=&saveAllDatabases +job.sysinfo=&systemInfo +job.rsync=&rsync setDromedarDayly %backup.current% + +# job.gitlab.backup=&gitlabBackup +job.chown=&chown bupsrv.bupsrv %backup.current% +# Reading metadata from mysql: +mysql.user=backup +mysql.code=Secret.Phrase +mysql.mode=file +# rsync set for dayly synchronisation +setDromedarDayly.user=extbup +setDromedarDayly.target=bupcaribou@dromedar:/opt/extbup/caribou/often/%factor% +setDromedarDayly.factor=4 +# Email data for error report: +# Receiver of the error messages: may be a blank separated list of email addresses +send.always=False +location={:s} +admin.email=hm.neutral@gmx.de +smtp.host=smtp.gmx.de +smtp.port=587 +smtp.sender=hm.neutral@gmx.de +smtp.user=hm.neutral@gmx.de +smtp.code=sEcReT +smtp.tls=True +'''.format(base.BaseTool.BasicStatics.hostname(True)) + self.storeExample(example) + + def jobGitlabBackup(self): + # gitlab_rails['backup_path'] = "/var/opt/gitlab/backups" + start = time.time() + backupPath = '/var/opt/gitlab/backups' + regExpr = re.compile(r'gitlab_rails\[\'backup_path\'\]\s*=\s*"(.*?)"') + lines = base.StringUtils.grepInFile('/etc/gitlab/gitlab.rb', regExpr, 1) + if len(lines) != 1: + self._logger.error('cannot find backup path in /etc/gitlab/gitlab.rb') + else: + matcher = regExpr.search(lines[0]) + if matcher: + backupPath = matcher.group(1) + self.clearDirectory(backupPath) + executable = '/usr/bin/gitlab-rake' + if not os.path.exists(executable): + self._logger.error('missing {}: could not save gitlab'.format(executable)) + else: + argv = [executable, 'gitlab:backup:create'] + self._processHelper.execute(argv, True) + backupFile = base.BaseTool.BasicStatics.firstFile(backupPath, '*.tar') + if backupFile == None: + self._logger.error('missing backup file in ' + backupPath) + else: + self.logFile(backupFile, 'gitlab backup file: %f %s %t (%r)', start) + target = self._targetDir + os.sep + 'gitlab_backup.tar' + nameFile = self._targetDir + os.sep + 'gitlab_name.txt' + base.StringUtils.toFile(nameFile + '\n#above: the original name of the backup file\n# current name: gitlab_backup.tar', backupFile) + self._logger.log('name saved into {}'.format(nameFile), 2) + try: + shutil.copy(backupFile, target) + os.unlink(backupFile) + self._logger.log('{} moved to {}'.format(backupFile, target), 3) + except OSError as exc: + self._logger.error('cannot move archive {:s} to {:s}: {:s}'.format(backupFile, target, str(exc))) + + def jobChown(self, argv): + '''Changes the owner of some directories (recursive). + @param argv: contains the owner[.group] and the directories + ''' + if len(argv) < 1: + self._logger.error('jobChown: missing owner') + elif re.match(r'\w+([.:]\w+)?$', argv[0]) == None: + self._logger.error('jobChown: wrong user[.group]: ' + argv[0]) + elif len(argv) < 2: + self._logger.error('jobChown: missing directory') + elif self._beingRoot: + owner = argv[0] + self._logger.log('jobChown ' + ' '.join(argv), 2) + for name in argv[1:]: + path = self.replaceMacros(name) + if not os.path.isdir(path): + self._logger.error('jobChown: not a directory: ' + path) + else: + self._logger.log('chown {:s} {:s}'.format(owner, path), 3) + argv = ['/bin/chown', '-R', owner, path] + self._processHelper.execute(argv, True) + self._logger.log('chmod -R g+w {:s}'.format(path), 3) + argv = ['/bin/chmod', '-R', 'g+rw', path] + self._processHelper.execute(argv, True) + + def jobDeleteFile(self, argv): + '''Saves some directories in some tar archivs. + @param argv: contains the source directories + ''' + for name in argv: + if os.path.exists(name): + self._logger.log('deleting ' + name, 3) + os.unlink(name) + if os.path.exists(name): + self._logger.error('cannot delete ' + name) + else: + self._logger.log('deleted: ' + name) + + def jobRSynchronize(self, argv): + '''Executes the program rsync to copy files to another host. + @param argv: contains the name of a parameter set and a list of source files/dirs + e.g.['setDromedarDayly', '%backup.current%'] + ''' + self._logger.log('jobRSynchronize ' + ' '.join(argv), 2) + ix = 0 + setName = argv[0] + user = self._configuration.getString(setName + '.user', '') + target = self._configuration.getString(setName + '.target', '') + factor = self._configuration.getInt(setName + '.factor', 0) + if user == '' or target == '' or factor == '': + self._logger.error('rsync set {} incomplete: user: {} target: {} factor: {}'.format(setName, user, target, '' if factor == 0 else str(factor) )) + else: + time = datetime.datetime.now().time() + hour = int(round(time.hour*60+time.minute / 60.0)) % factor + target = self.replaceMacros(target.replace('%factor%', str(hour))) + cmd = ['/usr/bin/sudo', '-u'] + cmd.append(user) + cmd.append('/usr/bin/rsync') + cmd.append('-a') + if self._verboseLevel >= 2: + cmd.append('-v') + cmd.append('--progress') + argv = argv[1:] + while ix < len(argv): + source = self.replaceMacros(argv[ix]) + cmd.append(source) + ix += 1 + cmd.append(target) + self._processHelper.execute(cmd, self._verboseLevel > 0) + + def jobSaveAllDatabases(self): + '''Saves all databases of a mysql system. + ''' + self._logger.log('saving all databases', 2) + dbTool = appl.DbTool.DbTool(self._globalOptions) + admin = dbTool._configuration.getString('admin.user') + code = dbTool._configuration.getString('admin.code') + dbTool.exportAllDatabases(admin, code, self._targetDir) + + def jobSaveAllDatabases2(self): + '''Saves all databases of a mysql system. + ''' + self._logger.log('saving all databases', 2) + dbs = self.allDatabases(True) + user = self._configuration.getString('mysql.user') + password = self._configuration.getString('mysql.code') + dbTool = appl.DbTool.DbTool(self._globalOptions) + if dbs != None: + for db in dbs: + name = db + currentUser = user + currentPassword = password + config = self.findConfig('db', db, 'webapps.d') + if config != None: + path = config.getString('directory') + if path != None: + name = os.path.basename(path) + '_' + db + currentUser = config.getString('user') + currentPassword = config.getString('password') + if currentPassword == None: + currentUser = user + currentPassword = password + if self._verboseLevel >= 3: + self._logger.log('saving db ' + db) + start = time.time() + outputFile = self._targetDir + os.sep + name + '.sql.gz' + dbTool.exportDatabase(db, currentUser, currentPassword, outputFile) + self.logFile(outputFile, '%f: %s %t created in %r', start) + + def jobSaveDirByTar(self, argv): + '''Saves some directories in some tar archivs. + @param argv: contains the source directories + ''' + self._logger.log('jobSaveDirByTar' + ' '.join(argv), 2) + + for source in argv: + if not source.startswith('@'): + self._logger.log('saving directory {:s} into tar'.format(source), 3) + name = self.fullPathToName(source) + self.saveDirectoryByTar(name, source, self._targetDir, None) + else: + pattern = self.replaceMacros(source[1:]) + configSet = appl.BackupBase.ConfigurationSet(pattern, self._logger) + for arguments in configSet.nextParameters(): + opts = None + if len(arguments) < 1: + self._logger.error('jobSaveDirByTar: configuration pattern does not contain at least 1 variable (directory[, excluded]): {} file: {}' + .format(pattern, configSet._lastHandledFile)) + break + else: + path = arguments[0] + if len(arguments) > 1: + excluded = arguments[1] + if excluded != None and len(excluded) > 2: + listExcluded = excluded[1:].split(excluded[0]) + for entry in listExcluded: + if entry != '': + if opts == None: + opts = ['--exclude=' + entry] + else: + opts.append('--exclude=' + entry) + name = self.fullPathToName(path) + self._logger.log('saving directory {:s} into tar'.format(source), 3) + self.saveDirectoryByTar(name, path, self._targetDir, opts) + + def jobSaveDirByZip(self, argv): + '''Saves some directories in some tar archives. + Note: target is taken from self._targetDir + @param argv: contains the source directories + ''' + if len(argv) < 1: + self._logger.error('SaveDirByZip: missing source') + else: + self._logger.log('jobSaveDirByZip' + ' '.join(argv), 2) + for source in argv: + if not source.startswith('@'): + name = self.fullPathToName(source) + self._logger.log('saving {:s} into {:s}'.format(source, name), 3) + self.saveDirectoryByZip(name, source, self._targetDir, None) + else: + pattern = self.replaceMacros(source[1:]) + configSet = appl.BackupBase.ConfigurationSet(pattern, self._logger) + for arguments in configSet.nextParameters(): + opts = None + if len(arguments) < 1: + self._logger.error('jobSaveDirByZip: configuration pattern does not contain at least 1 variable (directory[, excluded]): : {} file: {}' + .format(pattern, configSet._lastHandledFile)) + break + else: + path = arguments[0] + if len(arguments) > 1: + excluded = arguments[1] + if excluded != None and len(excluded) > 2: + listExcluded = excluded[:1].split(excluded[0]) + for entry in listExcluded: + if entry == '': + continue + if opts == None: + opts = ['-x', entry] + else: + opts.append(entry) + name = self.fullPathToName(path) + self._logger.log('saving {:s} into {:s} {:s}'.format(path, name, + '' if opts == None else ' '.join(opts)), 3) + self.saveDirectoryByZip(name, path, self._targetDir, opts) + + def jobSaveDirByZipLatest(self, argv): + '''Saves some directories in some tar archivs but only the "latest" files. + @param argv: contains the source directories + ''' + doIt = False + if len(argv) < 2: + self._logger.error('missing source for SaveDirByZipLatest') + else: + try: + interval = int(argv[0]) + doIt = True + except ValueError: + self._logger.error('SaveDirByZipLatest: argument 1 is not an integer: ' + argv[0]) + if doIt: + self._logger.log('jobSaveDirByZipLatest' + ' '.join(argv), 2) + for source in argv[1:]: + if not source.startswith('@'): + name = self.fullPathToName(source) + self._logger.log('saving {:s} into {:s}'.format(source, name), 3) + self.saveDirectoryByZip(name, source, self._targetDir, None, interval) + else: + pattern = self.replaceMacros(source[1:]) + configSet = appl.BackupBase.ConfigurationSet(pattern, self._logger) + for arguments in configSet.nextParameters(): + if len(arguments) < 1: + self._logger.error('jobSaveDirByZipLatest: configuration pattern does not contain 1 variable (directory): : {} file: {}' + .format(pattern, configSet._lastHandledFile)) + break + else: + if len(arguments) > 1: + self._logger.log("{} contains ignored parameter(s) below {}".format(configSet._lastHandledFile, arguments[0])) + path = arguments[0] + name = self.fullPathToName(path) + self._logger.log('saving {:s} into {:s}'.format(path, name), 3) + self.saveDirectoryByZip(name, path, self._targetDir, None, interval) + + def jobSaveMysql(self, argv): + '''Dumps a mysql database into a gzipped sql file. + @param argv: contains a list of configuration patterns, + e.g.['@%etc%/webapps.d/*.conf:db:user:password:sql.file'] + ''' + self._logger.log('jobSaveMysql ' + ' '.join(argv), 2) + dbTool = appl.DbTool.DbTool(self._globalOptions) + for pattern in argv: + if pattern.startswith('@'): + pattern = self.replaceMacros(pattern[1:]) + configSet = appl.BackupBase.ConfigurationSet(pattern, self._logger) + for arguments in configSet.nextParameters(): + if len(arguments) != 4: + self._logger.error('jobSaveMysql: configuration pattern does not contain 4 variables (db,user,pw,sqlfile): : {} file: {}' + .format(pattern, configSet._lastHandledFile)) + break + else: + domain = os.path.basename(configSet._lastHandledFile)[0:-5] + self._logger.log('saving db {:s}'.format(arguments[0]), 3) + # self.saveMysql(arguments[3], arguments[0], arguments[1], arguments[2], self._targetDir) + dbTool.exportDatabase(arguments[0], arguments[1], arguments[2], self._targetDir + os.sep + domain + '.sql.gz') + + def jobSynchronize(self, argv): + '''Synchronizes a directory from another. + @param argv: contains a list of configuration patterns, + e.g.['@%etc%/dirs.d/rsync_*.conf:source:target:deletetoo'] + ''' + self._logger.log('jobSynchronize ' + ' '.join(argv), 2) + ix = 0 + deleteToo = False + while ix < len(argv): + pattern = argv[ix] + ix += 1 + if pattern.startswith('--delete'): + if ix >= len(argv): + break + deleteToo = True + pattern = pattern[8:].lower() + if pattern.startswith('=f'): + deleteToo = False + pattern = argv[ix] + ix += 1 + if pattern.startswith('@'): + pattern = self.replaceMacros(pattern[1:]) + configSet = appl.BackupBase.ConfigurationSet(pattern, self._logger) + for arguments in configSet.nextParameters(): + if len(arguments) < 2: + self._logger.error('jobSynchronize: configuration pattern does not contain 2 variables (source,target[,deleteToo]): : {} file: {}' + .format(pattern, configSet._lastHandledFile)) + break + else: + deleteToo2 = deleteToo + if (len(arguments) > 2): + deleteToo2 = arguments[2].startswith('T') or arguments[2].startswith('t') + source = self.replaceMacros(arguments[0]) + target = self.replaceMacros(arguments[1]) + self.ensureDirectory(target) + self.synchronize(source, target, deleteToo2) + else: + source = self.replaceMacros(pattern) + if ix >= len(argv): + self._logger.error('jobSynchronize: missing target to source ' + pattern) + else: + target = self.replaceMacros(argv[ix]) + ix += 1 + self.ensureDirectory(target) + self.synchronize(source, target, deleteToo) + + def jobSaveWebApps(self, argv): + '''for all webapps: dump the database into the directory <base>/db + @param argv: contains a list of configuration patterns, + e.g.['@%etc%/webapps.d/*.conf:db:user:password:sql.file:directory'] + ''' + self._logger.log('jobSaveWebapps ' + ' '.join(argv), 2) + for pattern in argv: + if pattern.startswith('@'): + pattern = self.replaceMacros(pattern[1:]) + configSet = appl.BackupBase.ConfigurationSet(pattern, self._logger) + for arguments in configSet.nextParameters(): + if len(arguments) != 4: + self._logger.error('jobSaveMysql: configuration pattern does not contain 4 variables (db,user,pw,directory): : {} file: {}' + .format(pattern, configSet._lastHandledFile)) + break + else: + # config file without '.conf': + domain = os.path.basename(configSet._lastHandledFile)[0:-5] + db = arguments[0] + user = arguments[1] + code = arguments[2] + target = arguments[3] + if not os.path.isdir(target): + self._logger.error('missing directory {} for {}'.format(target, domain)) + else: + self._logger.log('saving db {:s}'.format(arguments[0]), 3) + tool = appl.DbTool.DbTool(self._globalOptions) + target += os.sep + 'db' + self.ensureDirectory(target) + tool.exportDatabase(db, user, code, target + os.sep + domain + '.sql.gz') + + def sendStatusEmail(self, subject, body): + '''Sends an email. + @param subject the email's subject + @param body the emails's content + ''' + recipient = self._configuration.getString('admin.email') + sender = self._configuration.getString('smtp.sender'), + host = self._configuration.getString('smtp.host') + port = self._configuration.getString('smtp.port') + user = self._configuration.getString('smtp.user') + code = self._configuration.getString('smtp.code') + value = self._configuration.getString('smtp.tls') + withTls = value != None and value.lower().startswith('t') + if not (recipient and sender and host and port and user and code): + self._logger.error('email configuration is wrong') + else: + net.EMail.sendStatusEmail(recipient, subject, body, sender, host, port, user, code, withTls, self._logger) + + def snapshot(self, argv): + '''Makes a snapshot. + @param argv: program arguments, e.g. ['dayly', '/media/cloud/mycloud/data', '--create-if-missing'] + ''' + (argv, opts) = self.splitArgsAndOpts(argv) + if len(argv) < 2: + self.usage('missing arguments') + else: + mode = argv[0] + dataDir = argv[1] + baseDir = os.path.dirname(dataDir) + snapshotBase = baseDir + os.sep + mode + argv = argv[2:] + createIfMissing = False + if len(opts) > 0: + for opt in opts: + if opt == '--create-if-missing': + createIfMissing = True + else: + self.usage('unknown option: ' + opt) + if mode not in ['dayly', 'weekly', 'monthly']: + self.usage('invalid mode {}: expected: dayly or weekly or monthly'.format(mode)) + elif not os.path.isdir(dataDir): + self.usage('not a data directory: ' + dataDir) + elif baseDir == '': + self.usage('missing the parent directory: ' + dataDir) + elif not base.FileHelper.distinctPaths(dataDir, snapshotBase): + self.usage('nested directories: {} [] / {} [{}]'.format(dataDir, os.path.realpath(dataDir), + snapshotBase, os.path.realpath(snapshotBase))) + else: + if not os.path.isdir(snapshotBase): + if not createIfMissing: + self.usage('not a snapshot directory: ' + snapshotBase) + else: + self.ensureDirectory(snapshotBase) + tool = appl.DirTool.DirTool(self._globalOptions) + snapshotDir = baseDir + self.relativeBackupPath(mode) + if os.path.exists(snapshotDir): + shutil.rmtree(snapshotDir, False) + if os.path.exists(snapshotDir): + os.rename(snapshotDir, snapshotDir + '.{:.0}'.format(time.time())) + tool.snapshot([dataDir, snapshotDir]) + + def smartInfo(self): + '''Assembles the SMART infos about some disk devices. + @return a list of lines describing the SMART info + ''' + devices = self._configuration.getString('smart.devices').split(',') + output = ['= SMART info'] + for device in devices: + output.append('== ' + device) + lines = self._processHelper.executeInputOutput(['/usr/sbin/smartctl', '-a', device]) + output += lines + return output + + def systemInfo(self, target): + '''Assembles interesting informations about the current linux system. + Manually installed packets + disk devices + mounts + @param target: the target directory + ''' + start = time.time() + # zcat /var/log/apt/history.log.*.gz | cat - /var/log/apt/history.log | grep -Po '^Commandline: apt-get install (?!.*--reinstall)\K.*' > $fnManuallyInstalled + path = '/var/log/apt' + nodes = os.listdir(path) + packets = [] + for node in nodes: + if node.startswith('history.log') and node.endswith('.gz'): + full = path + os.sep + node + with gzip.open(full, 'r') as fp: + for line in fp: + if line.startswith(b'Commandline: apt-get install'): + packets += line[29:].decode().strip().replace(' ', ' ').replace(' ', ' ').split(' ') + with open('/var/log/apt/history.log', 'r') as fp: + for line in fp: + if line.startswith('Commandline: apt-get install'): + packets += line[29:].strip().replace(' ', ' ').replace(' ', ' ').split(' ') + + mounts = [] + with open('/proc/mounts', 'r') as fp: + for line in fp: + parts = line.split(' ') + if parts[0].find('/') >= 0: + mounts.append(line) + lsblkInfo = subprocess.check_output('/bin/lsblk') + fdiskInfo = subprocess.check_output(['/sbin/fdisk', '-l']) + networkInfo = subprocess.check_output(['/sbin/ip', 'addr']) + routeInfo = subprocess.check_output(['/sbin/ip', 'route', 'show']) + with open('/proc/diskstats', 'r') as fp: + diskStats = fp.read() + lvmInstalled = self._beingRoot and os.path.exists('/sbin/vgdisplay') + if lvmInstalled and self._beingRoot: + pvInfoShort = subprocess.check_output('/sbin/pvs') + pvInfo = subprocess.check_output('/sbin/pvdisplay') + vgInfoShort = subprocess.check_output('/sbin/vgs') + vgInfo = subprocess.check_output('/sbin/vgdisplay') + lvInfoShort = subprocess.check_output('/sbin/lvs') + lvInfo = subprocess.check_output('/sbin/lvdisplay') + fnOutput = target + os.sep + 'system.info.gz' + if os.geteuid() == 0: + btrfsInfo = self.btrfsInfo() + smartInfo = self.smartInfo() + with gzip.open(fnOutput, 'wb') as fp: + fp.write(b'=== manually installed packets:\n') + for packet in packets: + if packet != '': + fp.write('apt-get install -y {:s}\n'.format(packet).encode()) + fp.write(b'\n=== lsblk:\n' + lsblkInfo) + fp.write(b'\n=== fdisk:\n' + fdiskInfo) + fp.write(b'\n=== mounts:\n' + ''.join(mounts).encode()) + fp.write(b'\n=== diskstats:\n' + diskStats.encode()) + if os.path.exists('/proc/mdstat'): + with open('/proc/mdstat', 'r') as fp2: + mdstat = fp2.read() + fp.write(b'\n=== mdstat:\n' + mdstat.encode()) + if lvmInstalled and self._beingRoot: + fp.write(b'\n=== pvs:\n' + pvInfoShort) + fp.write(b'\n=== vgs:\n' + vgInfoShort) + fp.write(b'\n=== lvs:\n' + lvInfoShort) + fp.write(b'\n=== pvdisplay:\n' + pvInfo) + fp.write(b'\n=== vgdisplay:\n' + vgInfo) + fp.write(b'\n=== lvdisplay:\n' + lvInfo) + if btrfsInfo != '': + fp.write(btrfsInfo.encode('utf-8')) + fp.write(b'\n=== ip addr:\n' + networkInfo) + fp.write(b'\n=== ip route:\n' + routeInfo) + self.logFile(fnOutput, '%f: %s %t created in %r', start) + + def testCompleteness(self, target): + '''Tests whether the backup is complete. + ''' + target = self._targetDir + dbs = self.allDatabases() + for db in dbs: + name = db + config = self.findConfig('db', db, 'webapps.d') + if config != None: + path = config.getString('directory') + if path != None: + name = os.path.basename(path) + '_' + db + outputFile = target + os.sep + name + '.sql.gz' + if not os.path.exists(outputFile): + self._logger.error('missing SQL save: ' + outputFile) + else: + self._logger.log('SQL file found: ' + outputFile, 3) + + def testReady(self, mode): + '''Tests whether a backup run has been finished normally. + @param mode: 'dayly', 'weekly', 'monthly' + ''' + self._logger.log('Testing whether the last backup mode [{}] has been finished normally'.format(mode), 1) + self._baseTargetDir = self._configuration.getString('target.path') + if self.ensureDirectory(self._baseTargetDir) != None: + [lockFile, readyFile] = self.createLockAndReadyFile(mode) + if os.path.exists(readyFile): + self._logger.log('ready file found: {}'.format(readyFile), 1) + else: + if not os.path.exists(lockFile): + error = 'backup [{}] not ready but no lockfile exists'.format(mode) + else: + date = datetime.datetime.fromtimestamp(os.stat(lockFile).st_mtime) + error = 'backup [{}] not ready. Start time: {}'.format(mode, date.strftime('%Y.%m.%d %H:%M:%S')) + self._logger.error(error) + subject = 'Backup {:s} not ready on {:s}'.format(mode, self._configuration.getString('location', base.BaseTool.BasicStatics.hostname(True))) + self.sendStatusEmail(subject, error) + +def usage(): + '''Returns an info about usage. + ''' + return """usage: backuptool [<global_opts>] <mode> + Dayly/weekly/monthly backup including system info and mysql backup +GLOBAL_OPTS +GLOBAL_MODES +<mode>: + completeness + tests whether the last backup task was successfully completed + dayly + dayly actions + monthly + monthly actions + snapshot mode data-dir [--create-if-missing] + makes a snapshot for the given mode and directory + mode: dayly or weekly or monthly + data-dir: the directory laying in the base directory containing 'dayly' ... + create-if-missing:the mode specific base (data-dir../dayly ...) will be created if needed + test-email + Sends an email to test email configuration + test-ready <mode> + Tests whether the last backup with <mode> has been finished successfully + webapps <password-backup-user> <pattern-nginx-sites-enabled1> ... + Creates a configuration from nginx configuration + weekly + weekly actions +Examples: +backuptool -q backup dayly +backuptool -r -v3 backup weekly + """ + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/backuptool', '-v3', 'backup', 'dayly'] + ''' + appInfo = base.BaseTool.ApplicationInfo('backup', 'appl/BackupTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = BackupTool(options) + rc = None + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'backup': + if len(argv) == 0: + mode = 'dayly' + else: + mode = argv[0] + rc = tool.doBackup(mode) + elif cmd == 'webapps': + if len(argv) < 2: + tool.usage('missing password and/or pattern(s)') + else: + rc = tool.createWebapp(argv[0], argv[1:]) + elif cmd == 'test-email': + tool.sendStatusEmail('Test email sending', 'Seems to work: email sending'); + elif cmd == 'test-ready': + if len(argv) < 1: + tool.usage('missing <mode> for command test-ready') + else: + mode = argv[0] + if mode != 'dayly' and mode != 'weekly' and mode != 'monthly': + tool.usage('unknown mode for command test-ready: ' + mode) + else: + tool.testReady(mode) + elif cmd == 'completeness': + tool.testCompleteness() + elif cmd == 'snapshot': + tool.snapshot(argv) + else: + tool.usage('unknown command: ' + cmd) + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/BenchTool.py b/appl/BenchTool.py new file mode 100755 index 0000000..1d5feb2 --- /dev/null +++ b/appl/BenchTool.py @@ -0,0 +1,243 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' +import time +import sys +import os.path +import math + +sys.path.insert(0, '/usr/share/pyrshell') +import base.BaseTool +import base.CryptoEngine + +class Statistics: + def __init__(self): + self._readOnly = False + self._readCount = 0 + self._bytesRead = 0 + self._writeCount = 0 + self._bytesWritten = 0 + self._timeRead = 0 + self._timeWrite = 0 + +class BenchTool (base.BaseTool.BaseTool): + + def __init__(self, globalOptions): + '''Constructor. + @param logger: the logger + ''' + self._globalOptions = globalOptions + base.BaseTool.BaseTool.__init__(self, globalOptions, 'benchtool.conf') + self._dirs = [] + self._files = [] + self._written = [] + self._factorPrint = 10000 + self._random = base.CryptoEngine.CryptoEngine(self._logger) + self._random.setSeedRandomly() + self._ratioRead = 0.80 + self._maxWritten = 1000 + self._statistics = Statistics() + self._bufferLength = 8192 + self._buffer = self._random.nextString(self._bufferLength, 'ascii95') + # in Blocks + self._maxFileLength = 4092 + self._ratioSmallFiles = 0.8 + self._fileNo = 0 + self._maxLoops = 0x7fffffffffff + self._maxWritten = 1000 + self._readOnly = False + + def benchFiles(self): + loopNo = 0 + while loopNo < self._maxLoops: + loopNo += 1 + if self._readOnly: + self.benchRead() + else: + current = self._random.nextInt(100000) + if current < 100000*self._ratioRead: + self.benchRead() + else: + self.benchWrite() + if loopNo % self._factorPrint == 0: + self.fileStatistics() + self.fileStatistics() + + def benchRead(self): + '''Reads a randomly choosen file from the tree. + ''' + self._statistics._readCount += 1 + ix = self._random.nextInt(len(self._files)) + start = time.time() + with open(self._files[ix], 'rb') as fp: + again = True + while again: + buffer = fp.read(self._bufferLength) + if len(buffer) == 0: + again = False + else: + self._statistics._bytesRead += len(buffer) + self._statistics._timeRead += time.time() - start + + def benchWrite(self): + '''Reads a randomly choosen file from the tree. + ''' + self._statistics._writeCount += 1 + if len(self._written) >= self._maxWritten: + ix = self._random.nextInt(len(self._written)) + os.unlink(self._written[ix]) + del self._written[ix] + ix = self._random.nextInt(len(self._files)) + start = time.time() + with open(self._files[ix], 'w') as fp: + if self._random.nextInt(100000) < 100000 * self._ratioSmallFiles: + # small files + uBound = self._random.nextInt(self._bufferLength, 16) + fp.write(self._buffer[0:uBound]) + self._statistics._bytesWritten += uBound + else: + for ix in range(self._random.nextInt(self._maxFileLength, 4)): + fp.write(self._buffer) + self._statistics._bytesWritten += self._bufferLength + self._statistics._timeWrite += time.time() - start + + def buildName(self): + '''Builds a unique filename inside the given file tree. + ''' + self._fileNo += 1 + path = self._dirs[self._random.nextInt(len(self._dirs))] + name = '{}/x!%{}.bench'.format(path, self._fileNo) + return name + + def calculate(self, argv): + '''Calulate benchmark: reads and write files. + @param argv: the program arguments, e.g. ['/home', '--read-only'] + ''' + def div(a,b): + try: + if abs(b) < 1E-20: + b = 1E-20 + rc = a / b + except ZeroDivisionError as exc: + print(str(exc)) + return rc + + args, options = self.splitArgsAndOpts(argv) + for opt in options: + if opt.startswith('--max-loops='): + self._maxLoops = self.integerOption(opt) + elif opt.startswith('--max-written='): + self._maxWritten = self.integerOption(opt) + elif opt.startswith('--factor-print='): + self._factorPrint = self.integerOption(opt) + else: + self.usage('unknown option: ' + opt) + loops = 0 + x = 1.03 + while loops < self._maxLoops: + loops += 1 + for ix in range(10000): + x = x * ix + math.sqrt(abs(x)*3) + x -= abs(math.pow(0.1 + abs(x), 0.33)) + x *= math.sin(x+div(loops, div(loops, loops * 5.3333))) + math.log(1+abs(x*x)) + a = 0.03 + abs(min(20 - div(x, (abs(x) + 10)), abs(div(x, (x + math.cos(div(x + 99, x + 3))))))) + x -= math.exp(a) + x = max(min(x, 1E10), -1E10) + div(ix + 77, -ix -37) + self._random.nextInt(100000)/100000 + if loops % self._factorPrint == 0: + self._logger.log('Calculate: {} loops: {:.3f} k'.format(x, loops / 1000.0)) + time.sleep(0.001) + self._result = x + + def files(self, argv): + '''File benchmark: reads and write files. + @param argv: the program arguments, e.g. ['/home', '--read-only'] + ''' + args, options = self.splitArgsAndOpts(argv) + for opt in options: + if opt.startswith('--max-loops='): + self._maxLoops = self.integerOption(opt) + elif opt.startswith('--max-written='): + self._maxWritten = self.integerOption(opt) + elif opt.startswith('--factor-print='): + self._factorPrint = self.integerOption(opt) + else: + self.usage('unknown option: ' + opt) + if len(argv) < 1: + self.usage('missing directory') + else: + baseDir = argv[0] + if not os.path.isdir(baseDir): + self.usage('not a directory: ' + baseDir) + else: + self.findFiles(baseDir) + self.benchFiles() + + def fileStatistics(self): + readMb = self._statistics._bytesRead / 1E6 + writeMb = self._statistics._bytesWritten / 1E6 + self._logger.log('read : files: {:4d} MBytes: {:10.3f} rate: {:10.3f} MByte/s'.format( + self._statistics._readCount, readMb, readMb / (self._statistics._timeRead + 0.000001))) + self._logger.log('write: files: {:4d} MBytes: {:10.3f} rate: {:10.3f} MByte/s'.format( + self._statistics._writeCount, writeMb, writeMb / (self._statistics._timeWrite + 0.000001))) + + def findFiles(self, baseDir): + '''Builds the list of the files in the given directory tree + @param baseDir: the base directory of the file tree + ''' + for node in os.listdir(baseDir): + full = baseDir + os.sep + node + if os.path.isdir(full): + self._dirs.append(full) + self.findFiles(full) + else: + self._files.append(full) + if len(self._files) % self._factorPrint == 0: + self._logger.log('files/dirs found: {}/{}'.format(len(self._files), len(self._dirs)), 2) + + def example(self): + example = '''# benchtool example configuration +''' + self.storeExample(example) + +def usage(): + return """usage: benchtool [<global_opts>] <mode> <args> + Benchmarks and/or burn in. +GLOBAL_OPTS +GLOBAL_MODES +<mode>: + files <baseDir> [--read-only] [--max-loops=<loops>] [--max-written=<written>] [--factor-print=<factor>] + Read and write randomly files. + calculate + Calculates some things + +Example: +benchtool -v0 files /media/pool +benchtool files /home --read-only +""" + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/dirtool', 'run'] + ''' + appInfo = base.BaseTool.ApplicationInfo('benchtool', 'appl/BenchTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = BenchTool(options) + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'files': + tool.files(argv) + elif cmd == 'calculate': + tool.calculate(argv) + else: + tool.usage("unknown command: " + cmd) + base.BaseTool.setLatestTool(tool) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/DbTool.py b/appl/DbTool.py new file mode 100755 index 0000000..670e5c5 --- /dev/null +++ b/appl/DbTool.py @@ -0,0 +1,941 @@ +#! /usr/bin/python3 +''' +DbTool: Scalable Vector Graphics tool + +@author: hm +''' +import os.path +import sys +import re +import time +import tempfile + +sys.path.insert(0, '/usr/share/pyrshell') +import base.BaseTool +import base.JavaConfig +import base.ProcessHelper + +DbToolResult = None + +def setResult(data): + global DbToolResult + DbToolResult = data + +def result(): + global DbToolResult + return DbToolResult + +def removeFromArrayIfExists(anArray, item): + for ix in reversed(range(len(anArray) )): + if item == anArray[ix]: + del(anArray[ix]) + +class DbTool (base.BaseTool.BaseTool): + '''Translates from one wiki syntax into another. + ''' + def __init__(self, globalOptions): + '''Constructor. + @param globalOptions: the basic configurations + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'dbtool.conf') + self._logger.setErrorFilter(re.compile(r'mysqldump:.*Warning.*Using a password'), True) + self._processTool = base.ProcessHelper.ProcessHelper(self._verboseLevel, self._logger) + # True: delete user/db without confirmation: use only in unittests! + self._forceDeleting = False + + def archiveWebapp(self, argv): + '''Deletes a web application: + Saving of the database, creation of a tar archive with directory and sql script. + Removing database and directory. + @param argv: program arguments, e.g. ['huber.de', '/media/backup/archive'] + ''' + if len(argv) < 2: + self.usage('too few arguments') + else: + argv, options = self.splitArgsAndOpts(argv) + if len(options) != 0: + usage('unknown option(s): ' ' + '.join(options)) + (admin, adminCode) = defaultAdmin(argv, 2, self) + domain = argv[0] + dirArchive = argv[1] + fnConfig = self.getTarget('/etc/pyrshell/webapps.d', domain + '.conf') + fnArchive = dirArchive + os.sep + domain + '.tgz' + if not os.path.isdir(dirArchive): + self.usage('missing archive dir: ' + dirArchive) + elif os.path.exists(fnArchive): + self.usage('archive already exists: ' + fnArchive) + elif not os.path.exists(fnConfig): + self.usage('missing ' + fnConfig) + else: + config = base.JavaConfig.JavaConfig(fnConfig, self._logger) + db = config.getString('db') + user = config.getString('user') + dirWebapp = config.getString('directory') + if not os.path.isdir(dirWebapp): + self.usage('missing directory of {}: {}'.format(domain, dirWebapp)) + else: + dirDb = dirWebapp + os.sep + 'db' + self.ensureDirectory(dirDb) + fnSql = dirDb + os.sep + domain + '.sql.gz' + self.exportDatabase(db, admin, adminCode, fnSql) + size = os.path.getsize(fnSql) + fnNginx = self.getTarget('/etc/nginx/sites-available', domain) + base.FileHelper.copyIfExists(fnNginx, dirDb + os.sep + domain + '.nginx', None, self._logger) + base.FileHelper.copyIfExists(fnConfig, dirDb + os.sep + domain + '.conf', None, self._logger) + self._logger.log('{} exported DB {}'.format(base.StringUtils.formatSize(size), fnSql)) + nodes = os.listdir(dirDb) + self._logger.log('content of {}:\n{}'.format(dirDb, ' '.join(nodes))) + self._processHelper.execute(['/usr/bin/du', '-sh', dirWebapp], True) + self.saveDirectoryByTar(domain, dirWebapp, dirArchive, None) + size = os.path.getsize(fnArchive) + self._logger.log('{}: archive {}'.format(base.StringUtils.formatSize(size), fnArchive)) + fnScript = base.FileHelper.tempFile('Remove.' + domain) + base.StringUtils.toFile(fnScript, '''#! /bin/bash +dbtool -v3 delete-db-and-user {} {} --no-saving $1 +rm -Rf {} {} {} +rm -f /etc/ssl/certs/{}.pem /etc/ssl/private/{}.pem /etc/nginx/sites-enabled/{} +'''.format(db, user, dirWebapp, fnNginx, fnConfig, domain, domain, domain)) + self._logger.log('ToDo: bash {}'.format(fnScript)) + + def buildArgvMysql(self, db, user, passwd): + '''Creates the argument vector to start the command mysql + @param db: the db to handle + @param user: the user for login + @param passwd: the password of user: None or '-': no password given + @return: the program arguments + ''' + argv = ['/usr/bin/mysql', '-u', user] + if passwd != None and len(passwd) > 0 and passwd != '-': + argv.append('-p' + passwd) + argv.append(db) + return argv; + + def createAdmin(self, admin, adminPassword, readOnly, superuser='root', superuserPassword=None): + '''Creates an user able to process all databases + @param admin: new user + @param adminPassword: the password of user + @param readOnly: True: the new user may not change any database + @param superuser: a user which can write to mysql + @param superuserPassword: the password of user + ''' + argv = self.buildArgvMysql('mysql', superuser, superuserPassword) + sql = '''GRANT {} ON *.* TO '{}'@'localhost' IDENTIFIED BY '{}'{}; +flush privileges;'''.format('SELECT, SHOW VIEW' if readOnly else 'ALL', admin, adminPassword, '' if readOnly else ' WITH GRANT OPTION') + self._logger.log('creating admin {}...'.format(admin), 1) + self._logger.log(sql, 4) + self._processTool.executeInput(argv, self._verboseLevel >= 2, sql) + + # create-and-import-all-webapps <dir-backup> <admin> <admin-passwd> + def createAndImportAllWebApps(self, dirBackup, adminUser, adminPasswd): + '''Creates for all webapps the DB and the user stored in the configuration and imports the SQL file. + @param backupDir: the directory containing the SQL files for import + @param adminUser: a user which can read mysql + @param adminPasswd: the password of adminUser + ''' + path = os.path.dirname(self.getTarget(self._configDir + '/webapps.d', 'dummy')) + files = os.listdir(path) + count = 0 + for filename in files: + if filename.endswith('.conf'): + count += 1 + config = base.JavaConfig.JavaConfig(path + os.sep + filename, self._logger) + db = config.getString('db') + user = config.getString('user') + passwd = config.getString('password') + domain = filename[0:-5] + sqlFileNode = config.getString('sql.file', 'sql.file_is_not_defined') + sqlFile = dirBackup + os.sep + sqlFileNode + '.sql.gz' + if db == None or user == None or passwd == None: + self._logger.error('missing auth data in ' + filename) + else: + self.createDbAndUser(db, user, passwd, adminUser, adminPasswd) + self.importWebApp(domain, sqlFile) + self._logger.log("= {} file(s) scanned".format(count), 2) + + def createAndImportWebApp(self, domain, backupDir, adminUser, adminPasswd): + '''Creates for all webapps the DB and the user stored in the configuration. + @param adminUser: a user which can read mysql + @param adminPasswd: the password of adminUser + ''' + path = os.path.dirname(self.getTarget(self._configDir + '/webapps.d', 'dummy')) + filename = path + os.sep + domain + '.conf' + config = base.JavaConfig.JavaConfig(filename, self._logger) + db = config.getString('db') + user = config.getString('user') + passwd = config.getString('password') + sqlFileNode = config.getString('sql.file', 'undefined_sql.file') + '.sql' + sqlFile = backupDir + os.sep + sqlFileNode + '.gz' + if not os.path.isdir(backupDir): + self.usage('not a directory: ' + backupDir) + elif not os.path.exists(sqlFile): + self.usage('missing SQL file: ' + sqlFile) + elif db == None or user == None or passwd == None: + self._logger.error('missing auth data in ' + filename) + else: + self.createDbAndUser(db, user, passwd, adminUser, adminPasswd) + self.importWebApp(domain, sqlFile) + + def createBackupAdmin(self, admin, adminPasswd): + '''Creates the backup admin stored in backup.conf. + @param admin: a user which can write to mysql + @param adminPasswd: the password of user + ''' + config = base.JavaConfig.JavaConfig('/etc/pyrshell/backup.conf', self._logger) + user = config.getString('mysql.user') + passw = config.getString('mysql.code') + if user != None and passw != None: + self.createBackupUser(user, passw, admin, adminPasswd) + + # create-db-and-user <db> <user> <passwd> <admin> <admin-pw> + def createDbAndUser(self, db, user, passwd, adminUser, adminPasswd): + '''Creates an user able to read all databases (not write) + @param db: name of the new database + @param user: new user + @param passwd: the password of user + @param adminUser: a user which can write to mysql + @param adminPasswd: the password of adminUser + ''' + argv = self.buildArgvMysql('mysql', adminUser, adminPasswd) + sql = '''GRANT ALL ON {}.* TO '{}'@'localhost' IDENTIFIED BY '{}' WITH GRANT OPTION; +flush privileges; +create database if not exists {};'''.format(db, user, passwd, db) + self._logger.log(' '.join(argv) + '\n' + sql) + self._logger.log('creating db {} and user {}...'.format(db, user), 1) + self._processTool .executeInput(argv, self._verboseLevel >= 2, sql) + + def createBackupUser(self, backupUser, backupPassword, user='root', userPassword=None): + '''Creates an user able to read all databases (not write) + @param backupUser: new user + @param backupPassword: the password of user + @param user: a user which can write to mysql + @param userPassword: the password of user + ''' + argv = self.buildArgvMysql('mysql', user, userPassword) + sql = '''GRANT USAGE ON *.* TO '{}'@'localhost' IDENTIFIED BY '{}'; +GRANT SELECT, LOCK TABLES ON mysql.* TO '{}'@'localhost'; +GRANT SELECT, LOCK TABLES, SHOW VIEW, EVENT, EXECUTE, TRIGGER ON *.* TO '{}'@'localhost' IDENTIFIED BY '{}'; +flush privileges;'''.format(backupUser, backupPassword, backupUser, backupUser, backupPassword) + self._logger.log('creating readonly user {}...'.format(backupUser), 1) + self._processTool.executeInput(argv, self._verboseLevel >= 2, sql) + + def createWebAppConfiguration(self, argv): + '''Creates a configuration file for a web application. + @param argv: the program arguments, e.g. ['example.com', '/home/example.com', 'dbexample', 'uexample', 'TopSecret'] + ''' + if len(argv) < 5: + self.usage('too few arguments') + else: + domain = argv[0] + directory = argv[1] + db = argv[2] + user = argv[3] + password = argv[4] + if re.match(r'^[\w.-]+$', domain) == None: + self.usage('illegal characters in domain: ' + domain) + if re.match(r'^[\w.-]+$', db) == None: + self.usage('illegal characters in db: ' + db) + if not os.path.isdir(directory): + self.usage('directory does not exists: ' + directory) + fn = self.getTarget('/etc/pyrshell/webapps.d', domain + '.conf') + base.StringUtils.toFile(fn, '''db={} +user={} +password={} +sql.file={} +directory={} +excluded= +'''.format(db, user, password, domain + '_' + db, directory)) + + def deleteDb(self, db, admin, passwd, doConfirm = True, doSave = True): + '''Deletes the database. + @param db: the db to delete + @param admin: a user with enough rights to delete the user + @param passwd: the password of the admin + @param doConfirm: True: the deletion must be confirmed (by typing the database name) + @param doSave: True: the database is saved into the tempororary directory + ''' + argv = self.buildArgvMysql('mysql', admin, passwd) + answer = db + if doConfirm and not self._forceDeleting and not base.BaseTool.unitTestIsActive(): + print('Do you want to delete db {}? Please enter the db name again: '.format(db)) + answer = input() + if answer != db: + self._logger.error('confirmation failed') + else: + self._logger.log('deleting database {}...'.format(db), 1) + sql = 'drop database `{}`;'.format(db) + self._processTool .executeInput(argv, self._verboseLevel >= 2, sql) + + def deleteDbAndUser(self, argv): + '''Deletes the database. + @param argv: program arguments, e.g. ['wordpress', 'jonny', '--no-save', '--no-confirmation'] + ''' + if len(argv) < 1: + self.usage('too few arguments') + else: + argv, options = self.splitArgsAndOpts(argv) + doConfirmation = True + doSaving = True + for opt in options: + if opt == '--no-confirmation': + doConfirmation = False + elif opt == '--no-saving': + doSaving = False + else: + self.usage('unknown option: ' + opt) + (admin, code) = defaultAdmin(argv, 2, self) + db = argv[0] + user = argv[1] if len(argv) > 1 else '-' + if doSaving: + sqlFile = base.FileHelper.tempFile(db + '.sql.' + str(int(time.time())) + '.gz') + self._logger.log('saving {} ...'.format(sqlFile), 1) + self.exportDatabase(db, admin, code, sqlFile) + self.deleteDb(db, admin, code, doConfirmation, doSaving) + if user != '-': + self.deleteUser(user, admin, code, doConfirmation) + + def deleteUser(self, user, admin, passwd, doConfirm = True): + '''Deletes the database user. + @param user: the user to delete + @param admin: a user with enough rights to delete the user + @param passwd: the password of the admin + @param doConfirm: True: the deletion must be confirmed (by typing the user name) + ''' + argv = self.buildArgvMysql('mysql', admin, passwd) + answer = user + if doConfirm and not self._forceDeleting and not (user == 'testadmin' and base.BaseTool.unitTestIsActive()): + print('Do you want to delete db user {}? Please enter the username again: '.format(user)) + answer = input() + if answer != user: + self._logger.error('confirmation failed') + else: + sql = '''delete from user where user='{}'; +flush privileges;'''.format(user) + self._logger.log('deleting database user {}...'.format(user), 1) + self._processTool .executeInput(argv, self._verboseLevel >= 2, sql) + + def example(self): + '''Creates a example configuration. + ''' + example = '''# dbtool example configuration +log.file=/var/log/local/dbtool.log +admin.user=admin +admin.code=TopSecret + ''' + self.storeExample(example) + + def exportAllDatabases(self, user, code, target): + '''Exports a database. + @param db: the DB's name + @param user: the DB user + @param code: '' or the password + @param target: the target file + ''' + # def findAllDbs(self, adminUser, adminPasswd, internalTo = False): + if user == None: + user = self._configuration.getString('admin.user') + code = self._configuration.getString('admin.code') + if user == None: + # configuration of backuptool: + user = self._configuration.getString('mysql.user') + code = self._configuration.getString('mysql.code') + if user == None: + self._logger.error('exportAllDatabases(): missing admin user') + else: + dbs = self.findAllDbs(user, code) + for db in dbs: + trg = target + os.sep + db + '.sql.gz' + self.exportDatabase(db, user, code, trg) + + def exportDatabase(self, db, user, code, target): + '''Exports a database. + @param db: the DB's name + @param user: the DB user + @param code: '' or the password + @param target: the target file + ''' + argv = ['/usr/bin/mysqldump', '--default-character-set=utf8mb4', '--single-transaction', '-u', user] + if code != '': + argv.append('-p' + code) + argv.append(db) + if target.endswith('.gz'): + self._processHelper.executeScript('''#! /bin/bash +/usr/bin/mysqldump --default-character-set=utf8mb4 --single-transaction -u{} '-p{}' '{}' | gzip -c > {} +'''.format(user, code, db, target)) + else: + self._processHelper.executeScript('''#! /bin/bash +/usr/bin/mysqldump --default-character-set=utf8mb4 --single-transaction -u{} '-p{}' '{}' > {} +'''.format(user, code, db, target)) + + def exportWebApp(self, argv): + '''Exports a web application's database. + @param argv: program arguments, e.g. ['x01.interfacemaker.com', '/tmp/x01.sql.gz'] + ''' + domain = argv[0] + argv = argv[1:] + if len(argv) == 0: + target = tempfile.gettempdir() + os.sep + domain + '.sql' + else: + target = argv[0] + argv = argv[1:] + fnConfig = self.getTarget('/etc/pyrshell/webapps.d', domain + '.conf') + if not os.path.exists(fnConfig): + self.usage('missing ' + fnConfig) + else: + config = base.JavaConfig.JavaConfig(fnConfig, self._logger) + db = config.getString('db') + user = config.getString('user') + code = config.getString('password') + if db == None or user == None or code == None: + self.usage('incomplete data in ' + fnConfig) + else: + self.exportDatabase(db, user, code, target) + + def filterRecords(self, argv): + '''Filters some records from a given table in a given SQL file into a given output SQL file. + @param argv: the program arguments, e.g. ['mysql.sql', 'user', 'users.sql', '--exclude=,root,'] + ''' + argv, options = self.splitArgsAndOpts(argv) + filterFunc = None + regExclude = None + for opt in options: + if opt.startswith('--exclude='): + regExclude = self.regExprCompile(opt[10:].encode('utf-8'), 'filterRecord', False) + else: + self.usage('unknown option: ' + opt) + if regExclude != None: + filterFunc = lambda record, excludeExpr: excludeExpr.search(record) == None + fnIn = argv[0] + table = argv[1] + fnOut = argv[2] + if not os.path.exists(fnIn): + self.usage('missing input file: ' +fnIn) + elif re.search(r'\W', table) != None: + self.usage('illegal char in table name: ' + table) + else: + self.filterRecordsInSqlFile(table, fnIn, fnOut, filterFunc, regExclude) + + def filterRecordsInSqlFile(self, table, sqlInput, sqlOutput, filterFunc, paramFilter): + '''Parses a SQL file and searches for a given table. These tables will be written to another file: + @param table: name of the table to extract + @param sqlInput: name of the input file (created from mysqldump) + @param sqlOutput: name of the output file + @param filterFunc: none or a method to filter the record + if objectFilter != None filter() is a method of objectFilter + otherwise filter() is .filter is a function + signature: filter(record): bool. If it returns True the record is written + @param paramFilter: 2nd parameter of filterFunc() + ''' + if not os.path.exists(sqlInput): + self.usage('missing input SQL file: ' + sqlInput) + else: + with open(sqlInput, 'rb') as fpInput, open(sqlOutput, 'wb') as fpOutput: + state = None + table1 = b' ' + table.encode('utf-8') + b' '; + table2 = b'`' + table.encode('utf-8') + b'`'; + countRecords = 0 + for line in fpInput: + ixStart = 0 + if state == None: + if line.startswith(b'DROP TABLE IF EXISTS'): + state = 'searchTable' + else: + fpOutput.write(line) + elif state == 'searchTable' and (line.startswith(b'CREATE') or line.startswith(b'create')): + if line.find(table1) > 0 or line.find(table2) > 0: + state = 'foundTable' + firstInsert = True + else: + state = 'searchTable' + elif state == 'foundTable': + if line.startswith(b'INSERT') or line.startswith(b'insert'): + countValues = 0 + if firstInsert: + fpOutput.write(b'LOCK TABLES `!` WRITE;\n'.replace(b'!', table.encode('utf-8'))) + firstInsert = False + ixStart = line.find(b'VALUES') + if ixStart < 0: + ixStart = line.find(b'values') + if ixStart < 0: + self._logger.error('missing "VALUES":' + line.encode('utf-8')[0:80]) + ixStart = 0 + else: + ixStart += 6 + if countValues == 0: + fpOutput.write(b'\n') + else: + fpOutput.write(b',\n') + fpOutput.write(line[0:ixStart]) + state = 'inInserts' + elif line.startswith(b'UNLOCK') or line.startswith(b'unlock'): + state = 'end' + fpOutput.write(b'UNLOCK TABLES;\n') + if state == 'inInserts': + if line.startswith(b'UNLOCK') or line.startswith(b'unlock') or line.startswith(b'DROP') or line.startswith(b'drop') or line.startswith(b'CREATE') or line.startswith(b'create'): + state = 'end' + fpOutput.write(b';\nUNLOCK TABLES;\n') + else: + if ixStart == 0: + fpOutput.write(b',') + while True: + record = None + ix = line.find(b'),(', ixStart) + if ix >= 0: + record = line[ixStart:ix+1] + ixStart = ix + 2 + else: + record = line[ixStart:].strip(b'\n\r;,') + ok = filterFunc(record, paramFilter) + if ok: + if countValues == 0: + fpOutput.write(b'\n') + else: + fpOutput.write(b',\n') + fpOutput.write(record) + countValues += 1 + countRecords += 1 + if ix < 0: + break + self._logger.log('found records: {}'.format(countRecords), 2) + + # all-dbs <admin> <admin-passwd> [<internal-too>] + def findAllDbs(self, adminUser, adminPasswd, internalTo = False): + '''Logs the database names (without internal dbs like mysql) + @param adminUser: a user which can read mysql + @param adminPasswd: the password of adminUser + @param internalTo: False: internal databases (mysql, ...) are ignored + @return: a list of all database names + ''' + argv = self.buildArgvMysql('mysql', adminUser, adminPasswd) + sql = '''show databases;''' + rc = self._processTool.executeInputOutput(argv, sql) + if len(rc) > 0 and rc[0] == 'Database': + rc = rc[1:] + removeFromArrayIfExists(rc, '') + if not internalTo: + removeFromArrayIfExists(rc, 'mysql') + removeFromArrayIfExists(rc, 'information_schema') + removeFromArrayIfExists(rc, 'performance_schema') + return rc + + # all-dbs <admin> <admin-passwd> [<internal-too>] + def findAllUsers(self, adminUser, adminPasswd): + '''Logs the database names (without internal dbs like mysql) + @param adminUser: a user which can read mysql + @param adminPasswd: the password of adminUser + @param internalTo: False: internal databases (mysql, ...) are ignored + @return: a list of all database names + ''' + argv = self.buildArgvMysql('mysql', adminUser, adminPasswd) + sql = '''select user from user;''' + self._logger.log(' '.join(argv) + '\n' + sql, 2) + rc = self._processTool.executeInputOutput(argv, sql) + setResult(rc) + return rc + + # create-webapp-dbs <admin> <admin-passwd> + def createWebAppDbsAndUser(self, adminUser, adminPasswd): + '''Creates for all webapps the DB and the user stored in the configuration. + @param adminUser: a user which can read mysql + @param adminPasswd: the password of adminUser + ''' + self.createBackupAdmin(adminUser, adminPasswd) + path = self._configDir + '/webapps.d' + files = os.listdir(path) + count = 0 + for filename in files: + if filename.endswith('.conf'): + count += 1 + config = base.JavaConfig.JavaConfig(path + os.sep + filename, self._logger) + db = config.getString('db') + user = config.getString('user') + passwd = config.getString('password') + if db == None or user == None or passwd == None: + self._logger.error('missing auth data in ' + filename) + else: + self.createDbAndUser(db, user, passwd, adminUser, adminPasswd) + self._logger.log("= {} file(s) scanned".format(count), 2) + + # import-all-webapps <sql-directory> + def importAllWebappDbs(self, sqlDirectory): + '''Imports all webapp dbs from files in a given directory. + Most of the parameters are read from the configuration files. + @param sqlDirectory: the directory containing the SQL files to import + ''' + path = self._configDir + '/webapps.d' + files = os.listdir(path) + count = 0 + for filename in files: + if filename.endswith('.conf'): + count += 1 + config = base.JavaConfig.JavaConfig(path + os.sep + filename, self._logger) + db = config.getString('db') + user = config.getString('user') + passwd = config.getString('password') + sqlFilePrefix = config.getString('sql.file') + if db == None or user == None or passwd == None or sqlFilePrefix == None: + self._logger.error('missing needed data in ' + filename) + else: + fnSql = sqlDirectory + '/' + sqlFilePrefix + '.sql' + if not os.path.exists(fnSql): + fnSql2 = fnSql + '.gz' + if os.path.exists(fnSql2): + fnSql = fnSql2 + else: + self._logger.error('sql not found: ' + fnSql) + fnSql = None + if fnSql != None: + cmd = '/bin/zcat' if fnSql.endswith('.gz') else '/bin/cat' + self._logger.log('importing {} into {}...'.format(fnSql, db), 1) + self._processTool.executeInChain([cmd, fnSql], None, ['/usr/bin/mysql', '-u', user, '-p' + passwd, db]) + + # import-webapp <domain> <sql-file> + def importWebApp(self, domain, sqlFile): + '''Imports a SQL file into the db of a webapp. + @param domain: the domain of the webapp, e.g. 'wolke.infeos.eu' + @param sqlFile: the file to import + ''' + filename = self.getTarget(self._configDir + '/webapps.d', domain + '.conf') + if not os.path.exists(sqlFile): + self._logger.error('sql file does not exist: ' + sqlFile) + elif not os.path.exists(filename): + self._logger.error('not found: ' + filename) + else: + configuration = base.JavaConfig.JavaConfig(filename, self._logger) + db = configuration.getString('db') + user = configuration.getString('user') + passwd = configuration.getString('password') + if db == None or user == None or passwd == None: + self._logger.error('missing needed data in ' + filename) + else: + cmd = '/bin/zcat' if sqlFile.endswith('.gz') else '/bin/cat' + argv1 = [cmd, sqlFile] + argv2 = ['/usr/bin/mysql', '-u', user, '-p' + passwd, db] + self._logger.log('importing {} into {}...'.format(sqlFile, db), 1) + self._processTool.executeInChain(argv1, None, argv2) + + def saveWebApp(self, domain, archive): + '''Saves the db into a subdirectory db and stores the home of the webapp into a tar achive. + @param domain: the domain of the webapp, e.g. 'cloud.infeos.de' + @param archive: the name of the tar achive (will be created) + ''' + fn = self.getTarget('/etc/pyrshell/webapps.d', domain + '.conf') + if not os.path.exists(fn): + self.usage('unknown web application ' + domain) + config = base.JavaConfig.JavaConfig(fn, self._logger) + homeWebApp = config.getString('directory') + opts = config.getString('excluded') + options = [] + if opts != None and opts != '': + sep = opts[0] + for opt in sep.split(opts[1:]): + options.append('--exclude=' + opt) + if not os.path.isdir(homeWebApp): + self.usage('home of {} not found: {}'.format(domain, homeWebApp)) + dbDir = self.ensureDirectory(homeWebApp + os.sep + 'db') + if dbDir == None: + self.error('save-webapp aborted: cannot create subdir db') + else: + fnDb = dbDir + os.sep + domain + '.sql.gz' + self.exportWebApp([domain, fnDb]) + self._logger.log('db saved to {}: {}'.format(fnDb, base.StringUtils.formatSize(os.path.getsize(fnDb))), 2) + name = os.path.basename(archive) + if name.endswith('.tgz'): + name = name[0:-4] + self.saveDirectoryByTar(name, homeWebApp, os.path.dirname(archive), None if len(options) == 0 else options) + + # show-tables <db> <user> <password> [<count-records>] + def showTables(self, db, user, passwd, countRecords): + '''Displays th tables of the given database. + @param db: the name of the db + @param user: a db user with access to db + @param passwd: the password of user + @param countRecords: True: the number of records is displayed too + ''' + argv = self.buildArgvMysql(db, user, passwd) + sql = 'show tables;'; + lines = self._processTool.executeInputOutput(argv, sql) + tables = lines[1:] + removeFromArrayIfExists(tables, '') + self._logger.log('= database {}:'.format(db)) + if not countRecords: + self._logger.log('\n'.join(tables)) + else: + DbToolResult = tables + for table in tables: + sql = 'select count(*) from {};'.format(table) + result = self._processTool.executeInputOutput(argv, sql) + if len(result) > 1: + count = result[1] + self._logger.log("{}: {}".format(table, count)) + + # show-webapps + def showWebApps(self): + '''Displays th tables of the given database. + @param domain: the domain of the webapp, e.g. 'wolke.infeos.eu' + ''' + path = self._configDir + '/webapps.d' + files = os.listdir(path) + DbToolResult = [] + for item in files: + if item.endswith('.conf'): + DbToolResult.append(item[:-5]) + self._logger.log(item[:-5]) + + # show-webapp-config <domain> + def showWebAppConfiguration(self, domain): + '''Displays th tables of the given database. + @param domain: the domain of the webapp, e.g. 'wolke.infeos.eu' + ''' + filename = self._configDir + '/webapps.d/' + domain + '.conf' + if not os.path.exists(filename): + self._logger.error('webapp configuration not found: ' + filename) + else: + content = base.StringUtils.fromFile(filename) + DbToolResult = content.split('\n') + self._logger.log(filename + ':\n' + content) + +def usage(): + '''Returns an info about usage. + ''' + return """dbtool [<global-opts>] <command> + Info and manipulation of (mysql) databases. +GLOBAL_OPTS +GLOBAL_MODES +<command>: + all-dbs [<admin> [<admin-passwd> [<internal-too>]]] + list all databases. + <internal-too>: 'true': internal databases like mysql are listed to + all-users [<admin> [<admin-passwd>]] + list all databases. + archive-webapp <domain> <dir-archive> [<admin> [<admin-passwd>]] + archives a webapp (directory, db, configuration) into a tar archive + create-admin [--read-only | -r] <admin> <admin-password> [<superuser> [<superuser-password>]] + creates an user which can access to all databases + create-and-import-webapp <domain> <dir-backup> [<superuser> [<superuser-password>]] + creates the DB and user, an imports the *.sql lying in the <dir-backup> + create-and-import-all-webapps <dir-backup> + creates the DBs and users for all web applications and imports the *.sql lying in the <dir-backup> + create-backup-user <user> <passwd> [<superuser> [<superuser-password>]] + creates an user which can access to all databases but readonly only + create-db-and-user <db> <user> <passwd> [<superuser> [<superuser-password>]] + creates a database (if not exists) and a user who can access to this db only + create-webapp-dbs [<superuser> [<superuser-password>]] + creates for all webapps db and a user (stored in the webapp configuration) + create-webapp-configuration <domain> <directory> <db> [ <user> [<password>]] + creates a configuration file in */webapps.d for a web application + delete-db-and-user <db> [<user> [<superuser> [<superuser-password>]]] [--no-saving] [--no-confirmation] + deletes the database <db> and (if given) a database user <user> connected as user <superuser> + --no-saving: the database is not saved before + --no-confirmation: no confirmation (retyping db/user name) is done + delete-user <user> [<superuser> [<superuser-password>]] + deletes the database user <user> connected as user <superuser> + export-all-dbs <directory> [<superuser> [<superuser-password>]] + exports all databases into a directory + export-db <db> <superuser> <superuser-password> <target> + exports a db into a sql file + export-webapp <domain> [<sqlfile>] + exports the database of the <domain>. user and password are taken from configuration + <sqlfile>: the output file. Default: /tmp/<domain>.sql + filter-records <source-sql> <table> <target-sql> [--exclude=<regexpr>] + reads an SQL file, searches for insert statements of the table <table> and writes a SQL file with this records + <source-sql>: name of the SQL file to read (formatted like from mysqldump) + <target-sql>: name of the result file + <regexpr>: a regular expression of the records which should not be copied + import-webapp <domain> <sql-file> + imports the sql-file (may be compressed) into the db of the webapp related to the domain + import-all-webapps <sql-directory> + import all webapps. <sql-directory> contains the sql files + save-webapp <domain> <tar-file> + store the database content in a subdirectory db and create a tar archive of the webapp directory + <tar-file>: a tar archive with the webapplication (files + db) + show-all-db-tables <user> <password> [<count-records>] + shows all databases with all tables. if <count-records> == true: the number of records is showed too + show-tables <db> [<superuser> [<superuser-password> [<count-records>]]] + shows the tables of the db. if <count-records> == true: the number of records is showed too + show-webapp-config <domain> + shows the configuration data of the web application for the <domain> + show-webapps + shows the domains of all webapps +example: + dbtool -v3 archive-webapp huber.de /media/backup/archive + dbtool -v3 show-tables wordpress wp_user TopSecret [<count-records>] + dbtool -v3 create-backup-user wordpress wp_user TopSecret root NotKnown + dbtool -v3 create-webapp-configuration example.com /home/example.com dbexample usrexample NeverUsed + dbtool -v3 create-and-import-all-webapps /backup/dayly/Mon + dbtool -v3 create-and-import-webapp www.example.com /backup/dayly/Mon dbadmin ExtremeSecret + dbtool -v3 delete-db-and-user testdb testusr --no-saving --no-confirmation + dbtool -v3 filter-records mysql.sql user users.sql --exclude=,root, +""" + +def defaultAdmin(argv, indexAdmin, tool): + '''Returns the default admin and its password. + If argv does not contains the data the configuration will be asked. + If nothing is found, ('root', '') is returned. + @param argv the argument vector + @param indexAdmin: the index of the admin in argv, the password has index+1 + @param tool: the DirTool instance (for configuration data) + @returns: a tuple (admin, password) + ''' + admin = tool._configuration.getString('admin.user') if len(argv) < indexAdmin + 1 or argv[indexAdmin] == '-' else argv[indexAdmin] + passwd = tool._configuration.getString('admin.code') if len(argv) < indexAdmin + 2 or argv[indexAdmin + 1] == '-' else argv[indexAdmin+1] + if admin == None: + admin = 'root' + if passwd == None: + passwd = '' + return (admin, passwd) + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/dbtool', 'run'] + ''' + appInfo = base.BaseTool.ApplicationInfo('dbtool', 'appl/DbTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = DbTool(options) + rc = None + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'archive-webapp': + tool.archiveWebapp(argv) + elif cmd == 'all-dbs': + # all-dbs <admin> <admin-passwd [<internal-too>] + (admin, code) = defaultAdmin(argv, 0, tool) + lines = tool.findAllDbs(admin, code, len(argv) > 4 and argv[4].upper().startswith('T')) + setResult(lines) + if tool._verboseLevel > 0: + print('\n'.join(lines)) + elif cmd == 'all-users': + (admin, code) = defaultAdmin(argv, 3, tool) + lines = tool.findAllUsers(admin, code) + setResult(lines) + if tool._verboseLevel > 0: + print('\n'.join(lines)) + elif cmd == 'create-db-and-user': + # create-db-and-user <db> <user> <passwd> <admin> <admin-pw> + if len(argv) < 3: + tool.usage('too few arguments') + else: + (admin, code) = defaultAdmin(argv, 3, tool) + tool.createDbAndUser(argv[0], argv[1], argv[2], admin, code) + elif cmd == 'create-admin': + # <user> <passwd> <user-adm> <passwd-adm> + argv, opts = tool.splitArgsAndOpts(argv) + readOnly = False + for opt in opts: + if opt == '-r' or opt == '--read-only': + readOnly = True + else: + usage('unknown option: ' + opt) + if len(argv) < 2: + tool.usage('too few arguments') + else: + (admin, code) = defaultAdmin(argv, 2, tool) + tool.createAdmin(argv[0], argv[1], readOnly, admin, code) + elif cmd == 'create-backup-user': + # <user> <passwd> <user-adm> <passwd-adm> + if len(argv) < 2: + tool.usage('too few arguments') + else: + (admin, code) = defaultAdmin(argv, 2, tool) + tool.createBackupUser(argv[0], argv[1], admin, code) + elif cmd == 'create-and-import-webapp': + # <domain> + if len(argv) < 2: + tool.usage('too few arguments') + else: + (admin, code) = defaultAdmin(argv, 2, tool) + tool.createAndImportWebApp(argv[0], argv[1], admin, code) + elif cmd == 'create-and-import-all-webapps': + # <domain> + if len(argv) < 1: + tool.usage('too few arguments') + else: + (admin, code) = defaultAdmin(argv, 1, tool) + tool.createAndImportAllWebApps(argv[0], admin, code) + elif cmd == 'create-webapp-dbs': + # create-webapp-dbs <admin> <admin-passwd> + if len(argv) < 1: + tool.usage('too few arguments') + else: + tool.createWebAppDbsAndUser(argv[0], '' if len(argv) < 2 else argv[1]) + elif cmd == 'create-webapp-configuration': + tool.createWebAppConfiguration(argv) + elif cmd == 'delete-db-and-user': + # <db> <user> <admin> <passwd-adm> + tool.deleteDbAndUser(argv) + elif cmd == 'delete-user': + # <user> <admin> <passwd-adm> + if len(argv) < 1: + tool.usage('too few arguments') + else: + (admin, code) = defaultAdmin(argv, 1, tool) + tool.deleteUser(argv[0], admin, code) + elif cmd == 'export-all-dbs': + if len(argv) < 1: + tool.usage('too few arguments') + else: + (admin, code) = defaultAdmin(argv, 1, tool) + tool.exportAllDatabases(admin, code, argv[0]) + elif cmd == 'export-db': + if len(argv) < 4: + tool.usage('too few arguments') + else: + tool.exportDatabase(argv[0], argv[1], '' if len(argv) <= 2 else argv[2], argv[3]) + elif cmd == 'export-webapp': + if len(argv) < 1: + tool.usage('too few arguments') + else: + tool.exportWebApp(argv) + elif cmd == 'filter-records': + if len(argv) < 3: + tool.usage('too few arguments') + else: + tool.filterRecords(argv) + elif cmd == 'import-webapp': + # import-webapp <domain> <sql-file> + if len(argv) < 2: + tool.usage('too few arguments') + else: + tool.importWebApp(argv[0], argv[1]) + elif cmd == 'import-all-webapps': + # import-all-webapps <sql-directory> + if len(argv) < 1: + tool.usage('too few arguments') + else: + tool.importAllWebappDbs(argv[0]) + elif cmd == 'save-webapp': + # save-webapp <domain> <tar-file> + if len(argv) < 2: + tool.usage('missing arguments') + lines = tool.saveWebApp(argv[0], argv[1]) + setResult(lines) + elif cmd == 'show-tables': + # show-tables <db> <user> <password> [<count-records>] + (user, code) = defaultAdmin(argv, 0, tool) + lines = tool.showTables(argv[0], user, code, len(argv) > 3 and argv[3].upper().startswith('T')) + setResult(lines) + elif cmd == 'show-all-db-tables': + # show-all-db-tables <user> <password> [<count-records>] + (user, code) = defaultAdmin(argv, 0, tool) + lines = tool.showAllDbsAndTables(argv[0], user, len(argv) > 2 and argv[3].upper().startswith('T')) + setResult(lines) + elif cmd == 'show-webapp-config': + # show-webapp-config <domain> + if len(argv) < 1: + usage('too few arguments') + else: + lines = tool.showWebAppConfiguration(argv[0]) + setResult(lines) + elif cmd == 'show-webapps': + # show-webapps + tool.showWebApps() + else: + tool.usage("unknown command: " + cmd) + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/DirTool.py b/appl/DirTool.py new file mode 100755 index 0000000..d707403 --- /dev/null +++ b/appl/DirTool.py @@ -0,0 +1,1206 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' + +import time +import sys +import os.path +import stat +import shutil +import datetime +import tarfile +import re + +sys.path.insert(0, '/usr/share/pyrshell') +import base.StringUtils +import base.BaseTool +import base.LinuxUtils +import base.FileHelper +import base.ProcessHelper + +class ProgramOptions: + '''Stores the common program options. + ''' + def __init__(self, parent): + '''Constructor. + @param parent: the DirTool instance + ''' + self._parent = parent + self._verboseLevel = parent._verboseLevel + self._logger = parent._logger + self._maxDepth = 999 + self._fileOnly = False + self._dirOnly = False + self._excluded = None + self._included = None + self._ignoreCase = False + self._regExprCaseMode = 0 + self._dirInfo = base.FileHelper.DirInfo() + self._blocksize = 1024*1024*16 + self._testDate = True + self._maxDifferenceCount = None + self._silent = False + self._short = False + self._infoCount = 5 + self._minSize = None + self._maxOldest = self._infoCount + self._maxYoungest = self._infoCount + self._maxSmallest = self._infoCount + self._maxLargest = self._infoCount + self._filesOnly = False + self._dirsOnly = False + self._humanReadable = True + self._orderDateSize = False + + def parseOptions(self, options): + '''Stores the given options. + @param options: the list of program options, e.g. ['--max-depth=3'] + ''' + for opt in options: + options = options[1:] + if opt.startswith('--pattern='): + self._includes = self._parent.regExprCompile(opt[10:], self._ignoreCase) + elif opt.startswith('--excluded='): + self._includes = self._parent.regExprCompile(opt[11:], self._ignoreCase) + elif opt.startswith('--max-depth='): + self._maxDepth = self._parent.integerOption(opt) + elif opt == '--file-only' or opt == '-f': + self._filesOnly = True + elif opt == '--dir-only' or opt == '-d': + self._dirsOnly = True + elif opt == '--i' or opt == '-ignore-case': + self._ignoreCase = True + elif opt.startswith('--blocksize'): + self._blocksize = self.integerOption(opt, 1024*1024*16) + elif opt == '-i' or opt == '--ignore-case': + self._ignoreCase = True + elif opt == '--ignore-time': + self._testDate = False + elif opt.startswith('--max-differences'): + self._maxDifferenceCount = self.integerOption(opt, 20) + elif opt == '-q' or opt == '--quiet': + self._silent = True + elif opt == '-s' or opt == '--short': + self._short = True + elif opt == '-t' or opt == '--test-content': + self._testContent = True + elif opt.startswith('-n') or opt.startswith('--count'): + self._infoCount = self.integerOption(opt) + elif opt.startswith('--min-size='): + self._minSize = self.integerOption(opt) + elif opt.startswith('--max-oldest='): + self._maxOldest = self.integerOption(opt) + elif opt.startswith('--max-youngest='): + self._maxYoungest = self.integerOption(opt) + elif opt.startswith('--max-smallest='): + self._maxSmallest = self.integerOption(opt) + elif opt.startswith('--max-largest='): + self._maxLargest = self.integerOption(opt) + elif opt == '--file-only' or opt == '-f': + self._filesOnly = True + elif opt == '--dir-only' or opt == '-d': + self._dirsOnly = True + elif opt == '--byte-size' or opt == '--mbyte-size': + self._humanReadable = False + elif opt == '--order-date-size': + self._orderDateSize = True + else: + self._parent.usage('unknown option ' + opt) + +class FileInfo: + def init(self): + '''Constructor. + ''' + self._name = None + self._size = None + self._modified = None + + def set(self, name, size, modified): + '''Sets the attributes. + @param name: the name without path + @param size: the size in bytes, -1 for directories + @param modified: the modify datetime + ''' + self._name = name + self._size = size + self._modified = modified + +class FileContainer: + '''Base class of file containers. + ''' + def __init__(self, options): + '''Constructor. + @param options: the program options + ''' + self._kind = None + self._options = options + self._currentFile = FileInfo() + self._action = 'list' + self._relPath = '' + self._base = '.' + self._dirInfo = base.FileHelper.DirInfo() + + def actionList(self): + '''Lists the metadata of a file/directory. + ''' + + if self._currentFile._size < 0: + size = '<dir>' + elif self._options._humanReadable: + size = "{:>8s}".format(base.StringUtils.formatSize(self._currentFile._size)) + else: + size = '{:13.6f} MB'.format(self._currentFile._size / 1000000) + fdate = datetime.datetime.fromtimestamp(self._currentFile._modified) + dateString = fdate.strftime("%Y.%m.%d %H:%M:%S") + if self._options._orderDateSize: + line = '{:s} {:>12s} {:s}{}{}{}'.format(dateString, size, self._base, os.sep, self._relPath, self._node) + else: + line = '{:>12s} {:s} {:s}{}{}{}'.format(size, dateString, self._base, os.sep, self._relPath, self._node) + if self._options._silent: + self._lines.append(line) + else: + print(line) + + def onSelected(self): + if self._currentFile._size >= 0: + self._dirInfo._fileCount += 1 + self._dirInfo._fileSizes += self._currentFile._size + if self._action == 'list': + self.actionList() + + def traverse(self, relPath, depth): + '''Traverses the container tree and does the wanted job. + @param relPath: the relative path + @param depth: the nesting level: 0: the first level + ''' + while self.nextNode(): + if not self.matches(): + if self._currentFile._size >= 0: + self._dirInfo._ignoredDirs += 1 + else: + self._dirInfo._ignoredFiles += 1 + else: + self.onSelected() + if self._isDir: + if depth >= self._options._maxDepth: + self._dirInfo._ignoredDirs += 1 + else: + self._dirInfo._dirCount += 1 + self._nextDir = relPath + os.sep + self._node + self.changeDir(self._nextDir) + self.traverse(self._nextDir, depth + 1) + +class TarContainer (FileContainer): + def __init__(self, archive, options): + FileContainer.__init__(self, options) + self._tar = tarfile.open(archive, 'r:*') + self._members = self._tar.getmembers() + self._currentMember = -1 + + def _finish(self): + '''Deconstructor. + ''' + self._tar.close() + + def changeDir(self, relPath): + '''Starts handling of the next directory. + @param relPath: the relative path from the base + ''' + pass + + def matches(self): + '''Tests whether the current file is not excluded by the option controlled filtering. + @return: True: the current file is not excluded + ''' + self._node = self._members[self._currentMember].name + self._isDir = self._members[self._currentMember].isdir() + rc = self._options._fileOnly and not self._options._isDir + rc = rc or self._options._dirOnly and self._options._isDir + if not rc: + rc = self._options._included == None or self._options._included.match(self._node, self._regExprCaseMode) + rc = rc or self._options._excluded == None or not self._options._excluded.match(self._node, self._options._regExprCaseMode) + self._currentFile.set(self._node, -1 if self._isDir else self._members[self._currentMember].size, self._members[self._currentMember].mtime) + return rc + + def nextNode(self): + '''Setups the next file in the container. + @return: False: no more file is available. + ''' + self._currentMember += 1 + return self._currentMember < len(self._members) + + +class DirectoryContainer (FileContainer): + def __init__(self, path, options): + FileContainer.__init__(self, options) + self._base = path + self._nodes = os.listdir(path) + self._currentNode = -1 + + def changeDir(self, relPath): + '''Starts handling of the next directory. + @param relPath: the relative path from the base + ''' + self._currentDir = self._base + relPath + self._nodes = os.listdir(self._currentDir) + + def matches(self): + '''Tests whether the current file is not excluded by the option controlled filtering. + @return: True: the current file is not excluded + ''' + self._node = self._nodes[self._currentNode] + self._full = self._base + os.sep + self._node + self._isDir = os.path.isdir(self._full) + rc = self._options._fileOnly and not self._isDir + rc = rc or self._options._dirOnly and self._isDir + if not rc: + rc = self._options._included == None or self._options._included.match(self._node, self._regExprCaseMode) + rc = rc or self._options._excluded == None or not self._options._excluded.match(self._node, self._regExprCaseMode) + if rc: + if not self._isDir: + self._dirInfo._fileCount += 1 + self._dirInfo._sizesFiles += os.path.getsize(self._full) + else: + if self._isDir: + self._ignoredDirs += 1 + else: + self._ignoredFiles += 1 + self._currentFile.set(self._node, -1 if self._isDir else os.path.getsize(self._full), os.path.getmtime(self._full)) + return rc + + def nextNode(self): + '''Setups the next file in the container. + @return: False: no more file is available. + ''' + self._currentNode += 1 + return self._currentNode < len(self._nodes) + +class DirTool (base.BaseTool.BaseTool): + + def __init__(self, globalOptions): + '''Constructor. + @param logger: the logger + ''' + self._globalOptions = globalOptions + base.BaseTool.BaseTool.__init__(self, globalOptions, 'dirtool.conf') + self._useRename = True + self._infos = [] + self._filenames = [] + self._maxDepth = 2048 + self._withDirs = True + self._withFiles = True + self._count = int(self._configuration.getString('count.default', '20')) + self._dateSize = True + self._humanReadableSize = True + self._euid = os.geteuid() + self._egid = os.getegid() + self._testDate = True + self._timePrecision = 2.0 + self._blocksize = 1024*1024*16 + self._differenceCount = 0 + self._maxDifferenceCount = 100 + self._differences = None + self._short = False + self._silent = False + self._testContent = False + self._dirInfo = None + + def buildExamples(self): + '''Prepares system to executes the examples of the usage message. + ''' + def setTime(fn, modified): + modified2 = datetime.datetime.strptime(modified, '%Y-%m-%d %H:%M:%S') + base.FileHelper.setModified(fn, None, modified2) + + def build(fn, content = 'line1', modified = '2018-01-03 07:03:53'): + self._logger.log('creating ' + fn, 1) + base.StringUtils.toFile(fn, content) + setTime(fn, modified) + + baseDir = '/tmp/%examples' + # self.ensureDirectory(baseDir) + baseDir1 = baseDir + os.sep + 'dir1' + self.ensureDirectory(baseDir1) + for no in range(5): + build(baseDir1 + os.sep + 'file{}.txt'.format(no + 1), + 'content of the file file{}.txt'.format(no) + '\nbla bla' * (no + 1), '2018-{:02d}-03 07:03:5{}'.format(no+1, no)) + no = 47 + for dirNo in range(1, 4): + no += 3 + subDir = baseDir1 + os.sep + 'subdir{}'.format(dirNo) + self.ensureDirectory(subDir) + for fileNo in range(4, 7): + no *= 7 + build(subDir + os.sep + 'data_{}.txt'.format(fileNo + 1), + 'content of the file file{}.txt'.format(fileNo) + '\nbla bla' * (no % 5 + 1), + '2017-{:02d}-{} 07:03:5{}'.format(fileNo+1, dirNo, dirNo)) + setTime(subDir, '2018-{:02d}-{:02d} 07:03:53'.format(no%12+1, dirNo + 7)) + + setTime(baseDir1, '2017-01-02 04:17:22') + baseDir2 = baseDir + os.sep + 'dir2' + shutil.rmtree(baseDir2, True) + shutil.copytree(baseDir1, baseDir2) + + os.unlink(baseDir + '/dir2/file2.txt') + os.unlink(baseDir + '/dir2/subdir1/data_6.txt') + + shutil.rmtree(baseDir + '/dir2/subdir2') + build(baseDir + '/dir2/subdir2', 'is a file instead of the name') + + build(baseDir + '/dir2/new.txt', 'line1\nline2\nline3') + build(baseDir + '/dir2/subdir1/new2.txt', 'line1\nline2\nline3') + + base.FileHelper.setModified(baseDir + '/dir2/file1.txt', None, datetime.datetime.strptime('2018-01-03 09:03:53', '%Y-%m-%d %H:%M:%S')) + base.FileHelper.setModified(baseDir + '/dir2/subdir1/data_5.txt', None, datetime.datetime.strptime('2018-01-03 09:03:53', '%Y-%m-%d %H:%M:%S')) + + build(baseDir + '/dir2/file4.txt', 'newer content in file4.txt', '2018-10-03 09:03:53') + build(baseDir + '/dir2/subdir3/data5.txt', 'line1\nline2', '2018-10-03 09:07:53') + + fn = baseDir + '/dir2/file5.txt' + content = base.StringUtils.fromFile(fn) + aTime = os.path.getmtime(fn) + build(fn, content.replace('e', 'E'), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(aTime))) + + fn = baseDir + '/dir2/subdir3/data_6.txt' + content = base.StringUtils.fromFile(fn) + aTime = os.path.getmtime(fn) + build(fn, content.replace('e', 'E'), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(aTime))) + + for dirNo in range(1, 4): + if dirNo != 2: + subDir = baseDir2 + os.sep + 'subdir{}'.format(dirNo) + setTime(subDir, '2018-{}-03 09:03:53'.format(1 + (dirNo + 7) % 12)) + setTime(baseDir2, '2018-04-03 09:03:53') + + + def compare(self, argv): + '''Compares two directory trees. + @param dir1: the first directory to compare + @param dir2: the second directory to compare + ''' + (argv, opts) = self.splitArgsAndOpts(argv) + if len(argv) < 2: + self.usage('missing arguments') + else: + dir1 = argv[0] + dir2 = argv[1] + argv = argv[2:] + if not os.path.isdir(dir1): + self.usage('dir1 is not a directory: ' + dir1) + elif not os.path.isdir(dir1): + self.usage('dir2 is not a directory: ' + dir2) + else: + self._blocksize = 1024*1024*16 + self._differenceCount = 0 + self._maxDifferenceCount = 100 + self._differences = [] + self._differenceCount = 0 + self._silent = False + self._short = False + self._testContent = False + self._testDate = True + caseSensitive = True + exclude = None + pattern = None + for opt in opts: + if opt.startswith('--blocksize'): + self._blocksize = self.integerOption(opt, 1024*1024*16) + elif opt.startswith('--pattern='): + pattern = opt[10:] + elif opt.startswith('--exclude='): + exclude = opt[10:] + elif opt == '-i' or opt == '--ignore-case': + caseSensitive = False + elif opt == '--ignore-time': + self._testDate = False + elif opt.startswith('--max-differences'): + self._maxDifferenceCount = self.integerOption(opt, 20) + elif opt == '-q' or opt == '--quiet': + self._silent = True + elif opt == '-s' or opt == '--short': + self._short = True + elif opt == '-t' or opt == '--test-content': + self._testContent = True + self._exclude = None if exclude == None else self.regExprCompile(exclude, 'compare-exclude', caseSensitive) + self._pattern = None if pattern == None else self.regExprCompile(pattern, 'compare-pattern', caseSensitive) + self._dirInfo = base.FileHelper.DirInfo() + self.compareDir(dir1, dir2) + base.BaseTool.setResult(self._differences) + if self._verboseLevel > 0: + info = self._dirInfo + tail = '' if info._fileSizes == 0 else ' with {}'.format( base.StringUtils.formatSize(info._fileSizes)) + self._logger.log('inspected {} dir(s) and {} file(s){}'.format( + info._dirCount, info._fileCount, tail)) + + def compareDir(self, dir1, dir2): + '''Compares all files two directories. + @param dir1: the first directory to compare + @param dir2: the 2nd directory to compare + @return: True: success False: stop processing + ''' + rc = False + self._dirInfo._dirCount += 1 + nodes1 = os.listdir(dir1) + nodes2 = os.listdir(dir2) + for node in nodes1: + full1 = dir1 + os.sep + node + isDir1 = os.path.isdir(full1) + if not isDir1 and self._pattern != None and self._pattern.match(node) == None: + self._logger.log('ignored (pattern): ' + node, 4) + self._dirInfo._ignoredFiles += 1 + continue + if self._exclude != None and self._exclude.match(node): + if isDir1: + self._dirInfo._ignoredDirs += 1 + else: + self._dirInfo._ignoredFiles += 1 + self._logger.log('ignored (exclude): ' + node, 4) + continue + if node not in nodes2: + self.compareLog('{} {}'.format('+' if self._short else 'additional:' , dir1 + os.sep + node)) + else: + full2 = dir2 + os.sep + node + type1 = base.FileHelper.fileType(full1) + type2 = base.FileHelper.fileType(full2) + if type1 != type2: + self.compareLog('{} {} / {} {}'.format('!' if self._short else 'different types:', type1, type2, full1)) + elif type1 == 'dir': + self.compareDir(full1, full2) + else: + self.compareFile(full1, full2) + rc = self._differenceCount < self._maxDifferenceCount + if not rc: + break + if rc: + for node in nodes2: + if self._pattern != None and self._pattern.match(node): + self._logger.log('ignored (pattern): ' + node, 4) + continue + if self._exclude != None and self._exclude.match(node): + self._logger.log('ignored (exclude): ' + node, 4) + continue + if node not in nodes1: + self.compareLog('{} {}'.format('-' if self._short else 'missing counterpart:' , dir1 + os.sep + node)) + rc = self._differenceCount < self._maxDifferenceCount + if not rc: + break + return rc + + def compareFile(self, file1, file2): + '''Compares two files. + @param file1: the first file to compare + @param file2: the 2ndfile to compare + @return: True: both files are equal + ''' + self._dirInfo._fileCount += 1 + info1 = os.lstat(file1) + info2 = os.lstat(file2) + rc = info1.st_size == info2.st_size + if not rc: + self.compareLog('{} {} / {} {}'.format('S' if self._short else 'size:', info1.st_size, info2.st_size, file1)) + elif self._testDate: + rc = abs(info1.st_mtime - info2.st_mtime) < self._timePrecision + if not rc: + self.compareLog('{} {} / {} {}'.format('T' if self._short else 'date:', + time.strftime('%Y.%m.%d %H:%M:%S', time.localtime(info1.st_mtime)), + time.strftime('%Y.%m.%d %H:%M:%S', time.localtime(info2.st_mtime)), + file1)) + if rc and self._testContent: + with open(file1, 'rb') as fp1, open(file2, 'rb') as fp2: + sumSize = 0 + while rc: + block1 = fp1.read(self._blocksize) + block2 = fp2.read(self._blocksize) + rc = len(block1) == len(block2) + if not rc: + self.compareLog('{} {} / {} {}'.format('L' if self._short else 'length:', sumSize + len(block1), sumSize + len(block2), file1)) + else: + rc = block1 == block2 + if not rc: + for ix in range(len(block1)): + if block1[ix] != block2[ix]: + desc = 'C [{}]' if self._short else 'different at position [{}]: ' + self.compareLog('{} {}'.format(desc.format(1 + sumSize + ix), file1)) + break + sumSize += len(block1) + if len(block1) == 0: + break + self._dirInfo._fileSizes += sumSize + return rc + + def compareLog(self, message): + '''Logs a difference of file tree comparism. + @param message: the difference as string + ''' + self._differenceCount += 1 + if self._differenceCount < self._maxDifferenceCount: + if self._differences != None: + self._differences.append(message) + if not self._silent: + self._logger.log(message) + + def traverseContainer(self, relPath, depth): + ''' + ''' + + def check(self, argv): + '''Searches for "wrong" files: + @param argv: command arguments, e.g. ['/home', '/etc' ] + ''' + def checkFile(name): + try: + with open(name, 'rb') as fp: + while fp.read(): + pass + except Exception as exc: + self._logger.error(name + ': ' + str(exc)) + def checkDir(full): + try: + for node in os.listdir(full): + if not os.path.isdir(full): + checkFile(full + os.sep + node) + except Exception as exc: + self._logger.error(full + ': ' + str(exc)) + # === def check + for arg in argv: + if not os.path.isdir(arg): + checkFile(arg) + else: + checkDir(arg) + + def dirInfo(self, argv): + '''Lists summary info and "extreme" files (e.g. the youngest files) of a directory tree + @param argv: the program arguments, e.g. ['/home', '--max-depth=7'] + ''' + argv, options = self.splitArgsAndOpts(argv) + filePattern = dirPattern = maxDepth = None + minSize = 1 + maxYoungest = maxLargest = maxOldest = count = maxSmallest = None + dirsOnly = filesOnly = False + trace = 0 + while len(options) > 0 and options[0].startswith('-'): + opt = options[0] + options = options[1:] + if opt.startswith('--pattern='): + dirPattern = opt[10:] + elif opt.startswith('--max-depth='): + maxDepth = self.integerOption(opt) + elif opt.startswith('-n') or opt.startswith('--count'): + count = self.integerOption(opt) + elif opt.startswith('--min-size='): + minSize = self.integerOption(opt) + elif opt.startswith('--max-oldest='): + maxOldest = self.integerOption(opt) + elif opt.startswith('--max-youngest='): + maxYoungest = self.integerOption(opt) + elif opt.startswith('--max-smallest='): + maxSmallest = self.integerOption(opt) + elif opt.startswith('--max-largest='): + maxLargest = self.integerOption(opt) + elif opt.startswith('--trace'): + trace = self.integerOption(opt) + elif opt == '--file-only' or opt == '-f': + filesOnly = True + elif opt == '--dir-only' or opt == '-d': + dirsOnly = True + if count != None: + if maxYoungest == None: + maxYoungest = count + if maxLargest == None: + maxLargest = count + if maxOldest == None: + maxOldest = count + if maxSmallest == None: + maxSmallest = count + if maxYoungest == None: + maxYoungest = 5 + if maxOldest == None: + maxOldest = 5 + if maxLargest == None: + maxLargest = 5 + if maxSmallest == None: + maxSmallest = 5 + info = None + if len(argv) == 0: + argv = ['.'] + for source in argv: + info = base.FileHelper.directoryInfo(source, filePattern, dirPattern, maxDepth, + info, maxYoungest, maxLargest, maxOldest, maxSmallest, minSize, dirsOnly, filesOnly, trace) + result = ['Directories: {} Files: {} / {}'.format(info._dirCount, info._fileCount, base.StringUtils.formatSize(info._fileSizes))] + result.append('Ignored: {} file(s) / {} dir(s)'.format(info._ignoredFiles, info._ignoredDirs)) + if info._maxSmallest != None and info._maxSmallest > 0: + result.append('The smallest files:') + for item in info._smallest: + full = item.split(':')[1] + statInfo = os.lstat(full) + name = full + if stat.S_ISLNK(statInfo.st_mode): + name += ' -> ' + os.readlink(full) + result.append(base.FileHelper.listFile(statInfo, name, True, True)) + if info._maxOldest != None and info._maxOldest > 0: + result.append('The oldest files:') + for item in info._oldest: + full = item.split(':')[1] + statInfo = os.lstat(full) + name = full + if stat.S_ISLNK(statInfo.st_mode): + name += ' -> ' + os.readlink(full) + result.append(base.FileHelper.listFile(statInfo, name, True, True)) + if info._maxLargest != None and info._maxLargest > 0: + result.append('The largest files:') + for item in info._largest: + full = item.split(':')[1] + statInfo = os.lstat(full) + if stat.S_ISLNK(statInfo.st_mode): + full = '-> ' + os.readlink(full) + result.append(base.FileHelper.listFile(statInfo, full, True, True)) + if info._maxYoungest != None and info._maxYoungest > 0: + result.append('The youngest files:') + for item in info._youngest: + full = item.split(':')[1] + statInfo = os.lstat(full) + name = full + if stat.S_ISLNK(statInfo.st_mode): + name += ' -> ' + os.readlink(full) + result.append(base.FileHelper.listFile(statInfo, name, True, True)) + base.BaseTool.setResult(result) + self._rawOutput = '\n'.join(result) + if self._verboseLevel > 0: + print(self._rawOutput) + return result + + def example(self): + example = '''# dirtool example configuration + log.file=/var/log/local/dirtool.log + ''' + self.storeExample(example) + + def findOptions(self, argv, isDate): + '''Evaluates the options relevant for date/size searchings. + @param argv: the program arguments to inspect + @param isDate: True: mode is oldest or youngest + @return: string: error message + array: the argument vector without the options. + ''' + rc = None + if len(argv) == 0: + return [os.sep] + else: + while len(argv) > 0: + arg = argv[0] + if not arg.startswith('-'): + break + if isDate and (arg == '-f' or arg == '--files-only'): + self._withDirs = False + self._withFiles = True + argv = argv[1:] + elif isDate and (arg == '-d' or arg == '--dirs-only'): + self._withDirs = True + self._withFiles = False + argv = argv[1:] + elif arg == '-b' or arg == '--byte-size': + self._humanReadableSize = False + argv = argv[1:] + else: + rc = 'unknown option: ' + arg + break + if rc == None: + rc = argv + return rc + + def handleOneFile(self, filename, statInfo, moreInteresting): + '''Compares a given files with the previous found files (self._files) and inserts if meaningful. + @param statInfo: the info of the current file + @param moreInteresting: a lambda function f(statFile1, statFile2, mayBeEqual): bool returning whether file1 replaces file2 + ''' + if len(self._filenames) == 0: + self._filenames.append(filename) + self._infos.append(statInfo) + else: + if len(self._filenames) < self._count: + # insert always: + last = self._infos[-1] + if moreInteresting(last, statInfo, True): + self._filenames.append(filename) + self._infos.append(statInfo) + else: + for ix in range(len(self._infos)): + if moreInteresting(statInfo, self._infos[ix], True): + self._filenames.insert(ix, filename) + self._infos.insert(ix, statInfo) + break + else: + # replace an entry + # more interesting than the last? + if moreInteresting(statInfo, self._infos[-1], False): + # more interesting than the first? + if moreInteresting(statInfo, self._infos[0], True): + self._filenames.insert(0, filename) + self._infos.insert(0, statInfo) + else: + for ix in range(len(self._infos)): + if moreInteresting(statInfo, self._infos[ix], False): + self._filenames.insert(ix, filename) + self._infos.insert(ix, statInfo) + break + # Remove the last entry: + del self._filenames[self._count] + del self._infos[self._count] + self.testOrder(moreInteresting) + + def handleTree(self, directory, depth, moreInteresting): + '''Finds recursively the n most interesting files in a directory tree. + @param directory: the directory to inspect + @param depth: the depth of the directory in the directory tree + @param moreInteresting: a lambda function f(statFile1, statFile2, mayBeEqual): bool returning whether file1 replaces file2 + ''' + self._logger.log(directory + os.sep + ' ...', 3) + if depth == 0: + try: + statInfo = os.lstat(directory) + accepted = (base.LinuxUtils.isExecutable(statInfo, self._euid, self._egid) + and base.LinuxUtils.isReadable(statInfo, self._euid, self._egid)) + if accepted: + self.handleOneFile(directory, statInfo, moreInteresting) + else: + self._logger.log('no permission: ' + directory, 2) + except FileNotFoundError: + self._logger.log('no permission: ' + directory, 2) + directory = None + dirs = [] + nodes = None + if directory != None: + try: + nodes = os.listdir(directory) + except PermissionError: + self._logger.log('no permission: ' + directory, 2) + directory = None + if directory != None: + for node in nodes: + if node != '.' and node != '..': + full = directory + os.sep + node + try: + statInfo = os.lstat(full) + except FileNotFoundError: + self._logger.log('no permission: ' + directory + os.sep + node, 2) + continue + isDir = stat.S_ISDIR(statInfo.st_mode) + if self._withDirs and isDir or self._withFiles and not isDir: + accepted = (base.LinuxUtils.isExecutable(statInfo, self._euid, self._egid) + and base.LinuxUtils.isReadable(statInfo, self._euid, self._egid)) + if accepted: + self.handleOneFile(full, statInfo, moreInteresting) + self._logger.log('no permission: ' + directory + os.sep + node, 2) + if isDir: + dirs.append(node) + if depth <= self._maxDepth: + for node in dirs: + self.handleTree(directory + os.sep + node, depth + 1, moreInteresting) + + def imageResize(self, argv): + '''resizes images. + @param argv: program arguments, e.g. ['src', 'trg', '--max-width=1024', '--max-height=768'] + ''' + argv, opts = self.splitArgsAndOpts(argv) + maxWidth = None + maxHeight = None + for opt in opts: + if opt.startswith('--max-width='): + maxWidth = self.integerOption(opt) + elif opt.startswith('--max-height='): + maxHeight = self.integerOption(opt) + else: + self.usage('unknown option: ' + opt) + if not os.path.exists('/usr/bin/identify') or not os.path.exists('/usr/bin/convert'): + self.usage('missing commands identify or convert: please install imagemagick') + elif len(argv) < 2: + self.usage('missing arguments') + else: + source = argv[0] + target = argv[1] + if not os.path.isdir(source): + self.usage('<source-dir> is not a directory: ' + source) + elif not os.path.isdir(target): + self.usage('<target-dir> is not a directory: ' + target) + else: + nodes = os.listdir(source) + regExtend = re.compile(r'\.(jpg|png|gif)$', re.I) + regDimension = re.compile(' (\d+)x(\d+) ') + for node in nodes: + src = source + os.sep + node + if regExtend.search(src) != None: + info = self._processHelper.executeInputOutput(['identify', src], None, False) + matcher = regDimension.search(info[0]) + if matcher != None: + width = int(matcher.group(1)) + height = int(matcher.group(2)) + newWidth = None + newHeight = None + if width > height: + # landscape + if maxWidth != None and width > maxWidth: + newWidth = maxWidth + newHeight = int(newWidth * height / width) + else: + # portrait + if maxHeight != None and height > maxHeight: + newHeight = maxHeight + newWidth = int(newHeight * width / height) + if newWidth != None: + trg = target + os.sep + node + start = time.time() + oldSize = os.path.getsize(src) + self._processHelper.execute(['convert', src, '-resize', '{}x{}'.format(newWidth, newHeight), trg], None, False) + base.FileHelper.setModified(trg, os.path.getmtime(src)) + newSize = os.path.getsize(trg) + duration = time.time() - start + self._logger.log('{} [{}x{}]: {:.1f}% ({:.3f} sec)'.format(node, width, height, newSize * 100 / oldSize, duration), 1) + + def jobLargest(self, argv): + '''Find the n youngest files in one or more directories. + @param argv: options and directories + @return: None or an error message + ''' + argv.append('--max-youngest=0') + argv.append('--max-smallest=0') + argv.append('--max-oldest=0') + rc = self.dirInfo(argv) + return rc + + def jobOldest(self, argv): + '''Find the n oldest files in one or more directories. + @param argv: options and directories + @return: None or an error message + ''' + argv.append('--max-largest=0') + argv.append('--max-smallest=0') + argv.append('--max-youngest=0') + rc = self.dirInfo(argv) + return rc + + def jobSmallest(self, argv): + '''Find the n smallest files in one or more directories. + @param argv: options and directories + @return: None or an error message + ''' + argv.append('--max-largest=0') + argv.append('--max-youngest=0') + argv.append('--max-oldest=0') + rc = self.dirInfo(argv) + return rc + + def jobYoungest(self, argv): + '''Find the n youngest files in one or more directories. + @param argv: options and directories + @return: None or an error message + ''' + argv.append('--max-largest=0') + argv.append('--max-smallest=0') + argv.append('--max-oldest=0') + rc = self.dirInfo(argv) + return rc + + def list(self, argv): + '''Lists a directory or a container (tar archive...) + @param argv: the program arguments, e.g. ['/home', '--max-depth=7'] + ''' + argv, options = self.splitArgsAndOpts(argv) + opts = ProgramOptions(self) + opts.parseOptions(options) + info = None + if len(argv) == 0: + argv = ['.'] + dirInfo = base.FileHelper.DirInfo() + for source in argv: + container = None + aClass, subClass = base.FileHelper.fileClass(source) + if aClass == 'container': + if subClass == 'tar' or subClass == 'tgz' or subClass == 'tbz': + container = TarContainer(source, opts) + elif subClass == 'dir': + container = DirectoryContainer(source, opts) + else: + self._logger.error('unknown archive type: {} / {}', subClass, source) + else: + self._logger.error('not a directory or an archive: ' + source) + if container != None: + container._dirInfo = dirInfo + container.traverse('', 0) + result = ['Directories: {} Files: {} / {}'.format(dirInfo._dirCount, dirInfo._fileCount, base.StringUtils.formatSize(dirInfo._fileSizes))] + result.append('Ignored: {} file(s) / {} dir(s)'.format(dirInfo._ignoredFiles, dirInfo._ignoredDirs)) + base.BaseTool.setResult(result) + self._rawOutput = '\n'.join(result) + if self._verboseLevel > 0: + print(self._rawOutput) + return result + + def listFiles(self): + '''Print the file list of the found files. + ''' + for ix in range(len(self._filenames)): + # statInfo, full, orderDateSize = True, humanReadable = True + base.FileHelper.listFile(self._infos[ix], self._filenames[ix], self._dateSize, self._humanReadableSize) + + def snapshot(self, argv): + '''Duplicates a source directory tree into a target with hard links: needs space only for directory info (meta data). + @precondition: source and target must lay in a common filesystem to allow making hardlinks. + @param argv: the program arguments, e.g. ['/media/server/data', '/media/server/backup/Monday'] + ''' + if len(argv) < 2: + self.usage('too few arguments') + else: + source = argv[0] + target = argv[1] + if not os.path.isdir(source): + self.usage('source is not a directory: ' + source) + elif not os.path.isdir(os.path.dirname(target)): + self.usage('parent of target is not a directory: ' + target) + elif os.path.isdir(target): + self.usage('target already exists: ' + target) + else: + process = base.ProcessHelper.ProcessHelper(self._verboseLevel, self._logger) + process.execute(['/bin/cp', '-al', source, target], True, False) + + def synchronize(self, argv): + '''Copies all changed/new files from a base directory to a "mirror". + But no file of the mirror will be deleted: they will be moved to a "safe". + @param argv: the program arguments, e.g. ['/media/data', '/media/backup', '/media/safe'] + ''' + if len(argv) < 3: + self.usage('too few arguments') + elif (not os.path.isdir(argv[0])): + self.usage('base is not a directory: ' + argv[0]) + elif (not os.path.isdir(argv[1])): + self.usage('mirror is not a directory: ' + argv[1]) + elif (not os.path.isdir(argv[2])): + self.usage('base is not a directory: ' + argv[2]) + else: + try: + self.synchronizeOneDir(argv[0], argv[1], argv[2], '') + except Exception as exc: + self._logger.error('the mirroring has failed: ' + str(exc)) + raise exc + + def synchronizeOneDir(self, baseDir, mirrorDir, safeDir, relPath): + '''Copies all changed/new files from a baseDir directory to a "mirrorDir". + But no file of the mirrorDir will be deleted: they will be moved to a "safeDir". + @param baseDir: the source directory + @param mirrorDir: the target directory + @param safeDir: the directory for "deleted" files from the target + @param relPath: defines the effective paths to process: will be added to baseDir, mirrorDir and safeDir + example: source/py + ''' + baseFull = baseDir + mirrorFull = mirrorDir + safeFull = safeDir + if relPath != '': + baseFull += os.sep + relPath + mirrorFull += os.sep + relPath + safeFull += os.sep + relPath + self._logger.log('processing ' + baseFull, 2) + baseNodes = os.listdir(baseFull) + mirrorNodes = os.listdir(mirrorFull) + # move deleted files from mirrorDir to safeDir: + for node in mirrorNodes: + if node not in baseNodes: + self.synchronizeMove(mirrorDir, safeDir, relPath, node) + # copy new/changed files from baseDir to mirrorDir + for node in baseNodes: + source = baseFull + os.sep + node + target = mirrorFull + os.sep + node + if not os.path.exists(target): + if os.path.isdir(source): + if not base.FileHelper.distinctPaths(source, target): + self._logger.error('nested directories: {} / {} [{} / {}]'.format( + source, target, os.path.realpath(source), os.path.realpath(target))) + else: + shutil.copytree(source, target) + else: + self._logger.log('{} -> {}'.format(source, target), 4) + shutil.copy2(source, target) + else: + if base.FileHelper.fileType(source) != base.FileHelper.fileType(target): + self.synchronizeMove(mirrorDir, safeDir, relPath, node) + elif not os.path.isdir(source): + infoSource = os.lstat(source) + infoTarget = os.lstat(target) + if abs(infoSource.st_mtime - infoTarget.st_mtime) > 2: + self.synchronizeMove(mirrorDir, safeDir, relPath, node) + self._logger.log('{} => {}'.format(source, target), 4) + shutil.copy2(source, target) + infoTarget = os.lstat(target) + if infoSource.st_mtime != infoTarget.st_mtime or infoSource.st_size != infoTarget.st_size: + self._logger.error('copy failed: {}/{}: {}/{} {}/{}'.format( + relPath, node, infoSource.st_mtime, infoTarget.st_mtime, + infoSource.st_size, infoTarget.st_size)) + else: + prefix = relPath + os.sep if relPath != '' else '' + self.synchronizeOneDir(baseDir, mirrorDir, safeDir, prefix + node) + + def synchronizeMove(self, mirror, safe, relPath, node): + '''Moves a file/directory from mirror to safe: + @param mirror: the source directory + @param safe: the target directory + @param relPath: defines the effective paths (in mirror and safe) + @param node: the node (filename without path) of the source and target file + ''' + relPath2 = os.sep + relPath if relPath != '' else '' + source = mirror + relPath2 + os.sep + node + dirTarget = safe + relPath2 + if not os.path.exists(dirTarget): + os.makedirs(dirTarget, 0o777) + if not os.path.exists(dirTarget): + self._logger.error('cannot create: ' + dirTarget) + else: + target = dirTarget + os.sep + node + self._logger.log('moving {} => {}'.format(source, target), 3) + if os.path.exists(target): + self._logger.error('target exists in safe: {} => {}'.format(source, target)) + os.rename(target, target + '.' + str(time.time())) + if self._useRename: + os.rename(source, target) + elif os.path.isdir(source): + shutil.copytree(source, target) + else: + shutil.copy2(source, target) + + def testOrder(self, moreInteresting): + for ix in range(len(self._infos) - 1): + if not moreInteresting(self._infos[ix], self._infos[ix+1], True): + print(self._filenames[ix] + '\n' + self._filenames[ix + 1] + '\n') + +def usage(): + return """usage: dirtool [<global_opts>] <mode> <args> + Offers some services in directory trees. + Note: a container is a directory or an archive (tar, zip). +GLOBAL_OPTS +GLOBAL_MODES +<mode>: + build-examples + prepares the system for executing the examples below + check <dir1> [<dir2...] + reads all files of the source directories (and its subdirectories) to find access problems + compare <dir1> <dir2> <opts> + compares two directory trees + <opt>: + --max-differences=<count> + the search is stopped if <count> differences has been found + --blocksize=<size> + files are read in this chunk size. Default: 16 MiByte + --exclude=<regular-expr> + files and dirs matching this pattern are ignored (not compared) + -i or --ignore-case + used for --pattern and/or --exclude + --ignore-time + only different size or content will be counted as difference + --pattern=<regular-expr> + only files and dirs matching this pattern are compared + -q or --quiet + only the summary is displayed + -s or opt == '--short' + the prefix of a difference notice is reduced to one char + -t or --test-content + the file content is inspected for differences + image-resize <source-dir> <target-dir> [--max-width=<width>] [--max-height=<height>] + resizes *.jpg or *.png images + info <path1> [ <path2> ... ] <opts> + displays a summary info about the given directories + largest <path1> [ <path2> ... ] + list the n youngest files in the directory tree + list <container> [<container2> ...] <opt> + show the metadata of the files of the container + <opt>: + --order-date-size + the displayed data: <date> <size> <name> instead of <size> <date> <name> + --byte-size or --mbyte-sizes + the size is displayed in MBytes (instead of "human readable" with different units + oldest <path1> [ <path2> ... ] + list the n youngest files in the directory tree(s) + smallest <path1> [ <path2> ... ] [--min-size=<min-size>] + list the n smallest files (but >= min-size) in the directory tree + sync <base> <mirror> <safe> + copy modified/new files from <base> to mirror. + no file from mirror will be deleted/replaced: instead it will be moved to safe + youngest [<opt_date>] <path1> [ <path2> ... ] + list the n youngest files in the directory tree(s) + <opts>: + -n=<count> or --count=<count> + number of displayed entries (replaces max-largest or max-youngest or max-oldest or max-smallest) + -d or --dir-only + the counters respects directory only + -f or --file-only + only files land in the youngest array + --max-depth=<depth> + the maximal nesting depth of directories. 0: only the called directory is inspected + --max-largest=<count> + the maximal length of the array for the largest files + --max-oldest=<count> + the maximal length of the array for the oldest files + --max-smallest=<count> + the maximal length of the array for the smallest files + --max-youngest=<count> + the maximal length of the array for the youngest files + --pattern=<wildcard-expr> + filename pattern, e.g. "*.png" +Example: +dirtool check /usr/share/pyrshell/examples/dir1 /usr/share/pyrshell/examples/dir2 +dirtool compare /usr/share/pyrshell/examples/dir1 /usr/share/pyrshell/examples/dir2 +dirtool compare /usr/share/pyrshell/examples/dir1 /usr/share/pyrshell/examples/dir2 --ignore-time --short --test-content +dirtool compare /usr/share/pyrshell/examples/dir1 /usr/share/pyrshell/examples/dir2 -s --exlude=sub.*2 "--pattern=[^u]*[1-3]" --ignore-case +dirtool info /usr/share/pyrshell/examples/dir1 --max-largest=2 --max-youngest=3 --file-only +dirtool info self._dir1 --count=0 --max-youngest=3 --max-oldest=4 --dir-only +dirtool -v4 largest /usr/share/pyrshell/examples/dir1 '--pattern=.*1.*' +dirtool -v4 youngest /usr/share/pyrshell/examples/dir1 --max-depth=0 +dirtool -v4 oldest /usr/share/pyrshell/examples/dir1 --file-only +dirtool latest /home/jonny -n5 +dirtool -v3 list /usr/share/pyrshell/unittest/data/example.tgz --dir-only +dirtool -v3 list /usr/share/pyrshell/unittest/data --order-date-size --file-only -mbytes-size +dirtool -v3 image-convert /pic /out --max-width=1920 --max-height=1080 +""" + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/dirtool', 'run'] + ''' + appInfo = base.BaseTool.ApplicationInfo('dirtool', 'appl/DirTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = DirTool(options) + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'build-examples': + tool.buildExamples() + elif cmd == 'check': + tool.check(argv) + elif cmd == 'compare': + tool.compare(argv) + elif cmd == 'info': + tool.dirInfo(argv) + elif cmd == 'largest': + tool.jobLargest(argv) + elif cmd == 'list': + tool.list(argv) + elif cmd == 'oldest': + tool.jobOldest(argv) + elif cmd == 'smallest': + tool.jobSmallest(argv) + elif cmd == 'snapshot': + tool.snapshot(argv) + elif cmd == 'sync': + tool.synchronize(argv) + elif cmd == 'youngest': + tool.jobYoungest(argv) + elif cmd == 'image-resize': + tool.imageResize(argv) + else: + tool.usage("unknown command: " + cmd) + base.BaseTool.setLatestTool(tool) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/FtpTool.py b/appl/FtpTool.py new file mode 100755 index 0000000..e4233bd --- /dev/null +++ b/appl/FtpTool.py @@ -0,0 +1,176 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' + +import time +import sys +import os.path +sys.path.insert(0, '/usr/share/pyrshell') +import base.BaseTool +import base.JavaConfig +import net.FtpEngine + +class FtpTool (base.BaseTool.BaseTool): + '''Implements a command interpreter for FTP. + ''' + def __init__(self, globalOptions): + '''Constructor. + @param configurationBase: directory containing the configuration file ftptool.conf + @param logger: None: use the logger given by the configuration file + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'ftptool.conf') + self._client = None + + def example(self): + '''Builds an example configuration. + @param options: an instance of GlobalOptions + ''' + exampleConfig = self._configDir + os.sep + 'ftptool.conf.example' + base.StringUtils.toFile(exampleConfig, 'log.file=/tmp/ftptool.log') + print('created: ' + exampleConfig) + subdir = self._configDir + os.sep + 'ftp.d' + base.BaseTool.BasicStatics.ensureDirectory(subdir) + exampleConfig = subdir + os.sep + 'example.conf' + base.StringUtils.toFile(exampleConfig, 'host=hamatoma.de\nport=21\nuser=jonny\ncode=Secret\n') + print('created: ' + exampleConfig) + + def setServer(self, name): + full = self._configDir + os.sep + 'ftp.d' + os.sep + name + '.conf' + self._client = None + if not os.path.exists(full): + self._logger.error('server {:s} is not configured: Please create {}'.format(name, full)) + else: + self._server = base.JavaConfig.JavaConfig(full, self._logger) + host = self._server.getString('host') + port = self._server.getString('port') + user = self._server.getString('user') + pw = self._server.getString('code') + if host == None or port == None or user == None or pw == None: + self._logger.error('missing FTP authority data for server {:s}. See {:s}'.format(name, full)) + else: + try: + port2 = int(port) + self._client = net.FtpEngine.FtpEngine(host, port2, user, pw, self._logger, self._verboseLevel) + except ValueError: + self._logger.error('port must be an integer: ' + port) + except Exception as exc: + self._logger.error('connection failed: {:s}:{:d} [{:s}]'.format(host, port2, str(exc))) + + def close(self): + '''Frees the resources. + ''' + if self._client != None: + self._client.close() + +def usage(): + '''Returns an info about usage + ''' + return """usage: ftptool <globalOpts> <task> [<server> [<arg1>...]]" +GLOBAL_OPTS +GLOBAL_MODES +<server>: + the name of a server which is defined in %etc%/ftp.d/<server>.conf +<task>: + du (or diskusage) <server> [<startDir> [<depth>]] + calculate file size (over all files and directories) + <startDir>: start directory, default: '/' + <depth>: only directories lower this depth will be displayed. default: 0 + info + print welcome message and features + compare <server> <ftpStart> <localStart> + compare a local file tree with an ftp file tree + <ftpStart>: ftp start directory + <localStart>: local start directory + lstree <server> <startDir1> [<startDir2>...] + lists the directory infos of <startDirX> + rmtree <server> [<startDir1> [<startDir2>...]] + : deletes one or more directories recursive + <startDirN>: directories to remove +example: +ftptool du jonny / 3 +ftptool info contabo +ftptool rmtree contabo /storage /jonny/trash +""" + +def buildClient(configurationBase): + config = configurationBase + os.sep + 'ftptool.conf' + +def addServer(options, argv): + '''Adds a server configuration to the configuration directory. + @param options: an instance of GlobalOptions + @param argv: the arguments, [<servername>, <host>, <user>, <password>[, <port>]] + ''' + if len(argv) < 4: + usage('missing argument(s): expected: <servername> <host> <user> <password> [<port>]') + else: + configFile = options._configDir + os.sep + 'ftp.d' + os.sep + argv[0] + try: + port = 21 if len(argv) < 4 else argv[4] + base.StringUtils.toFile(configFile, 'host={:s}\nport={}\nuser={}\ncode={}\n'.format(argv[1], port, argv[2], argv[3])) + print('created: ' + configFile) + except ValueError: + print('+++ port must be an integer, e.g. 21, not ' + argv[4]) + +def main(argv): + appInfo = base.BaseTool.ApplicationInfo('ftptool', 'appl/FtpTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = FtpTool(options) + if len(argv) > 1: + tool.setServer(argv[1]) + (cmd, argv) = tool.handleStandardCommands(argv) + if tool._client == None: + pass + elif cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd in ['du', 'diskusage']: + startDir = '/' if len(argv) <= 0 else argv[0] + tool._client._printDepth = -1 if len(argv) <= 1 else int(argv[1]) + (size, files, dirs) = tool._client.diskUsage(startDir, 0) + unit = 'm' if size < 1000000000 else 'g' + size2 = size / 1000000000.0 if unit == 'g' else size / 1000000.0 + digits = '9' if unit == 'g' else '6' + tool._logger.log(startDir + (": {:d} bytes [{:." + digits + "f} {:s}b] files: {:d} dirs: {:d}").format(size, size2, unit, files, dirs)) + elif cmd in ['info']: + tool._client.serverInfo() + elif cmd in ['compare']: + if len(argv) < 2: + tool.usage('to few arguments') + elif not os.path.isdir(argv[1]): + tool.usage('not a directory: ' + argv[1]) + else: + (size, files, dirs) = tool._client._client.compare(argv[0], argv[1], 0) + unit = 'm' if size < 1000000000 else 'g' + size2 = size / 1000000000.0 if unit == 'g' else size / 1000000.0 + digits = '9' if unit == 'g' else '6' + print(server + (": {:d} bytes [{:." + digits + "f} {:s}b] files: {:d} dirs: {:d}").format(size, size2, unit, files, dirs)) + elif cmd in ['rmtree']: + if len(argv) < 1: + tool.usage('too few arguments: missing directory') + else: + for directory in argv: + tool.tool._client.removeTree(directory) + elif cmd in ['lstree']: + if len(argv) < 1: + tool.usage('too few arguments: missing directory') + else: + if len(argv) < 2: + depth = 9999 + else: + try: + depth = int(argv[1]) + except ValueError: + tool._logger.error('depth must be an integer: ' + argv[1]) + depth = -1 + if depth >= 0: + tool._client._maxDepth = depth + tool._client.listTree(argv[0], 0) + tool.close() + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/Monitor.py b/appl/Monitor.py new file mode 100755 index 0000000..892ed51 --- /dev/null +++ b/appl/Monitor.py @@ -0,0 +1,1184 @@ +#! /usr/bin/python3 +''' +Created on 27.04.2018 + +@author: hm +''' +import sys +import os.path +import re +import time +import datetime +import traceback +import pwd +sys.path.insert(0, '/usr/share/pyrshell') +import base.BaseTool +import base.LinuxUtils +import net.EMail +import net.HttpClient +import urllib + +IGNORE_CASE = re.I +class Property: + + def __init__(self, name, value=None): + '''Constructor. + @param name: the property name + @param value: the value + ''' + self._id = name + self._value = value + + def copy(self, source): + '''Copies the properties from a template + @param source: the source, type: Property + ''' + if source._value != None: + self._value = source._value + + def dump(self, indent): + '''Dumps a property. + @param indent: indent level + @return: a string describing the property + ''' + if self._id == None: + self._id = None + if self._value == None: + value = '<None>' + elif type(self._value) == float: + value = '{:.3f'.format(self._value) + elif type(self._value) == int: + value = str(self._value) + else: + value = self._value + return ' ' * indent + self._id + ': ' + value + "\n" + +class ObservedProperty: + + def __init__(self, name, warnLimit, errorLimit, receivers): + '''Constructor. + @param name: the name + @param warnLimit: a higher value raises a warning + @param errorLimit: a higher value raises an error + ''' + self._id = name + self._warnLimit = warnLimit + self._errorLimit = errorLimit + self._receivers = receivers + + def copy(self, source): + '''Copies the properties from a template + @param source: the source, type: Property + ''' + if source._warnLimit != None: + self._warnLimit = source._warnLimit + if source._errorLimit != None: + self._errorLimit = source._errorLimit + if source._receivers != None: + self._receivers = source._receivers + + def dump(self, indent): + '''Dumps a observed property. + @param indent: indent level + @return: a string describing the property + ''' + rc = ' ' * indent + self._id + ': ' + str(self._warnLimit) + ' ' + str(self._errorLimit) + ' "' + self._receivers._name + '"\n' + return rc + +class Container: + + def __init__(self, aType, name, parent, keyword = None): + '''Constructor. + @param aType: the container type: 'host', 'disk' + @param name: the name of the container. '': the default container + @param parent: type Monitor or another container + ''' + self._type = aType + self._name = name + self._properties = dict() + self._observed = dict() + self._parent = parent + self._keyword = keyword if keyword != None else aType.lower() + + def copy(self, source): + '''Copies the properties from a source + @param source: the source, type: Property + ''' + for prop in self._observed: + if prop in source._observed: + self._observed[prop].copy(source._observed[prop]) + + def dump(self, indent,): + '''Dumps an container. + @param indent: indent level + @return: a string describing the container + ''' + rc = ' ' * indent + self._type + ' "' + self._name + '":\n' + for key in sorted(self._properties): + rc += self._properties[key].dump(indent + 3) + for key in sorted(self._observed): + rc += self._observed[key].dump(indent + 3) + return rc + +class WebSite(Container): + def __init__(self, name, monitor): + '''Constructor. + @param name: name of the group + @param monitor: the parent with type Monitor + ''' + Container.__init__(self, 'WebSite', name, monitor, 'site') + self._properties = { + 'url': Property('url', ''), + } + +class ReceiverGroup(Container): + + def __init__(self, name, monitor): + '''Constructor. + @param name: name of the group + @param monitor: the parent with type Monitor + ''' + Container.__init__(self, 'ReceiverGroup', name, monitor) + self._properties = { + 'warning': Property('warning', ''), + 'error': Property('error', '') + } + +class Disk(Container): + + def __init__(self, name, host): + '''Constructor. + @param name: name of the host, if empty default host is assumed + @param host: the parent with type Host + ''' + Container.__init__(self, 'Disk', name, host) + if '' not in host._parent._hosts: + defaultDisk = self + receiverGroup = ReceiverGroup('', host._parent) + else: + if name in host._parent._hosts['']._disks: + defaultDisk = host._parent._hosts['']._disks[name] + else: + defaultDisk = host._parent._hosts['']._disks[''] + receiverGroup = defaultDisk._properties['receivers'] + self._observed = { + 'used': ObservedProperty('used', '80%', '90%', receiverGroup), + } + self._properties = { + 'receivers': receiverGroup + } + self.copy(defaultDisk) + +class RaidDevice(Container): + def __init__(self, name, host): + '''Constructor. + @param name: the device name, e.g. 'md0' + @param host: the parent, type: Host + ''' + Container.__init__(self, 'RaidDevice', name, host, 'raid') + receiverGroup = host._properties['receivers'] + self._properties = { + 'receivers': receiverGroup, + 'raidtype': Property('raidtype'), + 'members': Property('members'), + 'blocks': Property('blocks') + } +class Host(Container): + + def __init__(self, name, monitor): + '''Constructor. + @param name: name of the host, if empty default host is assumed + @param monitor: the parent with type Monitor + ''' + Container.__init__(self, 'Host', name, monitor) + receiverGroup = monitor._receiverGroups[''] + if name == '': + disk = Disk('', self) + else: + disk = monitor._hosts['']._disks[''] + self._disks = { + '': disk + } + self._raids = dict() + self._properties = { + 'address': Property('address'), + 'receivers' : receiverGroup, + 'interval' : Property('interval', 60) + } + self._observed = { + 'load1': ObservedProperty('load1', 10, 20, receiverGroup), + 'load5': ObservedProperty('load5', 10, 20, receiverGroup), + 'load10': ObservedProperty('load10', 10, 20, receiverGroup), + 'processes': ObservedProperty('processes', 500, 1000, receiverGroup), + 'swap': ObservedProperty('swap', 100.01, 100.01, receiverGroup), + } + if name != '' and '' in monitor._hosts: + self.copy(monitor._hosts['']) + + def copy(self, source): + '''Copies the properties from a source + @param source: the source, type: Disk + ''' + Container.copy(self, source) + for name in self._disks: + if name in source._disks: + self._disks[name].copy(source._disks[name]) + + def dump(self, indent): + '''Dumps a host. + @param indent: indent level + @return: a string describing the receiver group + ''' + rc = "===\n" + Container.dump(self, 0) + for disk in sorted(self._disks): + rc += self._disks[disk].dump(indent + 3) + return rc + +class SmtpHost (Container): + + def __init__(self, name, monitor): + '''Constructor. + @param name: the name of the smtp host + @param monitor: the parent, type: Monitor + ''' + Container.__init__(self, 'SmtpHost', name, monitor) + self._properties = { + 'host': Property('host'), + 'port' : Property('port', '587'), + 'sender' : Property('sender'), + 'user' : Property('user'), + 'code' : Property('code'), + 'tls' : Property('tls', 'True') + } + +class Monitor (base.BaseTool.BaseTool): + '''Implements a monitor for hosts to detect service faults. + ''' + + def __init__(self, globalOptions, additionalConfigDir = None): + '''Constructor. + @param globalOptions: a instance of GlobalOptions + ''' + config = '/etc/pyrshell/monitor.d' if additionalConfigDir == None else additionalConfigDir + base.BaseTool.BasicStatics.ensureDirectory(config) + base.BaseTool.BaseTool.__init__(self, globalOptions, 'pymonitor.conf', False, config) + self._start = time.time() + self._loops = 0 + self._openProblems = dict() + self._openProblems = [] + # self._configDir = dirConfiguration + self._reloadRequestFile = '/tmp/{}.reload.request'.format(globalOptions._appInfo._applicationName) + self._readAllConfigs() + self._lastStatus = dict() + + def _readAllConfigs(self): + self._sites = dict() + self._hosts = dict() + self._smtpHosts = dict() + self._receiverGroups = dict() + self._receiverGroups[''] = ReceiverGroup('', self) + defaultConf = self._additionalConfigDir + os.sep + 'default.conf' + if os.path.exists(defaultConf): + self.readConfigurationFile(defaultConf) + else: + self._hosts[''] = Host('', self) + nodes = os.listdir(self._additionalConfigDir) + for node in nodes: + if node.endswith('.conf') and node != 'default.conf' and node != 'pymonitor.conf': + self.readConfigurationFile(self._additionalConfigDir + os.sep + node) + + def checkReload(self): + '''Tests whether a reload request exists. + If yes the configuration will be read again. + ''' + fn = self._configDir + '/reload.request' + if os.path.exists(self._reloadRequestFile): + self.reload() + os.unlink (self._reloadRequestFile) + if os.path.exists(self._reloadRequestFile): + self._logger.error('cannot delete ' + fn) + + def createSiteServer(self, argv): + '''Creates the configuration for an observed site on the server side. + @param argv: the program arguments, e.g. ['--nginx'] + ''' + webserver = 'nginx' + argv, options = self.splitArgsAndOpts(argv) + ip = None + domain = None + overwrite = False + for opt in options: + if opt == '--nginx': + webserver = 'nginx' + elif opt == '--apache': + webserver = 'apache' + elif opt == '--overwrite': + overwrite = True + elif opt.startswith('--ip='): + ip = opt[5:] + elif opt.startswith('--domain='): + domain = opt[9:] + else: + self.usage('unknown option: ' + opt) + if domain == None: + domain = base.BaseTool.BasicStatics.hostname(True) + if ip == None: + ip = self.publicIp() + if ip == None: + self.usage('cannot detect public ip. Please use the --ip=<ip> option') + baseDir = self.getTarget('/var', 'www' + os.sep + domain) + self.ensureDirectory(baseDir) + fn = baseDir + os.sep + 'index.html' + base.StringUtils.toFile(fn, '<html><body><p>Ups. Verirrt?</p></body></html>\n') + fn = baseDir + os.sep + 'index.php' + base.StringUtils.toFile(fn, '<?php\necho "<html><body><p>Ups. Verirrt?</p></body></html>";\n') + fn = baseDir + os.sep + 'domain.txt' + base.StringUtils.toFile(fn, domain + '\n') + if webserver == None: + if os.path.isdir('/etc/nginx'): + webserver = 'nginx' + elif os.path.isdir('/etc/apache2'): + webserver = 'apache' + if webserver == 'nginx': + available = self.getTarget('/etc/nginx', 'sites-available') + enabled = os.path.dirname(available) + os.sep + 'sites-enabled' + fn = available + os.sep + domain + if os.path.exists(fn) and not overwrite: + self.usage('{} exists. Use --overwrite to overwrite'.format(fn)) + base.StringUtils.toFile(fn, '''server {} + listen 80; + server_name {} {}; + root {}; + location / {} + allow all; + {} +{} +'''.format('{', domain, ip, baseDir, '{', '}', '}')) + linkTarget = enabled + os.sep + domain + if os.path.islink(linkTarget): + self._logger.log('deleting ' + linkTarget, 2) + os.unlink(linkTarget) + self._logger.log('creating symlink ' + linkTarget, 2) + os.symlink('../sites-available/' + domain, linkTarget) + if self._isRoot: + self._processHelper.execute(['/bin/systemctl', 'reload', 'nginx'], True) + elif webserver == 'apache': + available = self.getTarget('/etc/apache2', 'sites-available') + enabled = os.path.dirname(available) + os.sep + 'sites-enabled' + fn = available + os.sep + domain + '.conf' + if os.path.exists(fn) and not overwrite: + self.usage('{} exists. Use --overwrite to overwrite'.format(fn)) + base.StringUtils.toFile(fn, '''<VirtualHost *:80> + ServerName {} + ServerAlias {} + ServerAdmin webmaster@localhost + DocumentRoot {} + <Directory {}> + AllowOverride all + Require all granted + Order allow,deny + allow from all + </Directory> +</VirtualHost> +'''.format(domain, ip, baseDir, baseDir)) + linkTarget = enabled + os.sep + domain + '.conf' + if os.path.islink(linkTarget): + self._logger.log('deleting ' + linkTarget, 2) + os.unlink(linkTarget) + self._logger.log('creating symlink ' + linkTarget, 2) + os.symlink('../sites-available/' + domain + '.conf', linkTarget) + fn = baseDir + os.sep + 'index.html' + if self._isRoot: + self._processHelper.execute(['/bin/systemctl', 'reload', 'apache2'], True) + else: + self.usage('unknown webserver: ' + webserver) + + def dump(self): + '''Dumps a monitor. + @return: a string describing the monitor + ''' + rc = '' + for group in sorted(self._receiverGroups): + rc += self._receiverGroups[group].dump(0) + for host in sorted(self._smtpHosts): + rc += self._smtpHosts[host].dump(0) + for host in sorted(self._hosts): + rc += self._hosts[host].dump(0) + for site in sorted(self._sites): + rc += self._sites[site].dump(0) + return rc + + def diskLimitReached(self, limit, total, free): + '''Tests whether a limit is reached. + @param limit: in bytes or as percent + @param total: total amount of disk bytes + @param free: free disk bytes + @return: True: limit is reached + ''' + limitBytes = int (total * int(limit[0:-1]) / 100) if limit.endswith('%') else int(limit) + rc = limitBytes < int(total - free) + return rc + + def example(self): + '''Prints a configuration example to files and prints / returns a reference notice. + @param doPrint: True: the reference notice will be displayed. + @return: the reference notice, e.g. 'created: /etc/pyrshell/monitor.d/default.example' + ''' + content = '''# Example config for pymonitor: +log=/var/log/local/pymonitor.log +''' + self.storeExample(content) + contentDefault = ''' +receivergroup "" { + warning: hm.neutral@gmx.de + error: hm.neutral@gmx.de +} +receivergroup "std" { + warning: hm.neutral@gmx.de + error: hm.neutral@gmx.de +} +smtphost "" { + host: smtp.gmx.de + port: 587 + sender: hm.neutral@gmx.de + user: hm.neutral@gmx.de + code: sEcReT + tls: True +} +host "" { + receivers: std + disk "" { + receivers: std + used: 85% 90% + } +} +''' + self.storeExample(contentDefault, 'default.conf', self._additionalConfigDir) + content = '''host "{}" { +address: localhost +interval: 60 +''' + host = base.BaseTool.BasicStatics.hostname(False) + content = content.replace('{}', host) + diskInfos = base.LinuxUtils.diskFree() + diskDescription = '' + for info in diskInfos: + # info: [ name, total, free ] + total = 1 if info[1] == 0 else info[1] + warnLimit = (100 * (total - info[2]) / total + 100) / 2 + errorLimit = (warnLimit + 100) / 2 + diskDescription += '\tdisk "' + info[0] + '" {\n' + diskDescription += '\t\tused: {:.0f}% {:.0f}%\n'.format(warnLimit, errorLimit) + diskDescription += '\t}\n' + content += diskDescription + infos = base.LinuxUtils.mdadmInfo() + for info in infos: + # [name, raidType, members, blocks, status + content += '\traid "' + info[0] + '" {\n' + content += '\t\traidtype: ' + info[1] + '\n' + content += '\t\tmembers: ' + info[2] + '\n' + content += '\t\tblocks: {:d}\n'.format(info[3]) + content += '\t}\n' + + infos = base.LinuxUtils.load() + content += '\tload1: {:.1f} {:.1f}\n'.format(infos[0] * 10, infos[0] * 20) + content += '\tload5: {:.1f} {:.1f}\n'.format(infos[1] * 10, infos[1] * 20) + content += '\tload10: {:.1f} {:.1f}\n'.format(infos[2] * 10, infos[2] * 20) + content += '\tprocesses: {:.0f} {:.0f}\n'.format(int(infos[4] * 1.1), int(infos[4] * 2)) + infos = base.LinuxUtils.memoryInfo() + total = infos[2] if infos[2] != 0 else 1 + swapUsage = (100 + infos[3] * 100 / total) / 2 + content += '\tswap: {}% {}%\n'.format(swapUsage, (swapUsage+100)/2) + content += '}\n' + self.storeExample(content, host + '.conf', self._additionalConfigDir) + + def getLocalHost(self): + '''Returns the name of the host describing the localhost. + @return: the name of the local host + ''' + rc = None + for host in self._hosts: + if host != '': + if host == 'localhost': + rc = host + else: + rc = host + break + return rc + + def observe(self, hostname='localhost'): + '''Observes a host and return the new errors/warnings. + @param hostname: name of the host to observer: used for the configuration + @return: a tuple (newErrors, closedErrors) + ''' + self._logger.log('observe...', 3) + self._currentHostname = hostname + rc = self.observeCore(hostname) + self.observeDisks(hostname) + self.observeRaid(hostname) + rc += self.observeSites(hostname) + currentProblems = dict() + newProblems = dict() + closedProblems = [] + for message in rc: + parts = message.split('|') + key = parts[0] + ':' + parts[1] + if key in self._openProblems: + currentProblems[key] = self._openProblems[key] + else: + newProblems[key] = message + for key in self._openProblems: + if key not in newProblems and key not in currentProblems: + closedProblems.append(self._openProblems[key]) + self._openProblems = currentProblems + self._openProblems.update(newProblems) + return (newProblems.values(), closedProblems) + + def observerBuildMessage(self, section, location, isWarning, limitType, current, observed): + '''Builds the message used for the transporting all infos about an error/warning. + @param section: identifies the caller, e.g. 'core' + @param location: the part of the message specific for the caller, e.g. 'core detection' + @param isWarning: False: describes an error + @param limitType: load1,load5,load10 or swap + @param current: the current value + @param observed: the property describing the raised limit, type: ObservedProperty + @return: list of notice info: notice_type|property_key|message|receivers|time, 'E|localhost:disk:/home|disk usage: free: 0 of 512.000 GiByte + ''' + receivers = '' + if observed == None: + limit = '' + else: + limit = observed._warnLimit if isWarning else observed._errorLimit + if observed._receivers != None: + receivers = observed._receivers._properties['warning' if isWarning else 'error']._value + msg = 'S' if isWarning == None else ('W' if isWarning else 'E') + msg += '|{}:{}:{}|{} '.format(self._currentHostname, section, limitType, location) + msg += 'warning' if isWarning else 'error' + msg += ' ' + str(current) + if limit != None and limit != '': + msg += ' Limit: ' + str(limit) + msg += '|' + receivers + '|' + "{:.0f}".format(time.time()) + return msg + + def observeCore(self, hostname='localhost'): + '''Checks whether a load state and swap usage should be noticed (warning or error). + @return: list of notice info: notice_type|property_key|message|receivers|time + e.g. 'E|localhost:load5|load is to heavy: 5.10|a@bc.de|147382902.3928302 + ''' + def buildMessage(isWarning, limitType, current): + return self.observerBuildMessage('core', 'core detection', isWarning, limitType, current, host._observed[limitType]) + + def checkOne(rc, current, limitType): + if float(current) >= float(host._observed[limitType]._errorLimit): + rc.append(buildMessage(False, limitType, current)) + elif float(current) >= float(host._observed[limitType]._warnLimit): + rc.append(buildMessage(True, limitType, current)) + else: + self._lastStatus[host._name + ':core:' + limitType] = buildMessage(None, limitType, current) + + self._logger.log('observeCore...', 3) + rc = [] + if hostname in self._hosts: + infos = base.LinuxUtils.load() + host = self._hosts[hostname] + checkOne(rc, infos[0], 'load1') + checkOne(rc, infos[1], 'load5') + checkOne(rc, infos[2], 'load10') + checkOne(rc, infos[4], 'processes') + infos = base.LinuxUtils.memoryInfo() + total = infos[2] if infos[2] != 0 else 1 + usagePercent = 0 if total <= 1 else (total - infos[3]) * 100 / total + checkOne(rc, usagePercent, 'swap') + + return rc + + def observeDisks(self, hostname='localhost'): + '''Checks whether a disk usage should be noticed (warning or error). + @return: list of notice info: notice_type|property_key|message|receivers|time, 'E|localhost:disk:/home|disk usage: free: 0 of 512.000 GiByte + ''' + + def buildMessage(isWarning, diskName, total, free, limitType, observed): + total = 1 if total < 1E-6 else total + current = 'free: {:.3f} GiByte ({:.1f}%) of {:.3f} GiByte'.format(free / 1024.0 / 1024 / 1024, free * 100.0 / total, total / 1024.0 / 1024 / 1024) + return self.observerBuildMessage('disk', 'disk usage', isWarning, limitType, current, observed) + + self._logger.log('observeDisks...', 3) + rc = [] + diskInfos = base.LinuxUtils.diskFree() + if hostname in self._hosts: + host = self._hosts[hostname] + for info in diskInfos: + # info: [ name, total, free ] + if info[0] not in host._disks: + # rc.append('E|'+ hostname + ':disk:' + info[0] + '|disk not found|' + host._properties['receivers']._properties['error']._value) + pass + else: + disk = host._disks[info[0]] + if self.diskLimitReached (disk._observed['used']._errorLimit, info[1], info[2]): + msg = buildMessage(False, info[0], info[1], info[2], disk._name, disk._observed['used']) + self._logger.log(msg, 2) + rc.append(msg) + elif self.diskLimitReached (disk._observed['used']._warnLimit, info[1], info[2]): + msg = buildMessage(True, info[0], info[1], info[2], disk._name, disk._observed['used']) + self._logger.log(msg, 2) + rc.append(msg) + else: + self._lastStatus[host._name + ':disk:' + info[0]] = buildMessage(None, info[0], info[1], info[2], disk._name, disk._observed['used']) + return rc + + def observeRaid(self, hostname='localhost'): + self._logger.log('observeDisks...', 3) + '''Tests whether a raid is broken. + @return: list of notice info: notice_type|property_key|message|receivers|time, + e.g. 'E|localhost:raid|raid is broken|a@bc.de|147382902.3928302' + ''' + + def buildMessage(isWarning, limitType, raid, message): + if raid._properties['members']._value != None: + message += ' members: ' + raid._properties['members']._value + if raid._properties['blocks']._value != None: + message += ' blocks: ' + raid._properties['blocks']._value + return self.observerBuildMessage('raid', 'type: {} {}'.format(raid._properties['raidtype']._value, message), isWarning, limitType, '', None) + + self._logger.log('observeRaid...', 3) + infos = base.LinuxUtils.mdadmInfo() + host = self._hosts[hostname] + raidMap = dict() + rc = [] + for info in infos: + raidMap[info[0]] = info + for name in host._raids: + raid = host._raids[name] + if raid._name not in raidMap: + rc.append(buildMessage(False, raid._name, raid, 'missing raid device')) + else: + [name, raidType, members, blocks, status] = raidMap[raid._name] + if status == 'recover': + rc.append(buildMessage(True, raid._name, raid, 'rebuilding the raid')) + elif status == 'broken': + rc.append(buildMessage(False, raid._name, raid, 'raid is broken')) + else: + self._lastStatus[host._name + ':raid:' + raid._name] = buildMessage(None, raid._name, raid, 'OK') + return rc + + def observeSites(self, hostname): + '''Tests whether a site is reachable. + @param hostname: the hostname of 'localhost' + @return: list of notice info: notice_type|property_key|message|receivers|time, + e.g. 'E|localhost:site:https://wiki.example.com|site is not reachable|a@bc.de|147382902.3928302' + ''' + if self._verboseLevel >= 3: + self._logger.log('observeSites...') + rc = [] + for site in self._sites: + url = self._sites[site]._properties['url']._value + client = net.HttpClient.HttpClient(self._verboseLevel, self._logger) + content = client.getContent(url, 1) + # 404 returns an empty bytes object + if content == None or content == '' or content == b'' or content.find(b'404') >= 0: + receivers = self._hosts[hostname]._properties['receivers']._properties['error']._value + msg = 'E|{}:sites:{}|site is not reachable|{}|{:.0f}'.format(hostname, site, receivers, time.time()) + rc.append(msg) + else: + self._lastStatus['{}:sites:{}'.format(hostname, site)] = 'S|{}:sites:{}|site is running||{}'.format(hostname, site, time.time()) + client.close() + return rc + + def readConfigurationFile(self, filename): + '''Reads a configuration file. + @param filename: file to read + Example of a configuration file: + host "localhost" { + address: localhost; + disk "/" { + used: 80% 90% + } + load1: 3.0 5.0 + } + ''' + + def _error(msg): + '''Puts an error with context information. + @param msg: the error message + ''' + self._logger.error('{:s}-{:d}: {:s}\n{:s}'.format(filename, lineNo, msg, line)) + + def _checkEmails(emails): + '''Tests a list of emails. + @param emails: a string with email addresses separated by blanks + @return: True: emails correct + ''' + rc = True + for item in emails.split(): + if not reEmailAddress.match(item): + _error('invalid email address: ' + item) + rc = False + return rc + + def _observedValue(name, value): + rc = value[0:-1] if name == 'swap' and value[-1] == '%' else value + return rc + + with open(filename, 'r') as fp: + lineNo = 0 + containerStack = [] + currentContainer = None + reReceiverGroup = re.compile(r'receivergroup\s+"([^"]*)"\s*\{$') + reContainer = re.compile(r'(site|host|disk|raid|smtphost)\s+"([^"]*)"\s*\{$') + reProperty = re.compile(r'(\w+):\s*(.+)$') + reEmailAddress = re.compile(r'[-+\w.=!]+@[\w.]+[.][a-zA-z]+$') + for line in fp: + lineNo += 1 + line = line.strip() + if line == '' or line.startswith('#'): + continue + if line == '}': + if currentContainer == None: + _error('unexpected "}"') + else: + containerStack.pop() + currentContainer = None if len(containerStack) == 0 else containerStack[-1] + continue + matcher = reContainer.match(line) + if matcher: + aType, name = matcher.group(1), matcher.group(2) + if aType == 'host' or aType == 'smtphost' or aType == 'site': + if currentContainer != None: + _error('nested definition of ' + aType) + else: + if aType == 'host': + currentContainer = Host(name, self) + self._hosts[name] = currentContainer + elif aType == 'site': + currentContainer = WebSite(name, self) + self._sites[name] = currentContainer + else: + currentContainer = SmtpHost(name, self) + self._smtpHosts[name] = currentContainer + containerStack.append(currentContainer) + elif aType == 'disk': + if currentContainer == None: + _error('disk not inside a host definition') + elif currentContainer._keyword != 'host': + _error('disk not inside a host definition. Found: ' + currentContainer._keyword) + else: + disk = Disk(name, currentContainer) + currentContainer._disks[name] = disk + currentContainer = disk + containerStack.append(disk) + elif aType == 'raid': + if currentContainer == None: + _error('raid not inside a host definition') + elif currentContainer._keyword != 'host': + _error('raid not inside a host definition. Found: ' + currentContainer._keyword) + else: + raid = RaidDevice(name, currentContainer) + currentContainer._raids[name] = raid + currentContainer = raid + containerStack.append(raid) + else: + _error('unknown container type: ' + aType) + continue + matcher = reProperty.match(line) + if matcher: + name, propValue = matcher.group(1), matcher.group(2) + if currentContainer == None: + _error('property outside a container') + elif name not in currentContainer._properties and name not in currentContainer._observed: + _error('unknown property ' + name) + else: + if name in currentContainer._properties: + if name == 'receivers': + if propValue in self._receiverGroups: + currentContainer._properties[name] = self._receiverGroups[propValue] + else: + _error('unknown receivergroup: ' + propValue) + elif (name != 'error' and name != 'warning') or _checkEmails(propValue): + currentContainer._properties[name]._value = propValue + if name == 'address' and propValue != 'localhost' and not propValue.startswith('127.'): + currentContainer._name = propValue + else: + values = propValue.split() + if len(values) == 1: + currentContainer._observed[name]._errorLimit = _observedValue(name, values[0]) + currentContainer._observed[name]._receivers = currentContainer._properties['receivers'] + elif len(values) == 2: + currentContainer._observed[name]._warnLimit = _observedValue(name, values[0]) + currentContainer._observed[name]._errorLimit = _observedValue(name, values[1]) + currentContainer._observed[name]._receivers = currentContainer._properties['receivers'] + else: + if values[2] not in self._receiverGroups: + _error('unknown receivergroup: ' + values[2]) + else: + currentContainer._observed[name]._warnLimit = _observedValue(name, values[0]) + currentContainer._observed[name]._errorLimit = _observedValue(name, values[1]) + currentContainer._observed[name]._receivers = self._receiverGroups[values[2]] + + continue + matcher = reReceiverGroup.match(line) + if matcher: + if currentContainer != None: + _error('receiver group definition inside a container') + else: + group = ReceiverGroup(matcher.group(1), self) + self._receiverGroups[group._name] = group + currentContainer = group + containerStack.append(group) + continue + _error('unexpected input') + if len(containerStack) > 0: + _error('missing "}"') + + def reload(self): + '''Reads the configuration again. + ''' + self._logger.log('reloading configuration...') + self._readAllConfigs() + hostname = base.BaseTool.BasicStatics.hostname(False) + subject = hostname + ': reloaded' + duration = time.time() - self._start + start = datetime.datetime.fromtimestamp(self._start) + currentState = '' + states = list(self._lastStatus.values()) + states.sort() + for state in states: + info = state.split('|') + currentState += info[1] + ': ' + info[2].replace(' error', '') + '\n' + body = '{}\n\nRunning since: {} / {:.0f}d{:.0f}h{:.0f}m loops: {}\n\nOpen problems: {}\n\n{}\n\nCurrent state:\n{}\n'.format(subject, + start.strftime('%Y.%m.%d-%H:%M:%S'), + duration // 86400, duration % 86400 / 3600, duration % 3600 / 60, + self._loops, + len(self._openProblems), '\n'.join(self._openProblems), + currentState) + self.sendEmail(subject, body) + + def reloadRequest(self): + '''Requests a reload of the configuration of the running service. + ''' + base.StringUtils.toFile(self._reloadRequestFile, '') + os.chmod(self._reloadRequestFile, 0o666) + entry = pwd.getpwnam('pymonitor') + uid = self._configuration.getInt('uid', None if entry == None else entry.pw_uid) + if self._isRoot and uid != None: + os.chown(self._reloadRequestFile, uid, uid) + + def run(self, argv): + '''A never ending loop with regulary started observervations. + @param hostname: the hostname to find the matching configuration + ''' + hostname = base.BaseTool.BasicStatics.hostname(False) + interval=60 + if len(argv) > 0: + hostname = argv[0] + if len(argv) > 1: + interval = self.integerArgument(argv[1], 60) + if hostname not in self._hosts: + self._logger.error('unknown host: ' + hostname) + else: + try: + time.sleep(10) + host = self._hosts[hostname] + interval = int(host._properties['interval']._value) + if interval < 1: + interval = 1 + self._logger.log('starting loop (interval: {})...'.format(interval)) + # seconds since midnight + lastClear = time.time() % 86400 + self._loops = 0 + while True: + self._loops += 1 + current = time.time() % 86400 + if current < lastClear: + # day has changed + # enforce new error messages on long term errors + self._openProblems.clear() + lastClear = current + self.checkReload() + [newProblems, closedProblems] = self.observe(hostname) + for key in newProblems: + self.sendStatusEmail(key, True) + for key in closedProblems: + self.sendStatusEmail(key, False) + time.sleep(interval) + except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + self._logger.error(''.join(traceback.format_exception(exc_type, exc_value, exc_traceback, 8))) + self._logger.log('daemon finished') + + def sendStatusEmail(self, error, isProblem): + '''Sends one or many emails. + @param errors: a list of error information + @param isProblem: True: the errors have been started False: the errors have been finished + ''' + [aType, key, message, recipients, theTime] = error.split('|') + if len(recipients) > 0: + aType2 = 'error' if aType == 'E' else 'warning' + aDate = datetime.datetime.fromtimestamp(int(float(theTime))) + dateString = aDate.strftime('%d.%m.%Y %H:%M') + [host, scope, dummy] = key.split(':') + if isProblem: + subject = '[PROBLEM] ' + key + ' is on ' + aType2 + text = '''***** Service Monitoring on {:s} ***** +Scope: {:s} +Info: {:s} +Service: {:s} +When: {:s} +'''.format(key, host, message, scope, dateString) + else: + subject = '[RECOVERY] ' + key + ' is ok' + text = '''***** Service Monitoring on {:s} ***** +Scope: {:s} +Recovered from: {:s} +Service: {:s} +When: {:s} - {:s} +'''.format(key, host, message, scope, dateString, datetime.datetime.now().strftime('%d.%m.%Y %H:%M')) + if key in self._lastStatus: + info = self._lastStatus[key].split('|') + text += ''' +current status: {} +from {} +'''.format(info[2], time.strftime('%Y.%m.%d-%H:%M:%S', time.localtime(float(info[4])))) + self.sendEmail(subject, text, recipients) + + def sendEmail(self, subject, text, recipients = None): + '''Sends an email. + @param subject: the subject of the email + @param isProblem: True: the errors have been started False: the errors have been finished + ''' + email = net.EMail.EMail(subject, text) + smtp = self._smtpHosts[''] + if recipients == None: + recipients = self._receiverGroups['']._properties['warning']._value + if recipients == None or recipients == '': + self._logger.error('missing recipients in sendEmail(): subject: {} text:\n{}'.format(subject, text)) + else: + parts = recipients.split(' ') + email.setSmtpLogin(smtp._properties['host']._value, smtp._properties['port']._value, + smtp._properties['user']._value, smtp._properties['code']._value, smtp._properties['tls']._value == 'True') + sender = smtp._properties['sender']._value + cc = None if len(parts) < 2 else parts[1:] + #self._logger.debug('Email: To: {} CC: {} host: {} port: {} user: {} TLS: {}'.format( + # parts[0], '' if cc == None else ' '.join(cc), + # smtp._properties['host']._value, smtp._properties['port']._value, smtp._properties['user']._value, + # smtp._properties['tls']._value)) + try: + email.sendTo(sender, parts[0], cc) + except Exception as exc: + smtpArgs = smtp._properties['host']._value + smtp._properties['port']._value, smtp._properties['user']._value + smtp._properties['code']._value + smtp._properties['tls']._value + self._logger.error('sendmail [{}] to {} failed: {}\n{}\n{}'.format(smtpArgs, recipients, str(exc), subject, text)) + self._logger.debug('email sent to ' + recipients) + + def site(self, argv): + '''Displays the configuration of an observed website. + @param argv: program arguments, e.g. ['--scan'] + ''' + def _handleUrl(url, lines, logger): + if not url.startswith('http'): + url = 'http://' + url + client = net.HttpClient.HttpClient(self._verboseLevel, logger) + url2 = client.handleRequest(url, 'HEAD', 10) + if client._response != None and client._response.status >= 400 and client._response.status < 500: + url += '/works' + url2 = client.handleRequest(url, 'HEAD', 10) + if client._response == None or client._response.status != 200: + logger.error('site not available: ' + url) + else: + parts = urllib.parse.urlparse(url2) + domain = parts.netloc + ix = domain.find(':') + if ix > 0: + domain = domain[0:ix] + output = 'site "' + domain + '" {\n\turl: ' + url2 + '\n}\n' + logger.log(output) + lines.append(output) + + def _scanNginx(filename, lines, logger): + rexprPort = re.compile(r'^\s*listen\s+[sl\s]*(\d+)', IGNORE_CASE) + rexprServer = re.compile(r'^\s*server_name\s+(.*);', IGNORE_CASE) + with open(filename, "r") as fp: + lastPort = 80 + lastIsSsl = False + bestUrl = None + url = None + for line in fp: + matcher = rexprPort.match(line) + if matcher: + lastPort = int(matcher.group(1)) + lastIsSsl = line.lower().find('ssl') > 0 + continue + matcher = rexprServer.match(line) + if matcher: + names = matcher.group(1).split() + url = 'http' + isHttps = lastIsSsl or (lastPort >= 400 and lastPort < 500) + if isHttps: + url += 's' + port = '' if lastPort == 80 or lastPort == 443 else ':' + str(lastPort) + url += '://' + names[0] + port + if bestUrl == None: + bestUrl = url + elif isHttps: + bestUrl = url + break + if bestUrl: + _handleUrl(bestUrl, lines, logger) + + argv, options = self.splitArgsAndOpts(argv) + done = False + lines = [] + for opt in options: + if opt == '--scan': + aDir = self.getSource('/etc/nginx/sites-enabled') + if not os.path.isdir(aDir): + self.usage('--scan: not a directory: ' + aDir) + files = os.listdir(aDir) + for node in files: + _scanNginx(aDir + '/' + node, lines, self._logger) + done = True + else: + self.usage('unknown option: ' + opt) + if not done: + if len(argv) < 1: + self.usage('site: missing <url>') + else: + for url in argv: + _handleUrl(url, lines, self._logger) + base.BaseTool.setResult(lines) + + def test(self, argv): + '''Tests the configuration. + @param argv: program arguments + ''' + dump = self.dump() + fn = '/tmp/{}.test.dump.txt'.format(self._globalOptions._appInfo._applicationName) + with open(fn, "w") as fp: + fp.write(dump) + print('=== dump stored in ' + fn) + # notice_type|property_key|message|receivers|time + receivers = self._receiverGroups['']._properties['warning']._value + now = time.time() + host = self.getLocalHost() + msg = 'W|{}:testmessage:emailtest|This is only a test to check email sending capability|{}|{}'.format(host, receivers, now) + self.sendStatusEmail(msg, False) + +def usage(): + '''Returns an info about usage. + ''' + return '''Usage: monitor <mode> [<opts>] +GLOBAL_OPTS +GLOBAL_MODES +<mode>: + create-site-server [<opts>] + creates the configuration for sites to observe + <opt>: + --domain=<domain> + used for the virtual server, e.g. hawk.hamatoma.de + --ip=<ip> + the public ip, e.g. 217.0.3.99 + --nginx + the configuration is created for the webserver NGINX. This is the default behaviour + --overwrite + the existing configuration will be overwritten + daemon <opts> + starts a never ending loop for monitoring + <opts>: + -v<verbose-level> + -v<level> or --verbose-level=<level> + Verbose level: 0: no tracing 1: summary 2: standard 3: fine Default: 0 + -c<dir> or --configuration-directory=<dir> + configuraton directory with *.conf files. + Default: os.environ['MONTITOR_CONFIG'] or '/etc/pyrshell/monitor.d' + --host=<host> + host to observe. Default: os.environ['MONTITOR_HOST'] or <hostname> + -l<file> or --log=<file> + file for logging output. + Default: os.environ['MONTITOR_LOG'] or /var/log/local/<application>.log + --application=<application> + the name of the application. Default: pymonitor + example [<file>] + prints an example configuration file to a file or stdout + install <opts> + installs the daemon as a systemd service + <opts>: + --application=<application> + the name of the application. Default: pymonitor + --user=<user> + the daemon runs as this user. Default: <application> or 'pymonitor' + --group=<group> + the daemon runs under this group. Default: <application> or 'pymonitor' + --host=<host> + the name of the host used for the daemon. Must be defined in configuration + Default: localhost + --no-auto-start + the service does not start at boot time (systemctl disable <application>) + -l<file> or --log=<log> + file for logging output (of the daemon). + Default: os.environ['MONTITOR_LOG'] or /var/log/local/<application>.log + uninstall <opts> + --purge + remove configuration files too + --application=<application> + the name of the application. Default: 'pymonitor' + --hold-user + the user will not be deleted. + Note: the user will be deleted only if its name is equal to the application + --hold-group + the user will not be deleted + Note: the group will be deleted only if its name is equal to the application + site <url1> [<url2> ...] + prints the configuration for <urlN> + <urlN>: + the universal resource locator, e.g. https://wiki.hamatoma.de + site --scan + prints configuration filtered from NgInx configuration files in <directory> + Default: '/etc/nginx/sites-enabled' + test <opts> + tests the configuration files. + <opts>: + -c<dir> or --configuration-directory=<dir> + configuraton directory with *.conf files. + Default: os.environ['MONTITOR_CONFIG'] or '/etc/pyrshell/monitor.d' + -l<file> or --log=<file> + file for logging output. Default: /var/log/local/<application>.log + --email=<email> + email address for email sending test. + Default: warning part of the default ReceiverGroup +Examples: +pymonitor -v3 daemon +pymonitor reload +pymonitor -v3 sites --scan +pymonitor -v3 create-site-server --ip=208.33.99.5 --domain=gustl.example.com --overwrite +pymonitor -v3 create-site-server --apache --overwrite +pymonitor -v3 site http://life.sky.infeos.de/domain.txt +''' + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/pymonitor', 'run'] + ''' + serviceInfo = base.BaseTool.ServiceInfo('pymonitor', 'pymonitor', 'pymonitor', + 'A monitor for system resources, e.g. disk usage.', 'MONITOR', None) + appInfo = base.BaseTool.ApplicationInfo('pymonitor', 'appl/Monitor.py', usage, serviceInfo) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = Monitor(options) + if len(argv) == 0 and 'MONITOR_APPL' in os.environ: + argv = ['daemon'] + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'create-site-server': + tool.createSiteServer(argv) + elif cmd == 'daemon': + tool.run(argv) + elif cmd == 'site': + tool.site(argv) + elif cmd == 'test': + tool.test(argv) + elif cmd == 'reload': + tool.reloadRequest() + else: + tool.usage("unknown subcommand: " + cmd) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/RestoreTool.py b/appl/RestoreTool.py new file mode 100755 index 0000000..05397cf --- /dev/null +++ b/appl/RestoreTool.py @@ -0,0 +1,2174 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' + +import time +import datetime +import sys +import os.path +import stat +import subprocess +import tempfile +import gzip +import traceback +import re +import shutil +import fnmatch + +sys.path.insert(0, '/usr/share/pyrshell') +import base.JavaConfig +import base.Logger +import base.BaseTool +import net.EMail +import appl.BackupBase +import base.ProcessHelper +import appl.TextTool + +class GroupInfo: + + def __init__(self, name, gid, groupLine): + self._name = name + self._groupLine = groupLine + self._gid = gid + self._comments = None + + def asText(self): + rc = '' + if self._comments != None: + rc += '\n'.join(self._comments) + '\n' + rc += self._groupLine + '\n' + return rc + +class UserInfo: + def __init__(self, name, uid, passwdLine): + self._name = name + self._passwdLine = passwdLine + self._uid = uid + self._comments = None + + def asText(self): + rc = '' + if self._comments != None: + rc += '\n'.join(self._comments) + '\n' + rc += self._passwdLine + '\n' + return rc + +class PartitionInfo: + def __init__(self, device, uuid, fsType, label): + '''Constructor. + @param device: the device, e.g. '/dev/sda1' + @param uuid: the UUID of the partition + @param fsType: the filesystem type, e.g. 'ext4 + @param label: None or the label of the filesystem + ''' + self._device = device + self._uuid = uuid + self._fsType = fsType + self._label = label + self._osName = 'linux' + + +class RestoreTool (appl.BackupBase.BackupBase): + + def __init__(self, globalOptions): + '''Constructor. + @param logger: the logger + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'restoretool.conf') + self._backupConfiguration = base.JavaConfig.JavaConfig(self._configDir + '/backup.conf', self._logger) + self._processTool = base.ProcessHelper.ProcessHelper(self._verboseLevel, self._logger) + self._textTool = appl.TextTool.TextTool(self._globalOptions) + + def basics(self, dirBackup, week=None, day=None): + '''Prepares the following restauration. + @param dirBackup: a directory con + @param week: a weekno: 0..3 (weekno mod 4): locates the weekly backup + @param day: 'Mon' ... 'Sun': locates the dayly backup + ''' + base.BaseTool.BasicStatics.ensureDirectory('/opt/restore/etc_origin', self.logger) + base.BaseTool.BasicStatics.ensureDirectory('/opt/restore/etc', self.logger) + if not os.path.exists('/opt/restore/etc_origin/passwd'): + self.log('saving /etc to /opt/restore/etc_origin') + shutil.copy('/etc', '/opt/restore/etc_origin') + + def btrFs(self, argv): + '''Executes the btrfs command. + @param argv: the arguments, e.g. ['create-subvol', 'cave', 'home'] + ''' + if len(argv) < 1: + self.usage('missing <what>') + else: + what = argv[0] + argv = argv[1:] + if what == 'create-fs': + self.btrFsCreateFs(argv) + elif what == 'create-subvol': + self.btrFsCreateSubvolume(argv) + elif what == 'create-snapshot': + self.btrFsCreateSnapshot(argv) + else: + self.usage('unknown <what>: ' + what) + + def btrFsCreateFs(self, argv): + '''Creates a subvolume of a btrfs filesystem. + Steps: + if loop device: create image and loop device + create the rootMountpoint + create the automount interface for mounting + create-btrfs <device> <label> [<mount-path>] [--image=<path>:<size> ] [--force] + @param argv: the program arguments, e.g. ['/dev/loop33', 'fs.unittest', '/tmp/fs.unittest', '--size=150M'] + ''' + (argv, options) = self.splitArgsAndOpts(argv) + force = False + size = None + image = None + mediaDir = self.getTarget('/media') + for opt in options: + if opt.startswith('--image='): + arg = opt[8:] + expr = self.regExprCompile(r'^(.*):(\d+[gtm])$', 'btrFsCreateSubvolume()', False) + if expr != None: + matcher = expr.match(arg) + if matcher == None: + self.usage('wrong format {}. example: --image=/space/fs.img:100G'.format(arg)) + else: + image, size = (matcher.group(1), matcher.group(2)) + if size[-1].upper() == 'M' and int(size[0:-1]) < 115: + self.usage('invalid size: {} minimum is 115M'.format(size)) + elif opt == '--force': + force = True + else: + self.usage('unknown option: ' + opt) + if len(argv) < 2: + self.usage('too few arguments') + elif image != None and not argv[0].startswith('/dev/loop'): + self.usage('--image found, but device does not start with "/dev/loop": ' + argv[0]) + else: + device = argv[0] + labelRoot = argv[1] + mountPoint = argv[2] if len(argv) > 2 else mediaDir + os.sep + labelRoot + if os.path.exists(mountPoint) and not force: + self.usage('{} already exists. Use --force if needed. '.format(mountPoint)) + elif re.match('^[-\w._]+$', labelRoot) == None: + self.usage('illegal characters in subvolume: ' + labelRoot) + else: + if force and os.path.exists(mountPoint): + self._logger.log('try to create the btrfs volume {} which already exists (--force found)'.format(labelRoot)) + else: + self.ensureDirectory(mountPoint) + if image != None and size != None: + self._processHelper.execute(['truncate', '--size=' + size, image], True) + script = self.getTarget('/etc/pyrshell/boot.d', 'btrfs.{}.sh'.format(labelRoot)) + self._logger.log('creating {} ...'.format(script), 1) + base.StringUtils.toFile(script , '''#! /bin/bash +losetup {} {} +'''.format(device, image)) + os.chmod(script, 0o755) + self._processHelper.execute(['losetup', device, image], True) + self._processHelper.execute(['/bin/bash', script], True) + etcDir = self.getTarget('/etc') + fn = etcDir + os.sep + 'auto.btrfs.' + labelRoot + base.FileHelper.ensureFileExists(fn, '', self._logger) + self._textTool.replaceOrInsert(['^' + mountPoint, '{}\t-fstype=btrfs\t:{}'.format(mountPoint, device), fn, '--create-if-missing', '--max-depth=0']) + master = etcDir + os.sep + 'auto.master' + writeToMaster = True + if not os.path.exists(master): + self._logger.error('missing ' + master) + if force: + base.FileHelper.ensureFileExists(master) + else: + writeToMaster = False + if writeToMaster: + self._textTool.replaceOrInsert(['.*' + fn, '/-\t' + fn, master, '--create-if-missing', '--max-depth=0']) + if self._isRoot: + self._processHelper.execute(['btrfs', 'quota', 'enable', mountPoint], True) + cmd = 'btrfs subvolume list MOUNT | cut -d" " -f2 | xargs "-I{}" -n1 btrfs qgroup "create 0/{}" ";"'.replace('MOUNT', mountPoint) + self._processHelper.execute([cmd], True, 'shell') + self._processHelper.execute(['systemctl', 'reload', 'autofs'], True) + + def btrFsCreateSnapshot(self, argv): + '''Duplicates a subvolume as snapshot. + Steps: + @param argv: the program arguments, e.g. ['fs.system', 'home', '--mode=dayly', '-r'] + ''' + (argv, options) = self.splitArgsAndOpts(argv) + readonly = False + mode = None + for opt in options: + if opt.startswith('--mode='): + mode = opt[7:] + if mode not in ['dayly', 'weekly', 'monthly', 'date', 'now']: + self.usage('unknown mode: ' + mode + ' use dayly|weekly|monthly|date|now') + elif opt == '--read-only' or opt == '-r': + readonly = True + else: + self.usage('unknown option: ' + opt) + if len(argv) < 2: + self.usage('too few arguments') + else: + labelRoot = argv[0] + labelSource = argv[1] + if len(argv) < 3: + if mode == None: + self.usage('missing <label-snapshot> or --mode') + labelSnapshot = None + else: + if mode != None: + self.usage('do not use <label-snapshot> and --mode at the same time') + else: + mode = 'special' + labelSnapshot = argv[2] + if mode != None: + now = datetime.datetime.now() + labelSnapshot = labelSource + '.' + if mode == 'dayly': + labelSnapshot += now.strftime('%a').lower() + elif mode == 'weekly': + labelSnapshot += str(int(now.strftime('%W')) % 4) + elif mode == 'monthly': + labelSnapshot += now.strftime('%m') + elif mode == 'date': + labelSnapshot += now.strftime('%Y.%m.%d') + elif mode == 'now': + labelSnapshot += now.strftime('%Y.%m.%d-%H-%M-%S') + elif mode == 'special': + labelSnapshot = argv[2] + else: + usage('unknown mode: ' + mode) + mountPath = self.getTarget('/media', 'snapshots/{}/{}/{}'.format(labelSource, mode, labelSnapshot)) + self.ensureDirectory(os.path.dirname(mountPath)) + mountSource = self.getTarget('/media/{}/{}'.format(labelRoot, labelSource)) + if os.path.isdir(mountPath): + self._processHelper.execute(['btrfs', 'subvol', 'delete', mountPath], True) + args = ['btrfs', 'subvol', 'snapshot', mountSource, mountPath] + if readonly: + args.insert(3, '-r') + self._processHelper.execute(args, True) + + + def btrFsCreateSubvolume(self, argv): + '''Installs a btrfs filesystem. + @param argv: the program arguments, e.g. ['wk.fs', 'backup', '--size=100G'] + ''' + (argv, options) = self.splitArgsAndOpts(argv) + force = False + size = None + mountOptions = None + transfer = False + for opt in options: + if opt.startswith('--size='): + size = opt[7:] + if re.match(r'^\d+[gGtTmM$', size) == None: + self.usage('illegal size: {} example: 100G'.format(size)) + elif opt.startswith('--options='): + mountOptions = opt[10:] + for item in mountOptions.split(','): + if item == 'compress=zlib' or item == 'compress=lzo': + pass + elif item == 'nodatacow': + pass + else: + self.usage('unknown mount option: ' + item) + elif opt == '--move-files': + transfer = True + elif opt == '--force': + force = True + else: + self.usage('unknown option: ' + opt) + if len(argv) < 2: + self.usage('too few arguments') + else: + labelRoot = argv[0] + mountRoot = self.getTarget('/media', labelRoot) + labelSubVol = argv[1] + mountPointCreation = mountRoot + os.sep + labelSubVol + mountPoint2 = None if len(argv) < 3 else argv[2] + files = 0 + if not transfer and mountPoint2 != None and os.path.exists(mountPoint2): + files = len(os.listdir(mountPoint2)) + if not os.path.exists(mountRoot): + self.usage('root of btrfs not found: ' + labelRoot) + elif re.match('^[-\w._]+$', labelSubVol) == None: + self.usage('illegal characters in subvolume: ' + labelSubVol) + elif os.path.exists(mountPointCreation) and not force: + self.usage('subvolume already exists: ' + labelSubVol) + elif files > 0: + self.usage('{} files/dirs will be hidden (use --move-files)'.format(files)) + else: + if force and os.path.exists(mountPointCreation): + self._logger.log('try to create a subvolume which already exists (--force found)') + self._processHelper.execute(['btrfs', 'subvolume', 'create', mountPointCreation], True) + if mountPoint2 != None: + dirOld = None + if transfer and os.path.isdir(mountPoint2): + dirOld = '{}.{}'.format(mountPoint2, int(time.time())) + os.rename(mountPoint2, dirOld) + self.ensureDirectory(mountPoint2) + fn = self.getTarget('/etc', 'fstab') + self._textTool.readCurrent(fn, True) + ix, matcher = self._textTool.currentFind2(re.compile(r'^([^# ]\S+)\s+{}\s'.format(mountRoot))) + if matcher == None: + self._logger.error('not found in /etc/fstab: ' + mountRoot) + else: + dev = matcher.group(1) + key = '{} {}'.format(dev, mountPoint2) + options = 'subvol={},defaults,noatime,space_cache,autodefrag'.format(labelSubVol) + if mountOptions != None: + options += ',' + mountOptions + self._logger.log('mount options: ' + options, 1) + line = '{} btrfs {} 0 0'.format(key, options) + self._textTool.currentInsertAnchored(line, key) + self._textTool.writeCurrent() + self._processHelper.execute(['mount', '-v', mountPoint2], True) + if dirOld != None: + for item in os.listdir(dirOld): + self._logger.log('moving {} to {}'.format(item, mountPoint2), 2) + shutil.move(dirOld + os.sep + item, mountPoint2 + os.sep + item) + def example(self): + example = '''# texttool example configuration +log.file=/var/log/local/restoretool.log +php.upload_max_filesize=624M +php.max_file_uploads=102 +php.post_max_size=448M +php.max_execution_time=900 +php.max_input_time=630 +''' + self.storeExample(example) + + def expandIp(self, ip, template = None): + '''Expands an shortened ip to a valid IP address. + @param ip: the (shortened) ip + @param template: None or a template to build the IP address + @return: the valid ip + ''' + if ip == None and template != None: + ip = '.'.join(template.split('.')[0:-1]) + '.1' + else: + parts = ip.split('.') + templ = None if template == None else template.split('.') + if len(parts) == 1: + if template == None: + ip = '10.10.10.' + ip + else: + ip = '.'.join(templ[0:-1]) + '.' + ip + elif len(parts) == 2: + if template == None: + ip = parts[0] + '.10.10.' + parts[1] + else: + ip = parts[0] + '.' + '.'.join(templ[1:-1]) + '.' + parts[1] + if re.match(r'^(\d{1,3}\.){3}\d{1,3}$', ip) == None: + self.usage('not a valid IP address: ' + ip) + return ip + + def init(self, argv): + '''Executes the init command. + @param argv: the arguments, e.g. ['php'] + ''' + if len(argv) < 1: + self.usage('missing <what>') + else: + what = argv[0] + argv = argv[1:] + if what == 'etc': + self._logger.log('setting vm.swappiness...') + fn = self.createPath('/etc', 'sysctl.conf') + self._textTool.currentSimpleInsert(fn, '^vm.swappiness', 'vm.swappiness = 20', None) + elif what == 'dirs': + base.BaseTool.BasicStatics.ensureDirectory('/media/tmp') + base.BaseTool.BasicStatics.ensureDirectory('/media/trg') + base.BaseTool.BasicStatics.ensureDirectory('/media/src') + base.BaseTool.BasicStatics.ensureDirectory('/var/log/local') + base.BaseTool.BasicStatics.ensureDirectory('/var/cache/local') + elif what == 'linuxserver': + self.linuxServer() + elif what == 'local-bin': + tarNode = 'local_bin.tgz' + tar = '/tmp/' + tarNode + url = self._configuration.getString('url.download', 'https://public.hamatoma.de') + self._processTool.execute(['/usr/bin/wget', '-O', tar, url + '/' + tarNode], True) + self.restoreDirectoryByTar(tar, '/usr/local/bin', None, True, None, False) + elif what == 'apache': + self.installApache(argv) + elif what == 'nginx': + self.installNginx(argv) + elif what == 'php': + self.installPhp(argv) + elif what == 'mariadb': + packets = 'mariadb-common mariadb-server mariadb-client'.split(' ') + self._processTool.execute(['/usr/bin/apt-get', '-y', 'install' ] + packets, True) + elif what == 'cms': + packets = 'imagemagick php-redis redis-tools redis-server'.split(' ') + self._processTool.execute(['/usr/bin/apt-get', '-y', 'install'] + packets, True) + elif what == 'letsencrypt': + self.installLetsencrypt() + elif what == 'grub': + self.initGrub(argv) + else: + self.usage('unknown <what>: ' + what) + + def installApache(self, argv): + '''Installs the Apache Webserver. + @param argv: program arguments, e.g. ['--ports=81,444'] + ''' + argv, options = self.splitArgsAndOpts(argv) + ports = [80, 443] + for opt in options: + if opt.startswith('--ports='): + if re.match(r'^\d+,\d+$', opt[8:]) == None: + self.usage('not 2 comma separated ports in: ' + opt) + else: + parts = opt[8:].split(',') + ports = [int(parts[0]), int(parts[1])] + if self._isRoot: + packets = 'apache2 libapache2-mod-php libapache2-mod-php5.6 libapache2-mod-php7.0 libapache2-mod-php7.1 libapache2-mod-php7.2 libapache2-mod-php7.3' + args = ['/usr/bin/apt-get', '-y', 'install' ] + packets.split(' ') + self._processTool.execute(args, True) + service = 'apache2' + fn = self.getFilenameOrCopy('/etc/apache2', 'ports.conf') + self._textTool.readCurrent(fn, True) + (start, end) = self._textTool.findRegion('^', True, '^<', False) + if start >= 0: + self._textTool.currentReplace(r'Listen\s', 'Listen {}'.format(ports[0]), None, False, start, end) + (start, end) = self._textTool.findRegion('<IfModule ssl_module>', True, '</IfModule', True) + if start >= 0: + self._textTool.currentReplace(r'\s*Listen\s', '\tListen {}'.format(ports[1]), None, False, start, end) + (start, end) = self._textTool.findRegion('<IfModule mod_gnutls.c>', True, '</IfModule', True) + if start >= 0: + self._textTool.currentReplace(r'\s*Listen\s', '\tListen {}'.format(ports[1]), None, False, start, end) + self._textTool.writeCurrent(fn) + if self._isRoot: + self._processTool.execute(['a2enmod', 'rewrite', service], True) + self._processTool.execute(['systemctl', 'enable', service], True) + self._processTool.execute(['systemctl', 'start', service], True) + self._processTool.execute(['systemctl', 'status', service], True) + + def initGrub(self, argv): + '''Extends the grub/grub.cfg with switches to all other found grub.cfg files on other disks. + @param argv: arguments + ''' + argv, opts = self.splitArgsAndOpts(argv) + dry = False + for opt in opts: + if opt == '--dry': + dry = True + else: + self.usage('unknown option: ' + opt) + fnGrubConfig = self.getTarget('/boot/grub', 'grub.cfg') + if not os.path.exists(fnGrubConfig): + self._logger.error('missing ' + fnGrubConfig) + else: + # look for block devices with mountable filesystems containing /boot/grub or /grub + underTest = not fnGrubConfig.startswith('/boot') + if underTest: + fnDevices = self.getTarget('/boot/grub', 'blkid.out') + lines = base.StringUtils.fromFile(fnDevices, '\n') + else: + self._processTool.executeInputOutput(['blkid'], None) + lines = self._processTool._rawOutput.decode().split('\n') + regLine = re.compile(r'(/dev/\S+): (.*)') + regLabel = re.compile(r'LABEL="([^"]+)"') + regUUID = re.compile(r'\bUUID="([^"]+)"') + regType = re.compile(r'\bTYPE="([^"]+)"') + self._gptDisks = [] + sections = [] + for line in lines: + matcher = regLine.match(line) + if matcher != None: + dev = matcher.group(1) + if line.find('PTTYPE="gpt"') >= 0: + self._gptDisks.append(dev) + continue + attr = matcher.group(2) + matcher = regLabel.search(attr) + label = None if matcher == None else matcher.group(1) + matcher = regUUID.search(attr) + uuid = None if matcher == None else matcher.group(1) + matcher = regType.search(attr) + fsType = None if matcher == None else matcher.group(1) + partInfo = PartitionInfo(dev, uuid, fsType, label) + if fsType == 'btrfs': + self.initGrubSearchBtrFs(sections, partInfo) + elif fsType != None and fsType != 'vfat' and fsType != 'ntfs': + found = True + if not underTest: + self.initGrubMount(partInfo, None) + found = self.initGrubIsBootDir(self._mountPoint) + # found may be None! + if found == True: + partInfo._osName = self.initGrubFindOsInfo(self._mountPoint) + self.initGrubUnmount() + if found: + self.initGrubAddMenuEntry(sections, partInfo, '') + if len(sections) > 0: + self._textTool.readCurrent(fnGrubConfig) + again = True + while again: + start, end = self._textTool.findRegion(r'^menuentry "=>', True, r'menuentry|submenu', False) + again = start > 0 + if again: + self._logger.log('removing ' + self._textTool._lines[start], 2) + self._textTool.removeRegion(start, end) + start, end = self._textTool.findRegion(r'^menuentry', False, r'menuentry|submenu', False) + self._textTool._lines.insert(end, '\n'.join(sections)) + if dry: + self._logger.log('no changes if --dry found') + else: + self._textTool.writeCurrent(None, True) + + def initGrubFindOsInfo(self, rootDir): + '''Tries to find out the OS installed on the given partition + @param rootDir: the name of the mount point of the partition + @return: 'linux' or the name of the installed operating system + ''' + rc = 'linux' + etc = rootDir + os.sep + 'etc' + if os.path.isdir(etc): + nodes = os.listdir(etc) + for node in nodes: + if node.endswith('release'): + fn = etc + os.sep + node + if not os.path.isdir(fn): + osInfo = base.StringUtils.fromFile(fn, '\n')[0] + ix = osInfo.find('=') + if ix > 0: + osInfo = osInfo[ix+1:] + rc = osInfo.strip('"') + break + return rc + + def initGrubAddMenuEntry(self, sections, partInfo, prefix): + '''Adds one menu entry to the array. + @param sections: IN/OUT: the list of menu entries + @param partInfo: the data of the partition + @param prefix: prefix of the path containing the grub directory, e.g. '/fs.system/boot' + ''' + partName = 'part_msdos' + if len(self._gptDisks) > 0: + for disk in self._gptDisks: + if partInfo._device.startswith(disk): + partName = 'part_gpt' + break + fsName = 'btrfs' if partInfo._fsType == 'btrfs' else 'ext2' + section = '''menuentry "=> {} on {}" {} + insmod {} + insmod {} + search --no-floppy --fs-uuid --set=root {} + configfile {}/grub/grub.cfg +{} +'''.format(partInfo._osName, partInfo._device, '{', partName, fsName, partInfo._uuid, prefix, '}') + self._logger.log('{}: OS: {} label: {} type: {} location: {}/grub'.format(partInfo._device, partInfo._osName, partInfo._label, partInfo._fsType, prefix), 1) + sections.append(section) + + def initGrubIsBootDir(self, path): + '''Tests whether a path has a grub directory or not. + @param path: path to inspect + @return: None: not a grub directory + True: the path contains a grub directory with prefix '/boot' + False: the path contains a grub directory without prefix '/boot' + ''' + rc = os.path.exists(path + '/boot/grub/grub.cfg') + if not rc: + rc = os.path.exists(path + '/grub/grub.cfg') + if not rc: + rc = None + return rc + + def initGrubMount(self, partInfo, options): + '''Mounts the device in the partInfo. + @param partInfo: the partition info + @param options: the mount options + ''' + self._mountPoint = '/media/tmp.grub' + self.ensureDirectory(self._mountPoint) + opts = 'ro' if options == None else 'ro,' + options + device = None if partInfo._uuid == None else 'UUID=' + partInfo._uuid + device = partInfo._device if device == None else device + self._processTool.execute(['mount', '-o', opts, device, self._mountPoint], True) + if len(self._processTool._error) > 0: + matcher = re.search(r'already mounted on (.*).$', self._processTool._error[0]) + if matcher != None: + self._mountPoint = matcher.group(1) + self._logger.log('using other mountpoint: ' + self._mountPoint) + + def initGrubSearchBtrFs(self, sections, partInfo): + '''Adds one menu entry to the array. + @param sections: IN/OUT: the list of menu entries + @param partInfo: the data of the partition + ''' + self.initGrubMount(partInfo, 'subvolid=5') + path = self._mountPoint + nodes = os.listdir(path) + for node in nodes: + full = path + os.sep + node + isBootDir = self.initGrubIsBootDir(full) + if isBootDir != None: + partInfo._osName = self.initGrubFindOsInfo(full) + self.initGrubAddMenuEntry(sections, partInfo, '/' + (node if not isBootDir else node + '/boot')) + self.initGrubUnmount() + + + def initGrubUnmount(self, ): + '''Unmounts the device in the partInfo. + ''' + if self._mountPoint == '/media/tmp.group': + self._processTool.execute(['umount', self._mountPoint], True) + self._mountPoint = None + + def installNetworkFindInterface(self, pattern): + '''Finds the interface from the existing one. + Finds the network devices and return the first matching the pattern + @param pattern: None or a pattern e.g. 'eth*' + @return: None: no interface available otherwise: the first interface matching the pattern + ''' + rc = None + self._processTool.executeInputOutput(['ip', 'addr', 'show'], None) + lines = self._processTool._rawOutput.decode().split('\n') + if pattern == None: + pattern = '*' + for line in lines: + if len(line) == 0 or line[0].isspace(): + continue + # 3: wlp4s0: <BROADCAST + dev = line.split(' ')[1][0:-1] + if dev == 'lo': + continue + if fnmatch.fnmatch(dev, pattern): + if rc == None: + rc = dev + elif not rc.startswith('e') and dev.startswith('e'): + rc = dev + elif not rc.startswith('w') and not rc.startswith('e') and dev.startswith('w'): + rc = dev + if rc == None: + self.usage('unknown network interface: ' + pattern) + return rc + + def installNginx(self, argv): + '''Initializes the webserver NGINX. + @param argv: program arguments, e.g. '--well-known=/var/www/letsencrypt' + ''' + root = '/var/www/letsencrypt' + argv, options = self.splitArgsAndOpts(argv) + for opt in options: + if opt.startswith('--well-known='): + root = opt[13:] + else: + self.usage('unknown option: ' + opt) + if self._isRoot: + self._processTool.execute(['/usr/bin/apt-get', '-y', 'install', 'nginx-full', 'ssl-cert', 'ca-certificates'], True) + self._textTool.currentSimpleInsert('/etc/nginx/nginx.conf', 'client_max_body_size', ''' + client_max_body_size 512M; + ## Detect when HTTPS is used + map $scheme $fastcgi_https { + default off; + https on; + } + #client_body_temp_path /space/php_temp; + fastcgi_read_timeout 3600s; + fastcgi_request_buffering off; +''', 'include /etc/nginx/conf.d', False, True) + self.ensureDirectory(root) + self.ensureDirectory(root + os.sep + '.well-known') + fn = self.getTarget('/etc/nginx/snippets', 'letsencrypt.conf') + base.FileHelper.ensureFileExists(fn, createLetsEncryptConf(root)) + service = 'nginx' + if self._isRoot: + self._processTool.execute(['systemctl', 'enable', service], True) + self._processTool.execute(['systemctl', 'start', service], True) + self._processTool.execute(['systemctl', 'status', service], True) + + def installPhp(self, argv): + phpVersion = None if len(argv) <= 0 else argv[0] + packets = 'php-fpm php-cli php-json php-curl php-imap php-gd php-mysql php-xml php-zip php-intl php-imagick ' + packets += 'php-mbstring php-memcached memcached php-xdebug php-igbinary php-msgpack' + if phpVersion != None: + if self._ubuntuVersion != None: + self._processTool.execute(['add-apt-repository', '-y', 'ppa:ondrej/php'], True) + elif self._debianVersion != None: + if not os.path.exists('/etc/apt/sources.list.d/php.list'): + self._processTool.executeScript('''#! /bin/bash +wget -q https://packages.sury.org/php/apt.gpg -O- | sudo apt-key add - +echo "deb https://packages.sury.org/php/ {} main" | sudo tee /etc/apt/sources.list.d/php.list +'''.format(self.debianName())) + self.updateApt(True) + else: + self._logger.error('not DEBIAN, not Ubuntu. I am confused') + return + if phpVersion in ['7.2', '7.3', '7.0', '7.1', '5.6']: + packets = packets.replace('-', str(phpVersion) + '-') + if phpVersion == '5.6': + packets += ' php5.6-mcrypt php5.6-opcache php5.6-readline php5.6-sqlite3' + args = ['/usr/bin/apt-get', '-y', 'install' ] + packets.split(' ') + self._processTool.execute(args, True) + service = 'php{}-fpm'.format('' if phpVersion == None else phpVersion) + self._processTool.execute(['systemctl', 'enable', service], True) + self._processTool.execute(['systemctl', 'start', service], True) + self._processTool.execute(['systemctl', 'status', service], True) + + def jobGitlabRestore(self): + argv = ['/usr/bin/gitlab-rake', 'gitlab:backup:create'] + proc = subprocess.Popen(argv, stderr=subprocess.PIPE) + while True: + line = proc.stderr.readline() + if line == b'': + break + self._logger.error(line.decode()) + + def jobSaveAllDatabases(self): + '''Saves all databases of a mysql system. + ''' + self._logger.log('saving all databases', 2) + dbs = self.allDatabases(True) + user = self._configuration.getString('mysql.user') + password = self._configuration.getString('mysql.code') + if dbs != None: + for db in dbs: + name = db + currentUser = user + currentPassword = password + config = self.findConfig('db', db, 'webapps.d') + if config != None: + path = config.getString('directory') + if path != None: + name = os.path.basename(path) + '_' + db + currentUser = config.getString('user') + currentPassword = config.getString('password') + if currentPassword == None: + currentUser = user + currentPassword = password + if self._verboseLevel >= 3: + self._logger.log('saving db ' + db) + self.saveMysql(name, db, currentUser, currentPassword, self._targetDir) + + def linuxServer(self): + '''Makes the basic action on a linux server. + ''' + self.init(['etc']) + self.init(['dirs']) + packets = 'htop iotop curl tmux git etckeeper bzip2 zip unzip nfs-common nfs-kernel-server nmap rsync sudo apt-transport-https net-tools ntp btrfs-compsize'.split(' ') + self._processTool.execute(['/usr/bin/apt-get', '-y', 'install' ] + packets, True) + appInfo = base.BaseTool.ApplicationInfo('pyboot', None, None, None) + tool = base.BaseTool.InstallTool(appInfo, self._globalOptions) + tool.createSystemDScript('pyboot', 'pyboot', 'root', 'root', 'Starts shell scripts from /etc/pyrshell/boot.d at boot time.') + bootDir = '/etc/pyrshell/boot.d' + self.ensureDirectory(bootDir) + script = self.getTarget('/usr/local/bin', 'pyboot') + base.StringUtils.toFile(script, '''#! /bin/bash +DIR={} +LOG=/var/log/local/boot.log +export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +date "+%Y.%m.%d-%H:%M:%S start" >>$LOG +cd $DIR +for script in *.sh; do + if [ "$script" != "*.sh" ]; then + date "+%Y.%m.%d-%H:%M:%S $script" >>$LOG + ./$script $* + fi +done +date "+%Y.%m.%d-%H:%M:%S end" >>$LOG +systemctl stop pyboot +exit 0 +'''.format(bootDir), self._logger) + os.chmod(script, 0o755) + self._processTool.execute(['/bin/systemctl', 'enable', 'pyboot'], True) + self._processTool.execute(['/bin/systemctl', 'status', 'pyboot'], True) + currentZone = base.StringUtils.fromFile('/etc/timezone').strip() + if currentZone != 'Europe/Berlin': + self._logger.log('=== Curious timezone: {} Please execute: dpkg-reconfigure tzdata'.format(currentZone), 1) + + def network(self, argv): + '''Installs network entities. + net { info | static [ip [interface [gateway]]] | virt-manager netname | name-server [ip] } + info: displays relevant infos + ip: examples: 10.10.10.100 or 100 + interface: a pattern for the interface, e.g. eth0 or ens* Default: the first device given by the system except 'lo' + gateway: Default: the above ip with end number 1 + @param argv: the program arguments, e.g. ['100', 'eth1' ] + ''' + cmd = "info" + if len(argv) > 0: + cmd = argv[0] + argv = argv[1:] + if cmd == 'info': + self._processTool.executeInputOutput(['ip', 'addr', 'show'], None) + lines = self._processTool._rawOutput.decode().split('\n') + device = None + rc = [] + for line in lines: + matcher = re.match(r'\d: (\w+):', line) + if matcher != None: + device = matcher.group(1) + continue + matcher = re.match(r'\s+inet (\S+)/', line) + if matcher != None: + ip = matcher.group(1) + info = '{} {}'.format(device, ip) + rc.append(info) + print(info) + continue + base.BaseTool.setResult(rc) + elif cmd == 'static': + ip = None + iface = None + gateway = None + if len(argv) == 1: + ip = argv[0] + elif len(argv) == 2: + ip = argv[0] + iface = argv[1] + elif len(argv) == 3: + ip = argv[0] + iface = argv[1] + gateway = argv[2] + if ip == None: + ip = '10.10.10.100' + else: + ip = self.expandIp(ip) + iface = self.installNetworkFindInterface(iface) + gateway = self.expandIp(gateway, ip) + config = self.getTarget('/etc/network') + os.sep + 'interfaces' + self._textTool.readCurrent(config, True) + self._textTool.findRegion(r'^(auto|iface) ' + iface, True, r'^(auto|iface)', False, None, True) + count = 3 if ip.startswith('192') else (2 if ip.startswith('172') else 1) + netmask = ('255.' * count + '0.' * (4 - count))[0:-1] + text = '''auto {} +iface {} inet static + address {} + netmask {} + gateway {} + dns-nameservers {} + '''.format(iface, iface, ip, netmask, gateway, gateway) + self._textTool.replaceRegion(text) + self._textTool.writeCurrent() + elif cmd == 'virt-manager': + if len(argv) == 0: + self.usage('missing network name') + else: + netName = argv[0] + argv = argv[1:] + fn = tempfile.gettempdir() + os.sep + netName + '.xml' + base.StringUtils.toFile(fn, '''<network> + <name>{}</name> + <ip address='10.10.10.1' netmask='255.0.0.0'> + <dhcp> + <range start='10.10.10.20' end='10.10.10.99' /> + </dhcp> + </ip> +</network> +'''.format(netName)) + self._processTool.executeInput(['virsh', 'net-define', fn], False, '') + self._processTool.executeInput(['virsh', 'net-autostart', netName], False, '') + self._processTool.executeInput(['virsh', 'net-start', netName], False, '') + self._processTool.executeInput(['virsh', 'list'], True, '') + self._logger.log('''=== for all guests: +virsh GUEST_NAME + +<interface type='network'> + <source network='{}'/> + <model type='virtio'/> <-- This line is optional. +</interface>'''.format(netName)) + elif cmd == 'nameserver': + if len(argv) > 0: + ip = argv[0] + else: + ip = '9.9.9.9' + fn = self.getSource('/etc/network') + os.sep + 'interfaces' + self._textTool.readCurrent(fn, True) + (ix, matcher) = self._textTool.currentFind2(r'\s+gateway\s+(\S+)') + if ix != None: + ip = matcher.group(1) + (ix, matcher) = self._textTool.currentFind2(r'\s+dns-nameservers\s+(\S+)') + if ix != None: + ip = matcher.group(1) + fn = self.getTarget('/etc/systemd') + os.sep + 'resolved.conf' + if self._textTool.readCurrent(fn, True): + if not self._textTool.currentReplace(r'^\s*DNS=\s*', 'DNS=' + ip, + r'^\s*#\s*DNS\s*='): + self.usage('no DNS entry (comment too) found in ' + fn) + else: + self._textTool.writeCurrent() + else: + self.usage('unknown sub command: ' + cmd) + + def readGroups(self, full): + '''Reads the passwd file. + @param full: the filename to read + @result: a dictionary with (name : GroupInfo) pairs + ''' + self._textTool.readCurrent(full, True) + groups = dict() + lastComments = [] + for line in self._textTool._lines: + parts = line.split(':') + if len(parts) < 2: + continue + groups[parts[0]] = GroupInfo(parts[0], parts[2], line) + + for line in self._textTool._lines: + parts = line.split(':') + if not (len(line) > 0 and line[0] == '#'): + if len(lastComments) > 0: + groups[parts[0]]._comments = lastComments + lastComments = [] + else: + parts2 = line[1:].split(':') + if parts2[0] in groups: + groups[parts[0]]._comments = [line] + return groups + + def readPasswd(self, full): + '''Reads the passwd file. + @param full: the filename to read + @result: a dictionary with (name : UserInfo) pairs + ''' + self._textTool.readCurrent(full, True) + users = dict() + lastComments = [] + for line in self._textTool._lines: + parts = line.split(':') + if len(parts) < 2: + continue + users[parts[0]] = UserInfo(parts[0], parts[2], line) + + for line in self._textTool._lines: + parts = line.split(':') + if not (len(line) > 0 and line[0] == '#'): + if len(lastComments) > 0: + users[parts[0]]._comments = lastComments + lastComments = [] + else: + parts2 = line[1:].split(':') + if parts2[0] in users: + users[parts[0]]._comments = [line] + return users + + def readShadow(self, full): + '''Reads the shadow file. + @param full: the filename to read + @result: a pair of dictionaries, first is (name : line), the second (name: comments) + ''' + self._textTool.readCurrent(full, True) + linesByName = dict() + commentsByName = dict() + lastComments = [] + for line in self._textTool._lines: + parts = line.split(':') + if len(parts) < 2: + lastComments.append(line) + else: + if len(lastComments) > 0: + commentsByName[parts[0]] = '\n'.join(lastComments) + '\n' + lastComments = [] + else: + commentsByName[parts[0]] = '' + linesByName[parts[0]] = line + return (linesByName, commentsByName) + + def reconfigure(self, argv): + '''Change configuration files given from the internal configuration. + @param argv: arguments, e.g. ['php', '7.1'] + ''' + if len(argv) < 1: + self.usage('missing <what>') + else: + what = argv[0] + argv = argv[1:] + if what == 'local-bin': + self._textTool.currentSimpleInsert('/etc/sysctl.conf', '^vm.swappiness', 'vm.swappiness = 20') + elif what == 'letsencrypt': + self.reconfigureLetsencrypt(argv) + elif what == 'php': + self.reconfigurePhp(argv) + else: + self.usage('unknown <what>: ' + what) + + def reconfigureLetsencrypt(self, argv): + '''Switches letsencrypt certificate and private certificate. + @param argv: program arguments, e.g. ['on', 'abc.infeos.de'] + ''' + argv, opts = self.splitArgsAndOpts(argv) + all = False + for opt in opts: + if opt == '--all': + all = True + else: + self.usage('unknown option ' + opt) + if len(argv) < 1: + self.usage('missing arguments') + elif argv[0] != 'on' and argv[0] != 'off': + self.usage('wrong mode: use "on" or "off" not ' + argv[0]) + elif len(argv) == 1 and all: + mode = argv[0] + subdir = self.getTarget('/etc/nginx/sites-enabled', '') + nodes = os.listdir(subdir) + for node in nodes: + if node != 'default': + self.reconfigureLetsencrypt([mode, node]) + else: + mode = argv[0] + domain = argv[1] + domainRaw = domain + fn = self.getTarget('/etc/nginx/sites-enabled', domain) + if not os.path.exists(fn): + if domain.startswith('www.'): + domainRaw = domain[4:] + fn = self.getTarget('/etc/nginx/sites-enabled', domainRaw) + else: + fn = self.getTarget('/etc/nginx/sites-enabled', 'www.' + domain) + if not os.path.exists(fn): + self.usage('unknown domain: ' + domain) + fn = None + if fn != None: + self._textTool.readCurrent(fn, True) + regComment = re.compile(r'^\s*#') + changedLets = 0 + changedPrivate = 0 + for ix in range(len(self._textTool._lines)): + line = self._textTool._lines[ix] + if line.find('ssl_certificate') >= 0: + hasComment = regComment.match(line) != None + isLetsencrypt = self._textTool._lines[ix].find('live/latest') >= 0 + if mode == 'on': + if hasComment and isLetsencrypt: + self._textTool._lines[ix] = line.replace('#', '') + changedLets += 1 + elif not hasComment and not isLetsencrypt: + changedPrivate += 1 + self._textTool._lines[ix] = '#' + line + elif hasComment and not isLetsencrypt: + changedPrivate += 1 + self._textTool._lines[ix] = line.replace('#', '') + elif not hasComment and isLetsencrypt: + changedLets += 1 + self._textTool._lines[ix] = '#' + line + if changedLets + changedPrivate > 0: + self._logger.log('changed lines: letsencrypt: {} private: {}'.format(changedLets, changedPrivate), 2) + line = self._textTool.writeCurrent(fn, True) + else: + self._logger.log('nothing to change', 2) + + def reconfigureLetsencryptAdapt(self): + '''Adapts the link to the latest certificate. + ''' + baseDir = self.getTarget('/etc/letsencrypt', 'live') + nodes = os.listdir(baseDir) + # search the youngest directory + youngest = 0 + name = None + for node in nodes: + full = baseDir + os.sep + node + if os.path.isdir(full): + current = os.path.getmtime(full) + if current > youngest: + youngest = current + name = node + if name == None: + self._logger.error('no subdir in ' + baseDir) + else: + latest = baseDir + os.sep + 'latest' + if not os.path.islink(latest): + self._logger.error('not a link: ' + latest) + else: + link = os.readlink(latest) + if link == name: + self._logger.log('link is correct: ' + link) + else: + self._logger.log('exchanging link from {} to {}'.format(link, name)) + os.unlink(latest) + os.symlink(name, latest) + + def reconfigurePhp(self, argv): + '''Replace configuration data given by the internal configuration. + @param argv: the argument vector, e.g. ['7.1'] + ''' + argv = argv[1:] + variables = self._textTool.findVariables('php.', self._configuration) + if len(argv) == 0: + versions = os.listdir('/etc/php') + versions.sort() + version = versions[-1] + else: + version = argv[0] + argv = argv[1:] + if not re.match(r'\d+\.\d+', version): + self.usage('invalid version: ' + version) + fnConfig = '/etc/php/{}/fpm/php.ini'.format(version) + if not os.path.exists(fnConfig): + self.usage('missing {}: is version {} installed?'.format(fnConfig, version)) + else: + nodes = os.listdir('/etc/php/{}'.format(version)) + for node in nodes: + fnConfig = '/etc/php/{}/{}/php.ini'.format(version, node) + if os.path.exists(fnConfig): + missingDebug = False + configuration = base.StringUtils.fromFile(fnConfig).split('\n') + if not base.StringUtils.arrayContains(configuration, 'xdebug.remote_enabled'): + missingDebug = True + configuration += ''' +; Ist normalerweise in xdebug-spezifischer Konfiguration, z.B. mods.d/20-xdebug +;zend_extension="/usr/lib/php/20160303/xdebug.so" +xdebug.remote_port=9000 +xdebug.remote_enable=Off +xdebug.remote_handler=dbgp +xdebug.remote_host=127.0.0.1 +;xdebug.remote_connect_back=On +;xdebug.remote_log=/var/log/xdebug.log +xdebug.remote_autostart=1 +'''.split('\n') + content = '\n'.join(self._textTool.adaptVariables(variables, configuration)) + if missingDebug or self._textTool._hits > 0: + self.createBackup(fnConfig) + base.StringUtils.toFile(fnConfig, content) + if self._verboseLevel >= 2: + self._logger.log('{}: {} variable(s) changed{}'.format( + fnConfig, self._textTool._hits, + '' if not missingDebug else '\nxdebug setup added')) + + def remove(self, argv): + '''Removes some entities. + @param argv: program arguments, e.g. ['web-app', 'www.huber.de'] + ''' + if len(argv) < 1: + self.usage('missing argument(s)') + else: + mode = argv[0] + argv = argv[1:] + if mode == 'webapp': + if len(argv) < 1: + self.usage('missing argument(s)') + else: + domain = argv[0] + argv = argv[1:] + self.removeWebApp(domain, argv[1] if len(argv) > 0 else None) + else: + self.usage('unknown entity: ' + mode) + + def removeWebApp(self, domain, rootPasswd): + '''Removes a web application (described in /etc/pyrshell/webapps.d/<domain>.conf + @param domain: the id of the application + ''' + fnConfiguration = '/etc/pyrshell/webapps.d/{}.conf'.format(domain) + if not os.path.exists(fnConfiguration): + self.usage('missing ' + fnConfiguration) + else: + config = base.JavaConfig.JavaConfig(fnConfiguration, self._logger) + db = config.getString('db') + user = config.getString('user') + code = config.getString('password') + directory = config.getString('directory') + if db == None or user == None or code == None or directory == None: + self.usage('missing needed entries in ' + fnConfiguration) + elif not os.path.isdir(directory): + self.usage('not a directory: ' + directory) + else: + self.createBackup(directory) + self._logger.log('removing directory tree {}...'.format(directory), 1) + shutil.rmtree(directory, True) + fn = '{}.{}.sql'.format(domain, time.strftime('%Y.%m.%d.%H_%M_%S')) + self._processTool.execute(['mysqldump', '--default-character-set=utf8mb4', '--single-transaction', '-u', user, '-p' + code, db, '>' + fn], True, 'shell') + self._processTool.execute(['gzip', fn], True) + if rootPasswd != None: + user = 'root' + code = rootPasswd + self._processTool.execute(['mysqladmin', '-u', user, '-p' + code, 'drop', db], True) + self.createBackup(fnConfiguration) + self.ensureFileDoesNotExist(fnConfiguration) + + def restore(self, argv): + '''Fetch data from backup and installs them. + @param argv: the program arguments, e.g. ['clone'] + ''' + if len(argv) < 1: + self.usage('missing <what>') + else: + what = argv[0] + argv = argv[1:] + if what == 'clone': + self.restoreClone(argv) + elif what == 'etc': + dirToMerge = self.getTarget('/', 'etc2') + self.restoreMergeUsersAndGroups(dirToMerge) + self.restorePyRShell() + if os.path.isdir('/etc/nginx'): + self.restoreNginx() + elif what == 'pyrshell': + self.restorePyRShell() + elif what == 'nginx': + self.restoreNginx() + else: + self.usage('unknown <what>: ' + what) + + def restoreClone(self, argv): + '''Clones an archive into a "sibling directory" of a target. + Note: a sibling directory of a target has the same parent as the target itself. + @param argv: the program arguments, e.g. ['usr=local=bin.tgz', '/usr/local/bin' + ''' + if len(argv) < 2: + self.usage('missing arguments') + else: + archive = argv[0] + target = argv[1] + target2 = target + '2' + if not os.path.isdir(target): + self.usage('<target> is not a directory: ' + target) + elif os.path.exists(target2): + self.usage('clone already exists: ' + target2) + else: + parent = os.path.dirname(target) + tempDir = parent + os.sep + 'tmp.{}'.format(int(time.time())) + self.ensureDirectory(tempDir) + if archive.endswith('.zip'): + argv = ['/usr/bin/unzip', '-q', archive, '-d', tempDir] + self._processHelper.execute(argv, True) + elif archive.endswith('.tgz'): + argv = ['/bin/tar', 'xzf', archive, '--directory=' + tempDir] + self._processHelper.execute(argv, True) + nodes = os.listdir(tempDir) + if len(nodes) == 1 and os.path.isdir(tempDir + os.sep + nodes[0]): + os.rename(tempDir + os.sep + nodes[0], target2) + self.ensureFileDoesNotExist(tempDir) + else: + os.rename(tempDir, target2) + self._logger.log('created: {} (from {}'.format(target2, archive)) + + def restoreCopyFile(self, source, target): + '''Copies one file from a source to a target. + @param source: the source filename + @param target: the target filename + ''' + self.nearBackup(target, True) + if not os.path.exists(source): + self._logger.error('missing source: ' + source) + else: + self._logger.log('copying {} -> {}'.format(source, target)) + self.ensureDirectory(os.path.dirname(target)) + shutil.copy2(source, target) + + def restoreDirectoryByTar(self, archive, target, opts=None, tempDir=False, subdir=None, clearTarget=True): + '''Restores a directory from a tar archive. + @param archive: archive name + @param target: target directory + @param opt: None or an array of options like '-exclude=<pattern>' + @param tempDir: True: usage of a temporary directory + @param subdir: None or a subdirectory in the TAR archive + @param clearTarget: True: the target directory will be cleared before extracting + ''' + if not os.path.exists(archive): + self._logger.error('missing tar archive {}'.format(archive)) + elif not os.path.exists(target): + self._logger.error('missing target dir: ' + target) + elif not os.path.isdir(target): + self._logger.error('not a directory: ' + target) + else: + start = time.time() + tempBase = self.ensureDirectory('/tmp/restoretool') + if clearTarget: + self.clearDirectory(target) + trg = target if not tempBase else '/tmp/restoretool' + os.sep + 'trg' + if tempDir: + base.BaseTool.BasicStatics.ensureDirectory(trg, self._logger) + argv = ['/bin/tar', 'xzf', archive, '--directory=' + trg] + if subdir != None: + argv.append('./' + subdir) + if opts != None: + argv += opts + self._processTool.execute(argv, self._verboseLevel >= 3) + # os.chdir(oldDir) + if tempDir: + if subdir != None: + trg += os.sep + subdir + self._processTool.execute(['/usr/bin/rsync', '-a', trg + '/', target], self._verboseLevel >= 3) + self.logFile(archive, '%f: %s %t restored in %r', start) + + def restoreDirectoryByZip(self, archive, target, opts): + '''Restores a directory from a zip archive. + @param archive: archive name + @param target: target directory + @param opt: None or an array of options like '-exclude=<pattern>' + ''' + if not os.path.exists(archive): + self._logger.error('missing zip archive {}'.format(archive)) + elif not os.path.exists(target): + self._logger.error('missing target dir: ' + target) + elif not os.path.isdir(target): + self._logger.error('not a directory: ' + target) + else: + start = time.time() + self.clearDirectory(target) + argv = ['/usr/bin/unzip', '-q', archive, '-d', target] + self._processHelper.execute(argv, True) + # os.chdir(oldDir) + self.logFile(archive, '%f: %s %t restored in %r', start) + + def restoreFromBackup(self, mode): + '''Does the backup process controlled by configuration files. + @param mode: 'dayly', 'weekly' ... + ''' + logFile = self._configuration.getString('log.file') + if logFile != None and (not hasattr(self._logger, '_logfile') or os.path.abspath(logFile) != os.path.abspath(self._logger._logfile)): + logger2 = base.Logger.Logger(logFile, self._logger._verbose >= 0) + logger2.transferErrors(self._logger) + self._logger = logger2 + logFile2 = self._configuration.getString('log.mirror') + if logFile2 != None: + logger3 = base.Logger.Logger(logFile, False) + self._logger.setMirror(logger3) + self._mode = mode + configuration = self._backupConfiguration + self._baseTargetDir = configuration.getString('target.path') + self._targetDir = self._baseTargetDir + try: + if configuration.getString('job.' + mode) == None: + self._logger.error('missing job.{:s} in {:s} Wrong mode?'.format(mode, configuration._filename)) + elif self._targetDir == None: + self._logger.error('missing "{:s}" in {:s}'.format(self._targetDir, configuration._filename)) + elif not os.path.isdir(self._targetDir): + self._logger.error('{:s} is not a directory. See "target.path" in '.format(self._targetDir, configuration._filename)) + else: + relPath = self.relativeRestorePath(mode) + self._targetDir += relPath + if self.ensureDirectory(self._targetDir) != None: + if mode == 'dayly' or mode == 'weekly' or mode == 'monthly': + linkTarget = os.path.dirname(self._targetDir) + os.sep + 'current' + if os.path.exists(linkTarget): + os.unlink(linkTarget) + os.symlink(os.path.basename(self._targetDir), linkTarget) + statement = configuration.getString('job.' + mode); + if statement == None: + self._logger.error('missing job.' + mode + ' in ' + configuration._filename); + else: + self.doJob(configuration, statement) + except Exception as exc: + self._logger.error('backup aborted with exception: ' + str(exc)) + traceback.print_exc() + subject = None + if self._logger._errors > 0: + subject = 'Restore {:s} failed on {:s}'.format(mode, configuration.getString('location', base.BaseTool.BasicStatics.hostname(True))) + elif configuration.getString('send.always', 'False').startswith('T'): + subject = 'Restore {:s} finished on {:s}'.format(mode, configuration.getString('location', base.BaseTool.BasicStatics.hostname(True))) + if subject != None: + recipient = configuration.getString('admin.email') + now = datetime.datetime.now() + body = now.strftime('%Y.%m.%d %H:%M:%S') + ' Restore {:s} finished with {:d} error(s)\n\n'.format(mode, self._logger._errors) + body += '\n'.join(self._logger._firstErrors) + sender = configuration.getString('smtp.sender'), + host = configuration.getString('smtp.host') + port = configuration.getString('smtp.port') + user = configuration.getString('smtp.user') + code = configuration.getString('smtp.code') + value = configuration.getString('smtp.tls') + withTls = value != None and value.lower().startswith('t') + if recipient and sender and host and port and user and code: + net.EMail.sendStatusEmail(recipient, subject, body, sender, host, port, user, code, withTls, self._logger) + + def restoreMergeUsersAndGroups(self, dirMerge): + '''Merges two passwd (and shadow) files into one. + Priority has the file /etc/passwd (/etc/shadow, /etc/groups). + That means that a user existing in both files will taken from the first (/etc/passwd, /etc/shadow /etc/groups) + @param dirMerge: the directory containing 'etc2' which contains the 2end files to merge + ''' + dirSource = self.getSource('/', 'etc') + users = self.readPasswd(dirSource + os.sep + 'passwd') + contentPasswd = '\n'.join(self._textTool._lines) + '\n' + contentShadow = base.StringUtils.fromFile(dirSource + os.sep + 'shadow') + '\n' + groups = self.readGroups(dirSource + os.sep + 'group') + contentGroups = '\n'.join(self._textTool._lines) + '\n' + users2 = self.readPasswd(dirMerge + os.sep + 'passwd') + (linesByName, commentsByName) = self.readShadow(dirMerge + os.sep + 'shadow') + groups2 = self.readGroups(dirMerge + os.sep + 'group') + keys2 = users2.keys() + oldKeys = users.keys() + addendumPasswd = '' + addendumShadow = '' + addendumGroups = '' + keys2 = sorted(keys2) + for user in keys2: + if user not in oldKeys: + addendumPasswd += users2[user].asText().strip() + '\n' + if user in commentsByName.keys(): + addendumShadow += commentsByName[user].strip() + '\n' + if user in linesByName.keys(): + addendumShadow += linesByName[user].strip() + '\n' + keys2 = groups2.keys() + oldKeys = groups.keys() + for group in keys2: + if group not in oldKeys: + addendumGroups += groups2[group].asText() + dirTarget = self.getTarget('/', 'etc') + if dirTarget == '/etc': + self.createBackup(dirTarget + os.sep + 'passwd', 'etc') + self.createBackup(dirTarget + os.sep + 'shadow', 'etc') + self.createBackup(dirTarget + os.sep + 'group', 'etc') + base.StringUtils.toFile(dirTarget + os.sep + 'passwd', (contentPasswd + addendumPasswd).replace('\n\n', '\n')) + base.StringUtils.toFile(dirTarget + os.sep + 'shadow', (contentShadow + addendumShadow).replace('\n\n', '\n')) + base.StringUtils.toFile(dirTarget + os.sep + 'group', (contentGroups + addendumGroups).replace('\n\n', '\n')) + + def restoreMysql(self, name, db, user, password, target): + '''Restores a database previously stored by mysqldump. + @param dumpFile: containing the SQL statements to restore the db + @param db: database name + @param user: database use + @param password: database password + ''' + start = time.time() + oldFilter = self._logger._errorFilter + self._logger.setErrorFilter('mysqldump: [Warning] Using a password', True) + argv = ['/usr/bin/mysql', '-u', user, '-p' + password, '--single-transaction', db] + outputFile = target + os.sep + name + '.sql.gz' + errorFile = tempfile.gettempdir() + os.sep + 'backuptool.err.txt' + with gzip.open(outputFile, 'wb') as fpGZip, open(errorFile, 'w') as fpError: + proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=fpError.fileno()) + while True: + line = proc.stdout.readline() + if line == b'': + break + fpGZip.write(line) + self.errorFileToLog(errorFile) + self._logger.setErrorFilter(oldFilter) + self.logFile(outputFile, '%f: %s %t created in %r', start) + + def restoreNginx(self): + '''Restores the /etc/nginx-directory + ''' + source = self.getSource('/', 'etc2/nginx') + target = self.getTarget('/', 'etc/nginx') + if not os.path.isdir(source): + self.usage('missing shadow directory {}'.format(source)) + else: + self.ensureDirectory(target) + base.FileHelper.copyDirectory(source + os.sep + 'sites-available', target + os.sep + 'sites-available', 'update', self._logger, self._verboseLevel) + base.FileHelper.copyDirectory(source + os.sep + 'sites-enabled', target + os.sep + 'sites-enabled', 'update', self._logger, self._verboseLevel) + fn = source + os.sep + 'snippets/letsencrypt.conf' + if os.path.exists(fn): + self.restoreCopyFile(fn, target + os.sep + 'snippets/letsencrypt.conf') + self.restoreCopyFile(source + os.sep + 'nginx.conf', target + os.sep + 'nginx.conf') + source = self.getSource('/', 'etc2/ssl') + target = self.getTarget('/', 'etc/ssl') + if not os.path.isdir(source): + self.usage('missing shadow directory {}'.format(source)) + else: + self.ensureDirectory(target) + base.FileHelper.copyDirectory(source + os.sep + 'certs', target + os.sep + 'certs', 'update', self._logger, self._verboseLevel) + base.FileHelper.copyDirectory(source + os.sep + 'private', target + os.sep + 'private', 'update', self._logger, self._verboseLevel) + if os.path.isdir('/etc/letsencrypt/live'): + if not os.path.exists('/etc/letsencrypt/live/latest'): + self._logger('=== missing /etc/letsencrypt/live/latest') + + def restorePyRShell(self): + '''Restores the /etc/pyrshell-directory + ''' + source = self.getSource('/', 'etc2') + target = self.getTarget('/etc', 'pyrshell') + if not os.path.isdir(source): + self.usage('missing shadow directory {}'.format(source)) + else: + self.ensureDirectory(target) + base.FileHelper.copyDirectory(source + os.sep + 'pyrshell', target, 'clear', self._logger, self._verboseLevel) + + def storage(self, argv): + '''Executes the storage command. + @param argv: the arguments, e.g. ['ftp-service', '10225'] + ''' + if len(argv) < 1: + self.usage('missing <what>') + else: + what = argv[0] + argv = argv[1:] + if what == 'autofs': + self.storageAutoFs(argv) + elif what == 'disks': + self.storageDisks(argv) + elif what == 'ftp-service': + self.storageFtpService(argv) + elif what == 'nfs-share': + self.storageNFSShare(argv) + elif what == 'sftp-service': + self.storageSFtpService(argv) + elif what == 'lvm': + self.storageLvm(argv) + else: + self.usage('unknown <what>: ' + what) + + def storageDisks(self, argv): + '''Informs about disks. + @param argv: the program arguments, e.g. ['sd[bc]'] + ''' + pattern = None if len(argv) == 0 else argv[0] + lines = self.storageDiskList(pattern) + self._output = lines + base.BaseTool.setResult(lines) + base.BaseTool.setLatestTool(self) + print('\n'.join(lines)) + + def storageDiskList(self, pattern): + '''Finds disk info matching a pattern. + @param pattern: None or a regular expression. Only matching lines will be displayed + @return: an array of info lines, e.g. ['caribou-home', 'UUID="950924c0-a0cd-4bae-8b80-13ba5d3e7cbd"', 'TYPE="ext4"'] + ''' + def extract(prefix, parts): + rc = '' + for part in parts: + if part.startswith(prefix): + rc = ' ' + part + parts.remove(part) + break + return rc + + regExpr = None if pattern == None else re.compile(pattern) + lines = self._processHelper.executeInputOutput(['/sbin/blkid'], None, False) + rc = [] + for line in lines: + if regExpr == None or regExpr.search(line): + parts = line.split(' ') + info = parts[0].replace('/dev/', '').replace('mapper/', '') + parts = parts[1:] + info += extract('UUID=', parts) + info += extract('TYPE=', parts) + info += ' ' + ' '.join(parts) + rc.append(info) + return rc + + def storageAutoFs(self, argv): + '''Installs and configures automatic mounting. + @param argv: program arguments, e.g. ['bigtoy', '/', 'nfs:10101:/media/data/www'] + <server> <mount-base> <source1> [<source2 ... ] + ''' + (argv, options) = self.splitArgsAndOpts(argv) + autoSelect = None + for opt in options: + if opt.startswith('--auto-select='): + autoSelect = opt[14:] + else: + self.usage('unknown option: ' + opt) + if self._isRoot: + self._processTool.execute(['/usr/bin/apt-get', '-y', 'install', 'autofs'], True) + self._textTool.readCurrent(self.getTarget('/etc', 'auto.master'), False) + if len(argv) < 3: + self.usage('too few arguments') + else: + name = argv[0] + if re.match(r'^[-.\w]+$', name) == None: + self.usage('wrong characters in <server>: only letters+digits+underline+dot allowed') + else: + fnConfig = self.getTarget('/etc', 'auto.') + name + mountBase = argv[1] + if mountBase == '/': + mountBase = '/-' + argv = argv[2:] + no = 0 + for source in argv: + no += 1 + parts = source.split('|') + if len(parts) != 4: + self.usage('missing "|" (3 times) in:' + source) + else: + (srcType, addr, mountPoint, option) = parts + if no == 1: + self._textTool.currentReplace(r'{}\s+{}'.format(mountBase, fnConfig), mountBase + '\t' + fnConfig) + self._textTool.writeCurrent() + self._textTool.readCurrent(fnConfig, False) + if srcType == 'nfs': + # /home/wworker/jail/www -fstype=nfs4,rw,retry=0 localhost:/var/www + if option == '': + option = 'rw' + if addr.find(':/') < 0: + self.usage('wrong syntax of addr (ip:directory expected): ' + addr) + self._textTool.currentReplace(r'{}\s+\S+\s+{}'.format(mountPoint, addr), + '{}\t-fstype=nfs4,{},retry=0\t{}'.format(mountPoint, option, addr)) + elif srcType == 'disk': + if option == '': + option = 'rw' + fsType = 'ext4' + uuid = addr + if autoSelect != None: + list = self.storageDiskList(autoSelect) + if len(list) == 0: + self.usage('autoselection: nothing matches ' + autoSelect) + elif len(list) > 1: + self.usage('autoselection: more than one match:\n' + '\n'.join(list)) + else: + infos = list[0].split(' ') + if len(infos) > 1 and infos[1].startswith('UUID="'): + uuid = infos[1][6:-1] + else: + self.usage('autoselection: missing UUID: ' + list[0]) + if len(infos) > 2 and infos[2].startswith('TYPE="'): + fsType = infos[2][6:-1] + # UUID -fstype=ext4,rw,gid=1000,umask=002 :/dev/disk + self._textTool.currentReplace(r'{}\s+\S+'.format(uuid), + '{}\t-fstype={},{},umask=002\t:{}'.format(uuid, fsType, option, mountPoint)) + else: + self.usage('unknown source type: {} in {}'.format(srcType, source)) + self._textTool.writeCurrent() + if self._isRoot: + self._processHelper.execute(['/bin/systemctl', 'reload', 'autofs'], True) + + def storageFtpService(self, argv): + '''Installs the ftp server proftp. + Arguments: [servername [port [hostname]] + @param argv: the program arguments, e.g. ['Backup for Wordpress', '58773', 'sky.infeos.de'] + ''' + hostname = servername = None + port = 22 + if len(argv) > 0: + servername = argv[0] + elif len(argv) > 1: + port = self.integerArgument(argv[1], 22) + else: + hostname = argv[2] + if hostname == None: + hostname = base.BaseTool.BasicStatics.hostname(True) + if servername == None: + servername = hostname + packets = 'proftpd-basic'.split(' ') + self._processTool.execute(['/usr/bin/apt-get', '-y', 'install'] + packets, True) + text = '''<VirtualHost {}> + ServerName "{}" + MaxClients 20 + MaxLoginAttempts 1 + Port {} + # DeferWelcome prevents proftpd from displaying the servername + # until a client has authenticated. + DeferWelcome on + + # Limit normal user logins, because we only want to allow + # guest logins. + <Limit LOGIN> + DenyGroup !www-data + </Limit> +</VirtualHost> +'''.format(hostname, servername, port) + fn = self.getTarget('/etc/proftpd/proftpd.conf') + self._textTool.readCurrent(fn, True) + firstLine = text.split('\n')[0] + self._textTool.findRegion(firstLine, True, '<Virtual', True) + self._textTool.replaceRegion(text, r'</Limit>') + + def storageLvm(self, argv): + '''Installs an Locical Volume. + Steps: + create the LVM + create the mountpoint + create the automount interface for mounting + @param argv: the program arguments, e.g. ['VGSafe', 'LVBackup' '32G', '/media/safe/backup'] + ''' + (argv, options) = self.splitArgsAndOpts(argv) + for opt in options: + if True: + pass + else: + self.usage('unknown option: ' + opt) + if len(argv) < 4: + self.usage('too few arguments') + else: + volGroup = argv[0] + logVolume = argv[1] + size = argv[2] + mountPoint = argv[3] + device = '/dev/{}/{}'.format(volGroup, logVolume) + if not os.path.exists('/dev/' + volGroup): + self.usage('volume group not found: ' + volGroup) + elif re.match('^[-\w._+]+$', logVolume): + self.usage('illegal characters in logical volume: ' + logVolume) + elif re.match(r'^\d+[gGtTmM$', size) == None: + self.usage('illegal size: {} example: 100G'.format(size)) + elif os.path.exists(device): + self.usage('logical volume already exists: ' + logVolume) + else: + self.ensureDirectory(mountPoint) + self._processHelper.execute(['lvcreate', '--name=' + logVolume, '--size=' + size, volGroup], True) + key = 'vg.' + volGroup + fn = '/etc/auto.' + key + base.FileHelper.ensureFileExists(fn, '', self._logger) + appl.TextTool.TextTool.replaceOrInsert(['^' + device, '{}\t-fstype=ext2\t:{}'.format(device, mountPoint), '--create-if-missing', '--max-depth=0']) + master = '/etc/auto.master' + if os.path.exists(master): + self._logger.error('missing ' + master) + else: + appl.TextTool.TextTool.replaceOrInsert(['.*' + fn, '/-\t' + fn, '--create-if-missing', '--max-depth=0']) + self._processHelper.execute(['systemctl', 'reload', 'autofs'], True) + + def storageSFtpService(self, argv): + '''Installs an SecureFileTransferProtocol service. + Steps: + create the user if it does not exist + add user to group www-data and user www-data to group <user> + create the jail for chroot + create the mount points inside the jail remember the nodes are unique + configure and inform the ssh daemon + define the mount provider (must be chroot proof we use nfs) + define the autofs mounts + @param argv: the program arguments, e.g. ['jonny', '/var/www', '/srv/www' '--force'] + ''' + (argv, options) = self.splitArgsAndOpts(argv) + force = False + for opt in options: + if opt == '-f' or opt == '--force': + force = True + else: + self.usage('unknown option: ' + opt) + if len(argv) < 2: + self.usage('too few arguments') + else: + user = argv[0] + if re.match(r'^[\w]+$', user) == None: + self.usage('invalid characters in username: ' + user) + name = 'sftp_' + user + if not force and ('auto.' + name) in base.BaseTool.BasicStatics.findFiles('/etc', 'auto.*', False): + self.usage('the autofs name {} already exists. Use --force to allow that'.format(name)) + directories = argv[1:] + # we need unique nodes (for mount directories inside jail): + nodes = dict() + for item in directories: + node = os.path.basename(item) + if not os.path.isdir(item): + self.usage('not a directory: ' + item) + node2 = node + no = 0 + while node2 in nodes.values(): + no += 1 + node2 = node + str(no) + nodes[item] = node2 + # create the user and group if it does not exist: + base.BaseTool.BasicStatics.ensureUserAndGroup(user, user, self._logger) + # add user to group www-data and user www-data to group <user>: + self._processHelper.execute(['usermod', '-a', '-G', 'www-data', user], True) + self._processHelper.execute(['usermod', '-a', '-G', user, 'www-data'], True) + # create the jail for chroot: + jailDir = '/home/jails/' + user + self.ensureDirectory(jailDir, 0o755) + self.ensureSymbolicLink(jailDir, '/home/{}/jail'.format(user), True) + # create the mount points inside the jail: remember: the nodes are unique + for item in directories: + self.ensureDirectory(jailDir + os.sep + nodes[item], 0o775) + # configure and inform the ssh daemon: + config = self.getTarget('/etc/ssh', 'sshd_config') + self._textTool.readCurrent(config, True) + self._textTool.findRegion(r'^match User ' + user, True, r'^match', False) + self._textTool.replaceRegion('''match User {} + ChrootDirectory {} + ForceCommand internal-sftp + AllowTcpForwarding no +'''.format(user, jailDir)) + self._textTool.writeCurrent() + self._processHelper.execute(['/bin/systemctl', 'reload', 'sshd'], True) + # define the mount provider (must be chroot proof: we use nfs) + argv2 = [] + argv3 = [name, '/'] + for item in directories: + argv2.append('{}:localhost:rw'.format(item)) + argv3.append('nfs|localhost:{}|{}/{}|rw'.format(item, jailDir, nodes[item])) + self.storageNFSShare(argv2) + # define the autofs mounts + self.storageAutoFs(argv3) + self._logger.log('= do not forget to set password for ' + user, 1) + + def installLetsencrypt(self): + '''Prepare the system to make letsencrypt certificates. + ''' + packets = 'python3-acme python3-certbot python3-mock python3-openssl python3-pkg-resources python3-pyparsing python3-zope.interface'.split(' ') + self._processTool.execute(['/usr/bin/apt-get', '-y', 'install'] + packets, True) + base.BaseTool.BasicStatics.ensureDirectory('/var/www/letsencrypt/.well-known/acme-challenge', self._logger) + base.StringUtils.toFile('/var/www/letsencrypt/.well-known//acme-challenge/hi.txt', 'Hi') + if os.path.isdir('/usr/share/nginx/html') and not os.path.islink('/usr/share/nginx/html/.well-known'): + self._logger.log('creating link /usr/share/nginx/html/.well-known => /var/www/letsencrypt/.well-known', 1) + os.symlink('/var/www/letsencrypt/.well-known', '/usr/share/nginx/html/.well-known') + + def storageNFSShare(self, argv): + '''Initializes a NFS share on the server side. + @param argv: the arguments, e.g. ['/media/tmp:10.10.10.104'] + ''' + if len(argv) < 1: + self.usage('missing arguments') + else: + if not base.BaseTool.unitTestIsActive(): + self._processTool.execute(['apt-get', '-y', 'install', 'nfs-kernel-server'], True) + self._textTool.readCurrent('/etc/exports', False) + for arg in argv: + parts = arg.split(':') + if len(parts) != 3: + self._logger.error('wrong syntax of addr: use <path>:<ip>:<option> ' + arg) + else: + (path, ip, option) = parts + if not os.path.isdir(path): + self._logger.error('not a direcory: ' + path) + if re.match(r'^ro|rw$', option) == None: + self._logger.error('unknown option: ' + option) + option = 'rw' + self._textTool.currentReplace(r'^{}\s+{}'.format(path, ip), + '{}\t{}({},async,no_root_squash,subtree_check)'.format(path, ip, option)) + self._textTool.writeCurrent(None) + self._processTool.execute(['/usr/sbin/exportfs', '-r'], True) + self._processTool.execute(['/usr/sbin/exportfs', '-a'], True) + self._processTool.execute(['/usr/sbin/exportfs', '-v'], True) + + def synchronize(self, source, target, deleteToo): + '''Synchronizes a directory from another. + Copies the newer or not existing files from the source to the target, recursive + @param source: the source directory + @param target: the target directory + @param deleteToo: files in target but not existing in source will be deleted + ''' + + def logError(arg): + self._logger.error('error while removing directory {:s}: {:s}'.format(fullTarget, arg)) + + sourceNodes = os.listdir(source) + targetNodes = os.listdir(target) + dirs = [] + for node in sourceNodes: + fullSource = source + os.sep + node + fullTarget = target + os.sep + node + if not base.FileHelper.distinctPaths(fullSource, fullTarget): + self._logger('nested directories: {} / {} [{} / {}]'.format( + fullSource, fullTarget, os.path.realpath(fullSource), os.path.realpath(fullTarget))) + continue + doCopy = False + infoSource = os.lstat(fullSource) + if stat.S_ISDIR(infoSource.st_mode): + dirs.append(node) + else: + try: + infoTarget = os.lstat(fullTarget) + doCopy = infoTarget.st_mtime > infoSource.st_mtime + except OSError: + doCopy = True + if doCopy: + self._logger.log('copying {:s}'.format(fullSource), 3) + shutil.copy(fullSource, fullTarget) + if deleteToo: + if node in targetNodes: + targetNodes.remove(node) + if deleteToo: + for node in targetNodes: + fullTarget = target + os.sep + node + self._logger.log('deleting {:s}'.format(fullTarget), 3) + if os.path.isdir(fullTarget): + shutil.rmtree(fullTarget) + else: + os.unlink(fullTarget) + for node in dirs: + trgDir = target + os.sep + node + if os.path.exists(trgDir): + if not os.path.isdir(trgDir): + os.unlink(trgDir) + os.mkdir(trgDir) + else: + os.mkdir(trgDir) + self.synchronize(source + os.sep + node, trgDir, deleteToo) + + def systemInfo(self, target): + '''Assembles interesting informations about the current linux system. + Manually installed packets + disk devices + mounts + @param target: the target directory + ''' + start = time.time() + # zcat /var/log/apt/history.log.*.gz | cat - /var/log/apt/history.log | grep -Po '^Commandline: apt-get install (?!.*--reinstall)\K.*' > $fnManuallyInstalled + path = '/var/log/apt' + nodes = os.listdir(path) + packets = [] + for node in nodes: + if node.startswith('history.log') and node.endswith('.gz'): + full = path + os.sep + node + with gzip.open(full, 'r') as fp: + for line in fp: + if line.startswith(b'Commandline: apt-get install'): + packets += line[29:].decode().strip().replace(' ', ' ').replace(' ', ' ').split(' ') + with open('/var/log/apt/history.log', 'r') as fp: + for line in fp: + if line.startswith('Commandline: apt-get install'): + packets += line[29:].strip().replace(' ', ' ').replace(' ', ' ').split(' ') + + mounts = [] + with open('/proc/mounts', 'r') as fp: + for line in fp: + if line.split(' ')[0].find('/') >= 0: + mounts.append(line) + + lsblkInfo = subprocess.check_output('/bin/lsblk') + with open('/proc/diskstats', 'r') as fp: + diskStats = fp.read() + with open('/proc/mdstat', 'r') as fp: + mdstat = fp.read() + if self._beingRoot: + pvInfoShort = subprocess.check_output('/sbin/pvs') + pvInfo = subprocess.check_output('/sbin/pvdisplay') + vgInfoShort = subprocess.check_output('/sbin/vgs') + vgInfo = subprocess.check_output('/sbin/vgdisplay') + lvInfoShort = subprocess.check_output('/sbin/lvs') + lvInfo = subprocess.check_output('/sbin/lvdisplay') + fnOutput = target + os.sep + 'system.info.gz' + with gzip.open(fnOutput, 'wb') as fp: + fp.write(b'=== manually installed packets:\n') + for packet in packets: + if packet != '': + fp.write('apt-get install -y {:s}\n'.format(packet).encode()) + fp.write(b'\n=== lsblk:\n' + lsblkInfo) + fp.write(b'\n=== mounts:\n' + ''.join(mounts).encode()) + fp.write(b'\n=== diskstats:\n' + diskStats.encode()) + fp.write(b'\n=== mdstat:\n' + mdstat.encode()) + if self._beingRoot: + fp.write(b'\n=== pvs:\n' + pvInfoShort) + fp.write(b'\n=== vgs:\n' + vgInfoShort) + fp.write(b'\n=== lvs:\n' + lvInfoShort) + fp.write(b'\n=== pvdisplay:\n' + pvInfo) + fp.write(b'\n=== vgdisplay:\n' + vgInfo) + fp.write(b'\n=== lvdisplay:\n' + lvInfo) + self.logFile(fnOutput, '%f: %s %t created in %r', start) + + def updateApt(self, force=False): + '''Tests whether the last "apt update" command is younger than one day. + If not the command is executed. + @param force: True: the marker file will be removed: apt-get update is executed always + ''' + doUpdate = False + fileMarker = self._configDir + '/last.apt.update.mrk' + if force: + doUpdate = True + elif not os.path.exists(fileMarker): + doUpdate = True + else: + status = os.lstat(fileMarker) + doUpdate = status.st_mtime + 86400 < time.time() + if doUpdate: + if not base.BaseTool.unitTestIsActive(): + self._processTool.execute(['apt-get', 'update'], True) + # change file date: + base.StringUtils.toFile(fileMarker, '') + os.chmod(fileMarker, 0o666) + +def usage(): + '''Returns an info about usage + ''' + return """usage: [<global_opts>] <mode> + some actions for restauration of servers +GLOBAL_OPTS +GLOBAL_MODES +<mode>: + btrfs <what> + <what>: + create-fs <device> <label> [<mount-path>] [--image=<image>:<size>] [--force] + creates a btrfs filesystem on <device> and autofs mounting + <device>: the device on which the btrfs filesystem is created + <label>: the label (name) of the btrfs filesystem + <mount-path>: the mountpoint Default: /media/<label> + <path>: the full filename of the image + <size>: the size of the loop device image: <count><unit>, e.g. 32G for 32 GBYte or 2T vor 2 TByte + --force: works too if the filesystem seems already to exist + create-snapshot <label-root> <label-subvol> [<label-snapshot>] [--mode=<mode>] [--read-only | -r] + <label-root>: the mountpoint of the while btrfs device (must be mounted) + <label-subvol>: the label of the subvolume which is to copy + <label-snapshot>: the label of the new snapshot: needed if missing --mode + <mode>: dayly | weekly | monthly | now: the label of the snapshot depends of the given mode + e.g. 'dayly' appends '.sun' to the subvol label if the current day is Sunday + --read-only or -r: the snapshot will be not be writable + create-subvol <label-root> <label-subvol> [<mount-path>] [--size=<size>] [--force] [--options=<mount-opts>] [--move-files] + <label-root>: the mountpoint of the while btrfs device (must be mounted) + <label-subvol>: the label of the new subvolume, e.g. "backup" (should not exist) + <mount-path>: the relative path of the subvolume. Default: <label-subvol> under /media/<label-root> + <size>: the size of the logical volume: <count><unit>, e.g. 32G for 32 GBYte or 2T vor 2 TByte + --force: works if seems the subvolume already exists + --move-files: if the mount directory contains dirs/files they will be moved to the new subvolume + <mount-opts>: delimited with ',': compress=zlib compress=lzo nodatacow + init <what> + <what>: + apache <opts> + installs the Apache webserver + <opt>: + --ports=<port-http>,<port-https> + port for HTTP and HTTPS, e.g. --ports=81,444 + cms + installs packages relevant for content management systems: imagemagick redis-server... + dirs + creates the standard directories (if needed) + etc + initializes /etc + grub [--dry] + extends /boot/grub/grub.cfg + --dry: show infos, do not change grub.cfg + letsencrypt + prepares the system for creation of letsencrypt certificates + linuxserver + installs boot daemon, swapiness setting, useful packages + local-bin + initializes /usr/local/bin + mariadb + installs the database management system mariadb + nginx <opts> + installs the webserver NGINX + <opt>: + --well-known=<dir> + <dir> is the directory containing a .well-known directory (used for letsencrypt) + php <version> + installs packages relevant for PHP + net <what> + <what>: + info + displays relevant infos + static [ip [interface [gateway]]] + defines an interface with a static ip + ip: examples: 10.10.10.100 or 100 + interface: eth0 or ens18 Default: the first device given by the system + gateway: Default: the above ip with end number 1 + virt-manager netname + defines a "host to guest" network in a virtmanager environment + netname: identifier of the internal network + nameserver [ip] + defines the DNS nameserver + reconfigure <what> + <what>: + letsencrypt <mode> { <domain> | --all } + switches the letsencrypt certificate and private certificate + <mode>: 'on': letsencrypt certificate will be switched on. 'off': the private certificate will be switched on + <domain>: specifies the filename in /etc/nginx/sites-enabled. With or without 'www.' + --all: switch all domains in /etc/nginx/sites-enabled + php [<version>] + adapts the php ini files of the given version + if no version is given the highest installed version will be taken + remove <what> + <what>: + restore <what> + <what>: + clone <archive-prefix> <target> + search archive (zip/tar), extracts it as sibling of the target + e.g. "local=bin.tgz" becomes /usr/local/bin2 + etc + merge passwd, shadow and group, includes restore of 'pyrshell', 'nginx' + pyrshell + recovers the /etc/pyrshell directory + nginx + recovers the /etc/nginx + storage <what> + <what>: + autofs server mount-base source1 [source2 ... ] <opts> + installs and configures automatic mounting + server: defines the name of the configuration, e.g. the NFS server + mount-base: '/' or a common mount point + source: type|extern-address|mount-point|option + option: rw: read write ro: read only + examples: + nfs|10.10.10.1:/media/data/www|/media/www|ro + disk|2e8b6f9b-6fc2-4463-8c58-d812104baef9|/media/usb|rw + <opt>: + --auto-select=<pattern> + for disks: a regular expression. Must be unique. See <what>="disk" below + disks [<pattern>] + informs about disks, e.g. UUID + <pattern>: a regular expression for any info, e.g. disk type, disk name... + ftp-service [port] + installs proftp + port: the serviced port + nfs-share <addr1> [<addr2>...] + Creates /etc/exports. Import: see storage autofs + <addrX>: <directory>:<client-ip>:<option> + <option>: rw: read write ro: readonly + example: /media/www:10.10.10.104:rw + sftp-service <user> <directory1> [<directory2> ...] + installs an SFTP service with exclusive access to some directories (chroot) + <user>: this user can login + <directoryX>: theese directories are available via SFTP + lvm <vol-group> <log-volume> <size> <mountpoint> + <vol-group>: volume group (must exist) + <log-volume>: logical volume (will be created) + <size>: the size of the logical volume: <count><unit>, e.g. 32G for 32 GBYte or 2T vor 2 TByte +Examples: +restoretool -q restore etc +restoretool -v3 init php 7.3 +restoretool -v3 init nginx --well-known=/var/www/letsencrypt +restoretool -v3 init grub --dry +restoretool -v3 init linuxserver +restoretool -v3 net info" +restoretool -v3 net static 100 ens18" +restoretool -r -v3 reconfigure php 7.1 +restoretool -v3 storage autofs bigtoy / "nfs|10.10.10.1:/media/data/www|/media/www|rw" +restoretool -v3 storage autofs disk --auto-select=sda1 +restoretool -v3 storage ftp-service 58725" +restoretool -v3 storage nfs-share "/media/www:10.10.104:rw" "/media/www/cloud:10.10.103:ro" +restoretool -v3 storage sftp-service coder /var/www /home/ws/php --force" +restoretool -v3 storage disks "sd[bc]" +restoretool -v3 storage lvm sixpack 100G /media/sixpack/cl/cloud.infeos.de +restoretool -v3 storage btrfs sixpack cloud cl/cloud --size=32G +restoretool -v3 btrfs create-fs /dev/sda4 data +restoretool -v3 btrfs create-fs /dev/loop22 fs.unittest --image=/tmp/unittest.img:150M --force +restoretool -v3 btrfs create-subvol raid databases /var/lib/mysql --options=nodatacow --move-files +restoretool -v3 btrfs create-snapshot fs.system @home --mode=dayly --read-only +restoretool -v3 restore clone usr=local=bin.tgz /usr/local/bin +restoretool -v3 restore etc +restoretool -v3 reconfigure letsencrypt on www.example.de +restoretool -v3 reconfigure letsencrypt off www.example.de +restoretool -v3 reconfigure letsencrypt off --all +""" + +def createLetsEncryptConf(root): + return ''' +location ^~ /.well-known/acme-challenge/ { + default_type "text/plain"; + root $ROOT; +} +# Hide /acme-challenge subdirectory and return 404 on all requests. +# It is somewhat more secure than letting Nginx return 403. +# Ending slash is important! +location = /.well-known/acme-challenge/ { + return 404; +} +'''.replace('$ROOT', root) + + def example(self): + '''Creates a example configuration file. + ''' + example = '''# restoretool example configuration +log.file=/var/log/local/.log +backup.directory=/media/backup +url.download=https://public.hamatoma.de + +php.memory_limit = 2048M +php.upload_max_filesize = 512M +php.max_file_uploads = 100 +php.post_max_size = 512M +php.max_execution_time = 600 +php.max_input_time = 600 +''' + self.storeExample(example) + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/restoretool', 'script', '-', 'data.csv'] + ''' + appInfo = base.BaseTool.ApplicationInfo('restoretool', 'appl/RestoreTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = RestoreTool(options) + (cmd, argv) = tool.handleStandardCommands(argv) + rc = None + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'init': + tool.updateApt() + rc = tool.init(argv) + elif cmd == 'net': + tool.network(argv) + elif cmd == 'reconfigure': + tool.updateApt() + rc = tool.reconfigure(argv) + elif cmd == 'remove': + rc = tool.remove(argv) + elif cmd == 'restore': + rc = tool.restore(argv) + elif cmd == 'btrfs': + rc = tool.btrFs(argv) + elif cmd == 'storage': + rc = tool.storage(argv) + else: + tool.usage("unknown command: " + cmd) + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/SvgTool.py b/appl/SvgTool.py new file mode 100755 index 0000000..b3d382e --- /dev/null +++ b/appl/SvgTool.py @@ -0,0 +1,741 @@ +#! /usr/bin/python3 +''' +svgtool: Scalable Vector Graphics tool + +@author: hm +''' +import os.path +import sys +import re +import time +import math +import functools + +sys.path.insert(0, '/usr/share/pyrshell') +import base.BaseTool +import base.StringUtils + +gSvgToolPeriod = 4 + +class Column: + '''Implements a series of data, e.g. one dimension of a data collection. + ''' + def __init__(self, title, parent): + '''Constructor. + @param title: the column's title + @param parent: the array containing the column + ''' + self._parent = parent + self._title = title + self._factor = 1 + if title.find('GByte') >= 0: + self._factor = 1024*1024*1024 + elif title.find('MByte') >= 0: + self._factor = 1024*1024 + elif title.find('KByte') >= 0: + self._factor = 1024 + self._min = 1E100 + self._max = -1E100 + self._average = 0.0 + self._reducedRange = None + self._values = [] + self._desc = False + self._asc = False + self._dataType = None + self._offset = 0 + + def add(self, value): + [value, dataType] = base.StringUtils.toFloatAndType(value.strip()) + if dataType == 'int': + dataType = 'float' + if dataType == 'undef': + raise ValueError(value) + if self._dataType == None: + self._dataType = dataType + elif dataType != self._dataType: + raise ValueError('mixed data types: {} / {}'.format(dataType, self._dataType) ) + self._values.append(value) + + def average(self): + '''Returns the average of the values. + @return: the average + ''' + return self._average / self._factor + + def findMinMax(self, spreadRange, spreadFactor, maxAverageQuotient = 40): + '''Finds the minumum and the maximum of the data: + spreadRange is given as % value. The extrema will be set in this way + that only points inside this range are displayed. + Example: data = [-5, 1, 2, 7, 99], spreadRange is 60%. + the data inside the range are [1, 2, 7]. _max = 7, _min=1 + @param spreadRange: a % value: only data in this range will be displayed + @param spreadFactor: @pre: greater or equal 1 + if abs(extremum-endOfRange) / range <= spreadFactor: the range is expanded to the extremum + Example: data [0.5, 1, 2, 7, 99] max=7 min=1 range=7-1=6 + abs(0.5-7)/6=1.099 1.099<1.1 => _min=0.5 + abs(99-1)/6=16 16>1.1 => _max=99 + @param maxAverageQuotient: if max*min > 0 and (max / min < maxAverageQuotient: no clipping is done + ''' + if spreadRange < 100 and len(self._values) > 0: + # if 100 _min and _max are already set + minValues = [] + maxItems = len(self._values) * (100.0 - spreadRange) / 100 + # round up. +1: we want the extremum outside of the excluded range: plus one item + countMax = int(maxItems + 0.5) + 1 + countMin = int(maxItems) + 1 + maxValues = [] + sum = 0.0 + for val in self._values: + if type(val) == str: + val = float(val) + sum += val + if len(minValues) < countMin: + minValues.append(val) + minValues.sort() + elif val < minValues[-1]: + minValues[-1] = val + minValues.sort() + if len(maxValues) < countMax: + maxValues.append(val) + maxValues.sort() + elif val > maxValues[0]: + maxValues[0] = val + maxValues.sort() + # get the maximum of the found values: + self._min = minValues[-1] + # get the minimum of the found values: + self._max = maxValues[0] + distance = self._max - self._min + # we use the full range if the difference of the full range and the calculated range is less than 10%: + self._average = sum / len(self._values) + if maxValues[-1] - self._min <= distance * spreadFactor: + self._max = maxValues[-1] + if self._max - minValues[0] <= distance * spreadFactor: + self._min = minValues[0] + if sum > 0 and self._max / self._average > maxAverageQuotient: + self._min = minValues[0] + self._max = maxValues[-1] + + def extremum(self, minimumNotMaximum): + '''Returns the minimum or the maximum of the column. + @param minimumNotMaximum: true: returns the minumum otherwise: the maximum + @return the minimum or the maximum of the column divided by _factor + ''' + if minimumNotMaximum: + return self._min / self._factor + else: + return self._max / self._factor + + def getRange(self): + '''Returns the difference between maximum and minimum of the column. + @return the difference between maximum and minimum of the column divided by _factor + ''' + return (self._max - self._min) / self._factor + + def normalize(self, offset): + '''Scales the values to the avarage + varianz + ''' + # column._max = functools.reduce(lambda rc, item: item if item > rc else rc, column._values, -1E+100) + sumValues = functools.reduce(lambda rc, item: rc + item, self._values) + standardDeviation = math.sqrt(functools.reduce(lambda rc, item: rc + item*item, self._values)) / len(self._values) + average = sumValues / len(self._values) + self._reducedRange = average + max(standardDeviation, average) + self._offset = offset + + def getValue(self, index): + '''Gets the index-th value of the column. + @param index: index of _values[] + @return the index-th value, diviced by _factor + ''' + rc = self._values[index] + if type(rc) != float: + rc = base.StringUtils.toFloat(rc) + return rc / self._factor + + def toString(self, index): + value = self.getValue(index) + rc = base.StringUtils.toString(value, self._dataType) + return rc + +class AxisScale: + '''Implements the x or y axis of a graph. + ''' + + def __init__(self, column, maxScales): + '''Constructor. + @param column: the column info related to the scale + @param maxScales: maxScales / 2 < scale-count <= maxScales. scale-count is the number of markers on the scale + ''' + self._column = column + if column._max == -1E+100: + column._max = functools.reduce(lambda rc, item: item if item > rc else rc, column._values, -1E+100) + column._min = functools.reduce(lambda rc, item: item if item < rc else rc, column._values, +1E+100) + self._scaleSize = column._reducedRange if column._reducedRange != None else column.getRange() + rangeScale = "{:e}".format(self._scaleSize) + if self._scaleSize == 0: + self._countScales = 1 + self._lastScale = 0 + self._scaleStep = 1 + elif column._dataType == 'date': + self._countScales = self._scaleSize + self._lastScale = 0 + else: + digit = rangeScale[0] + if digit == '1' or digit == '2' or digit == '3': + lastScale = "{:.1e}".format(column.getRange()) + self._countScales = int(lastScale[0] + lastScale[2]) + self._lastScale = float(lastScale) + else: + self._lastScale = float("{:.0e}".format(column.getRange())) + self._countScales = int(digit) + if self._countScales == 0: + self._countScales = maxScales + elif self._countScales < 0: + self._countScales = - self._countScales + while self._countScales * 2 <= maxScales: + self._countScales *= 2 + while self._countScales > maxScales: + self._countScales //= 2 + self._scaleStep = self._lastScale / self._countScales + + def indexData(self, index, length): + '''Returns the data of a marker with a given index. + @param index: the index of the marker (< _countScales) + @param length: the length of the axis (width for x and height for y) + @return: [posMarker, label] + ''' + if self._countScales == 0 or self._lastScale == 0 or self._scaleSize == 0: + posMarker = 0 + label = '' + else: + posMarker = int(index * length * self._lastScale / self._scaleSize / self._countScales) + value = self._column.extremum(True) + index * self._scaleStep + dataType = self._column._dataType + label = "{}".format(base.StringUtils.toString(value, dataType)) + if dataType == 'datetime': + if index == 0: + self._firstDate = label.split(' ')[0] + else: + current = label.split(' ') + if current[0] == self._firstDate: + label = current[1] + else: + self._firstDate = current[0] + elif dataType == 'float' or dataType == 'int': + if label == '0.000': + label = "{:.2g}".format(value) + else: + while label.endswith('0'): + label = label[:-1] + if label.endswith('.'): + label = label[:-1] + if len(label) > 6: + label = "{:.2g}".format(value) + return [posMarker, label] + +class SvgTool (base.BaseTool.BaseTool): + '''Creates SVG graphics. + ''' + def __init__(self, globalOptions): + '''Constructor. + @param globalOptions: the basic configurations + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'svgtool.conf') + self._columns = [] + self._rexprNo = re.compile(r'^\s*[+-]?\d+([.,]\d+([eE][+-]?\d+)?)?\s*$') + self._color = 'black' + self._strokeWidth = 1 + self._output = [] + self._fontSize = 7 + self._colors = ['black', 'red', 'green', 'magenta', 'orange', 'brown'] + + def convertToMovingAverage(self, data, span = 5): + '''Converts an array of values inplace into an array of values with moving average. + @param data: IN/OUT: the array of values + @param span: the number of values which is used to calculate the average + ''' + window = [] + sum = 0 + spanHalf = int(span / 2) + spanHalf2 = span - spanHalf + for ix in range(span): + window.append(data[ix]) + sum += data[ix] + if ix >= spanHalf: + data[ix-spanHalf] = sum / len(window) + for ix in range(spanHalf2, len(data) - spanHalf): + sum -= window[0] + window = window[1:] + window.append(data[ix + spanHalf]) + sum += window[-1] + data[ix] = sum / span + for ix in range(len(data) - spanHalf, len(data)): + sum -= window[0] + window = window[1:] + data[ix] = sum / len(window) + + def diagram(self, argv, usage): + '''Creates a SVG diagram. + @param argv: arguments + @return: None: OK otherwise: error message + ''' + rc = None + source = argv[0] + target = argv[1] + argv = argv[2:] + if not os.path.exists(source): + rc = "input file {} does not exist".format(source) + else: + fp = None + if target != '-': + fp = open(target, "w") + self.readCsv(source) + width = self._configuration.getInt('width', 1000) + height = self._configuration.getInt('height', 500) + if width < len(self._columns[0]._values): + self.shrinkData(width) + axisAreaWidth = self._configuration.getInt('axis.area.width', 15) + spreadRange = 90 + spreadFactor = 1.1 + maxAverageQuotient = 1.0 + title = 'Diagram' + movingAverage = None + for arg in argv: + if arg.startswith('--width'): + width = self.integerOption(arg) + elif arg.startswith('--height'): + height = self.integerOption(arg) + elif arg.startswith('--axis-area-width'): + axisAreaWidth = self.integerOption(arg) + elif arg.startswith('--spread-range'): + spreadRange = self.integerOption(arg) + if spreadRange < 50 or spreadRange > 100: + self.usage('invalid value (allowed: 50..100): ' + arg) + elif arg.startswith('--moving-average'): + movingAverage = self.integerOption(arg, 5) + elif arg.startswith('--spread-factor'): + spreadFactor = self.floatArgument(arg) + elif arg.startswith('--max-average-quotient'): + maxAverageQuotient = self.integerOption(arg) + if maxAverageQuotient < 1: + self.usage('invalid value (allowed: >= 1): ' + arg) + elif arg.startswith('--title='): + title = arg[8:] + else: + self.usage('unknown options: ' + arg) + self._logger.log('start ' + title) + self.htmlStart(title) + self.svgStart(width, height) + self.xAxis(width, height, axisAreaWidth, 0) + for ix in range(len(self._columns) - 1): + self._color = self._colors[ix % len(self._colors)] + aProperty = 'stroke-dasharray="{},{}'.format(5*(ix + 1), 3) + for ix2 in range(ix + 1): + aProperty += ',1,1' + aProperty += '"' + if movingAverage != None: + self.convertToMovingAverage(self._columns[ix + 1]._values, movingAverage) + self._columns[ix + 1].findMinMax(spreadRange, spreadFactor, maxAverageQuotient) + self.polyline(width, height, axisAreaWidth, 0, ix+1, aProperty) + self.yAxis(width, height, axisAreaWidth, ix + 1, self._color) + self.svgEnd() + self.htmlLegend(); + self.htmlEnd() + if fp == None: + for line in self._output: + print(line) + else: + for line in self._output: + fp.write(line + '\n') + fp.close() + self._logger.log('end ' + title) + return rc + + def example(self): + '''Creates an example configuration file and example data files (sinus.csv and sinus.html). + ''' + example = '''# svgtool example configuration +log.file=/var/log/local/svgtool.log +width=1000 +height=500 +axis.area.width=15 +''' + self.storeExample(example) + global gSvgToolPeriod + name = '/tmp/sinus.csv' + content = 'x;sin(x);cos(x);tan(x)\n' + maxX = 500-15 + for ix in range(maxX): + x = ix * gSvgToolPeriod * 3.141592 / maxX + content += '{};{};{};{}\n'.format(x, math.sin(x), math.cos(x), min(1, max(-1, math.tan(x)))) + base.StringUtils.toFile(name, content) + self._logger.log('created: ' + name) + + def firstLine(self, line): + '''Evaluates the first line. + Searches the separator and the titles (if they exists) + @param line: the first line to inspect + ''' + cTab = line.count('\t') + cComma = line.count(',') + self._columns = [] + cSemicolon = line.count(';') + if cTab >= cComma and cTab >= cSemicolon: + self._separator = '\t' + elif cSemicolon >= cComma and cSemicolon >= cTab or cSemicolon > 0 and cSemicolon == cComma - 1: + self._separator = ';' + else: + self._separator = ',' + titles = line.split(self._separator) + isNumeric = True + for title in titles: + self._columns.append(Column(title, self)) + if self._rexprNo.match(title) == None: + isNumeric = False + if isNumeric: + self.numericLine(line, 1) + for ix in range(len(titles)): + self._columns[ix]._title = "col{:d}".format(ix + 1) + + def htmlEnd(self): + self._output.append('</body>\n</html>') + + def htmlLegend(self): + '''Writes the legend of the dialog as HTML table. + ''' + xCol = self._columns[0] + self._output.append('<table border="1"><thead><tr><td>Titel</td><td>Schnitt</td><td>Minimum</td><td>Maximum</td><td></td></tr></thead>\n'); + self._output.append('<tbody>\n<tr style="color: blue"><td>{}:</td><td>{}</td><td>{}</td><td>{}</td><td>{} Werte</td></tr>\n' + .format(xCol._title, '', base.StringUtils.toString(xCol.extremum(True), xCol._dataType, 2), + base.StringUtils.toString(xCol.extremum(False), xCol._dataType, 2), len(xCol._values))) + + for ix in range(len(self._columns) - 1): + yCol = self._columns[ix + 1] + self._output.append('<tr style="color: {}"><td>{}:</td><td>{:.6f}</td><td>{:.6f}</td><td>{:.6f}</td><td></td></tr>\n' + .format(self._colors[ix % len(self._colors)], yCol._title, yCol.average(), yCol.extremum(True), + yCol.extremum(False))) + self._output.append('</tbody>\n</table>\n') + + def htmlStart(self, title): + '''Starts a HTML script. + ''' + self._output.append('<html>\n<body>\n<h1>{}</h1>\n'.format(title)) + + def numericLine(self, line, lineNo): + '''Evaluates a "numeric" line (a list of values) + Searches the separator and the titles (if they exists) + @param line: the line to inspect + @param lineNo: the line number + ''' + values = line.split(self._separator) + if len(values) != len(self._columns): + self._logger.error('wrong column number in line {}: {} instead of {}'.format(lineNo, len(values), len(self._columns))) + for ix in range(len(values)): + if ix < len(self._columns): + self._columns[ix].add(base.StringUtils.toString(values[ix], self._columns[ix]._dataType)) + + def polyline(self, width, height, axisAreaWidth, indexX, indexY, properties = None): + '''Converts the CSV data into a polyline. + @param width: the length of the x dimension + @param height: the length of the y dimension + @param axisAreaWidth: the width of the axis area (x and y) + @param indexX: the column index of the x data + @param indexy: the column index of the Y data + @param properties: None or additional SVG properties for polyline, e.g. 'stroke-dasharray="5,5" + ''' + self._output.append('<polyline style="fill:none;stroke:{};stroke-width:{}"{}'.format(self._color, self._strokeWidth, ' ' + properties if properties != None else '')) + line = 'points="' + xCol = self._columns[indexX] + yCol = self._columns[indexY] + vWidth = max(1E-10, xCol.getRange()) + vHeight = max(1E-10, yCol.getRange()) + vUsable = (height - axisAreaWidth) + for ix in range(len(xCol._values)): + x = axisAreaWidth + int((xCol.getValue(ix) - xCol.extremum(True)) * (width - axisAreaWidth) / vWidth) + yRange = yCol.extremum(False) - yCol.extremum(True) + if yCol.getValue(ix) != None: + # a1 = yCol.getValue(ix) + # aE = yCol.extremum(True) + # aR = yCol._reducedRange + # bring y into 0..max + y = (yCol.getValue(ix) - yCol.extremum(True)) + aY0 = y + # normalize into 0..1: + if yRange != 0.0: + y = y / yRange + # aYnorm = y + if yCol._reducedRange != None and yCol._reducedRange != 0: + y /= yCol._reducedRange + yPixel = int(vUsable - y * vUsable) + line += "{:g},{:g} ".format(x, yPixel) + self._output.append(line + '" />') + + def putCsv(self, target): + '''Puts the internal columns into a CSV file + @param target: the full name of the result file + ''' + with open(target, "w") as fp: + line = '' + for col in self._columns: + line += col._title + ';' + fp.write(line[0:-1] + "\n") + for ix in range(len(self._columns[0]._values)): + line = '' + for col in self._columns: + line += col.toString(ix) + ';' + fp.write(line[0:-1] + "\n") + + def readCsv(self, source): + '''Reads a CSV file with the diagram data. + @param source: the filename, e.g. 'diagram/data1.csv' + ''' + with open(source, "r") as fp: + lineNo = 0 + for line in fp: + line = line.strip() + lineNo += 1 + if lineNo == 1: + self.firstLine(line) + else: + self.numericLine(line, lineNo) + # Remove empty columns: + count = len(self._columns) - 1 + for ix in range(count, -1, -1): + column = self._columns[ix] + if column._max == -1E+100: + column._max = functools.reduce(lambda rc, item: base.StringUtils.toFloat(item) if base.StringUtils.toFloat(item) > rc else rc, column._values, -1E+100) + column._min = functools.reduce(lambda rc, item: base.StringUtils.toFloat(item) if base.StringUtils.toFloat(item) < rc else rc, column._values, +1E+100) + # column.normalize((1 + ix % 5) / count * 0.8) + self.returnToZero() + + def returnToZero(self): + '''Find gaps in x values and set behind every gap a "return to zero" line + example: + x;y;z + 1;99;77 + 2;100;70 + 20;90;60 + There is a gap between line 2 and 3. Minimum gap length is 1 (between line 1 and 2) + We insert "3;0;0" and "19;0;0" + Result: + x;y;z + 1;99;77 + 2;100;70 + 3;0;0 + 19;0;0 + 20;90;60 + ''' + columnX = self._columns[0] + self._minGap = +1E+100 + [last, dummy] = base.StringUtils.toFloatAndType(columnX.getValue(0)) + for ix in range(len(columnX._values) - 1): + [current, dummy] = base.StringUtils.toFloatAndType(columnX._values[1+ix]) + if current - last < self._minGap: + self._minGap = current - last + if self._minGap < 5*60: + self._minGap = 5*60 + [last, dummy] = base.StringUtils.toFloatAndType(columnX.getValue(-1)) + for ix in range(len(columnX._values) - 1, 1, -1): + [current, dummy] = base.StringUtils.toFloatAndType(columnX.getValue(ix-1)) + if last - current > self._minGap: + columnX._values.insert(ix, last - self._minGap) + columnX._values.insert(ix, current + self._minGap) + for col in range(len(self._columns)): + if col > 0: + self._columns[col]._values.insert(ix, 0) + self._columns[col]._values.insert(ix, 0) + last = current + self.putCsv('/tmp/corrected.csv') + + def simpleLine(self, x1, y1, x2, y2, properties = None, color = None): + line = '<line x1="{}" y1="{}" x2="{}" y2="{}" stroke="{}" stroke-width="{}" {}/>'.format( + x1, y1, x2, y2, color if color != None else self._color, self._strokeWidth, properties if properties != None else '') + self._output.append(line) + + def simpleText(self, x, y, text): + self._output.append('<text x="{}" y="{}" fill="{}" font-size="{}">{}</text>'.format(x, y, self._color, self._fontSize, text)) + + def shrinkData(self, count): + '''Returns an array of columns with count elements per column. + Input is self._columns. + @pre: the first column contains the x data. + @post: the x values (first column) of the result are equidistant. + @post: the local extrema (minimum and maximum) will be saved + @param count: the number of items of each column of the result + @return: the array of the converted columns + ''' + xValues = self._columns[0]._values + rc = [] + if count <= 0 or len(xValues) <= count: + rc = self._columns[:] + else: + xOut = Column(self._columns[0]._title, rc) + rc.append(xOut) + step = (xValues[-1] - xValues[0]) / (count - 1) + x = xValues[0] + for ix in range(count): + xOut._values.append(x) + x += step + + for ixCol in range(len(self._columns) - 1): + yCol = self._columns[1+ixCol] + yValues = yCol._values + yOut = Column(yCol._title, rc) + rc.append(yOut) + ixLastSrc = -1 + yMiddle = 0 + for ixTrg in range(count): + xTrg = xOut._values[ixTrg] + ixLastSrc += 1 + yMin = yValues[ixLastSrc] + if ixTrg == 0: + yOut._values.append(yMin) + elif ixTrg == count - 1: + yOut._values.append(yValues[-1]) + else: + yMax = yMin + while xValues[ixLastSrc] <= xTrg: + if yValues[ixLastSrc] < yMin: + yMin = yValues[ixLastSrc] + elif yValues[ixLastSrc] > yMax: + yMax = yValues[ixLastSrc] + ixLastSrc += 1 + # 4 cases: max: min: line up: line down: + # yHigh: a i i u d + # a a i u d + # yLow: u d + # xLow xHigh + if yOut._values[ixTrg - 1] > yMax: + # y[ixTrg-1] is line down or max: + yOut._values.append(yMin if ixTrg <= 1 or yValues[ixTrg - 2] > yValues[ixTrg - 1] else yMiddle) + else: + # y[ixTrg-1] is line up or min + yOut._values.append(yMax if ixTrg <= 1 or yValues[ixTrg - 2] < yValues[ixTrg - 1] else yMiddle) + yMiddle = (yMax - yMin) / 2 + return rc + + def svgEnd(self): + self._output.append('</svg>\n'); + + def svgStart(self, width, height): + '''Starts the SVG block. + @param width: the width of the SVG area + @param height: the height of the SVG area + ''' + self._output.append('<svg height="{}" width="{}">\n'.format(height, width)) + + def xAxis(self, width, height, axisAreaWidth, indexX): + '''Creates the x axis. + @param width: the length of the x dimension + @param height: the length of the y dimension + @param axisAreaWidth: the width of the axis area (x and y) + @param indexX: the column index of the x values + ''' + color = self._color + self._color = 'blue' + self.simpleLine(axisAreaWidth, height - axisAreaWidth, width, height - axisAreaWidth) + xCol = self._columns[indexX] + axis = AxisScale(xCol, min((width - axisAreaWidth) / 50, 20)) + y1 = height - axisAreaWidth - self._strokeWidth * 3 + y2 = height - axisAreaWidth + self._strokeWidth * 3 + for ix in range(int(axis._countScales)): + [pos, label] = axis.indexData(ix, width - axisAreaWidth) + x = axisAreaWidth + pos + self.simpleLine(x, y1, x, y2) + self.simpleText(x - 10, y2 + axisAreaWidth / 2, label) + if ix > 0: + self.simpleLine(x, y1 - 5, x, 0, 'stroke-opacity="0.1" stroke-dasharray="5,5"', 'rgb(3,3,3)') + self._color = color + + def yAxis(self, width, height, axisAreaWidth, indexY, color): + '''Creates the x axis. + @param width: the length of the x dimension + @param height: the length of the y dimension + @param axisAreaWidth: the width of the axis area (x and y) + @param indexY: the column index of the y values + ''' + color2 = self._color + self._color = color + self.simpleLine(axisAreaWidth, 0, axisAreaWidth, height - axisAreaWidth) + yCol = self._columns[indexY] + axis = AxisScale(yCol, min((height - axisAreaWidth) / 50, 20)) + x1 = axisAreaWidth - self._strokeWidth * 3 + x2 = axisAreaWidth + self._strokeWidth * 3 + for ix in range(int(axis._countScales)): + [pos, label] = axis.indexData(ix, height - axisAreaWidth) + y = height - axisAreaWidth - pos + self.simpleLine(x1, y, x2, y) + self.simpleText(1+(indexY-1)*30, y, label) + if indexY == 1 and ix > 0: + self.simpleLine(x2 + 5, y, width, y, 'stroke-opacity="0.1" stroke-dasharray="5,5"', 'rgb(3,3,3)') + self._color = color2 + +def usage(): + '''Returns an info about usage + ''' + return """svgtool [<opts>] <command> + Builds Scalable Vector Graphics embedded in HTML. +GLOBAL_OPTS +GLOBAL_MODES +<command>: + x-y-diagram <input-file> <output-file> <opts> + <output-file> + '-': output will be put to the stdout otherwise: the HTML will be put to this file + <opt>: + --width=<width> + the width of the drawing area in pixel. Default: 1000 + --height=<height> + the height of the drawing area in pixel. Default: 500 + --axis-area-width=<width> + the width of the area containing the axis and the related labels (for x and y axis). Default: 15 + --max-average-quotient=<value> + if max/avg(values) < maxAvgQuotient: no clipping is done. Default: 5 + --moving-average=<window-length> + prepare data with "moving average": for each value a "window" (values and neigbours, symetic left + and right) is used to build the average: this average is used instead of the value + default windows width: 5 + --spread-range=<value> + a % value: only data in this range will be displayed. Default: 90 + --spread-factor + if abs(extremum-endOfRange) / range <= spreadFactor: the range is expanded to the extremum + Example: data [0.5, 1, 2, 7, 99] max=7 min=1 range=7-1=6 + abs(0.5-7)/6=1.099 1.099<1.1 => _min=0.5 + abs(99-1)/6=16 16>1.1 => _max=99 + --title=<title> + Default: Diagram +example: + svgtool -v2 x-y-diagram /tmp/sinus.csv /tmp/sinus.html --width=1920 --height=1024 "--title=Trigonometric functions from [0, 4*pi]" +""" + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/svgtool', 'run'] + ''' + if len(argv) > 2 and argv[1] == 'example': + global gSvgToolPeriod + try: + gSvgToolPeriod = int(argv[2]) + except ValueError: + pass + appInfo = base.BaseTool.ApplicationInfo('svgtool', 'appl/SvgTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = SvgTool(options) + rc = None + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'x-y-diagram': + if len(argv) < 2: + tool.usage('too few arguments') + else: + rc = tool.diagram(argv, usage) + else: + tool.usage('unknown command: ' + cmd) + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/TextTool.py b/appl/TextTool.py new file mode 100755 index 0000000..a5d6e22 --- /dev/null +++ b/appl/TextTool.py @@ -0,0 +1,1313 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' + +import sys +import os.path +import fnmatch +import re +import time +import tempfile +import shutil + +sys.path.insert(0, '/usr/share/pyrshell') +import base.BaseTool +import base.LinuxUtils +import base.FileHelper +import base.PythonToTypeScript +import base.CryptoEngine +import base.StringUtils + +textToolResult = None +textToolTool = None + +class ReplaceRegionInfo: + def __init__(self): + self._regionStart = None + self._regionEnd = None + self._replacement = None + self._startIncluded = True + self._endExcluded = False + +class ReplaceInfo: + def __init__(self, pattern, line, anchor, above, maxCount): + self._pattern = pattern + self._line = line + self._anchor = anchor + self._above = above + self._maxCountPerFile = maxCount + self._countPerFile = 0 + self._regionStart = -1 + self._regionEnd = 0 + +class TextTool (base.BaseTool.BaseTool): + + def __init__(self, globalOptions): + '''Constructor. + @param globalOptions: the basic configurations + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'texttool.conf') + self._out = [] + self._euid = os.geteuid() + self._egid = os.getegid() + self._list = False + self._onlyMatch = False + self._format = None + self._reverse = False + self._excluded = None + self._missing = False + self._dirMustBeWritable = False + self._fileMustBeWritable = False + self._target = None + self._hits = 0 + self._countPerLine = None + self._table = None + self._patternReference = r'[$]\d+' + self._references = [] + self._lines = [] + self._currentChanged = False + self._replaceInfo = None + self._fileCount = 0 + self._maxFileCount = 0x7ffffff + self._noName = False + self._replaceRegionInfo = None + + + def adaptConfiguration(self, argv): + '''Replaces values of given variables in a configuration. + @param variables: an array of strings with variable definitions, e.g. ['abc=123', 'command.log = "/var/log/command.log"'] + @param configuration: the configuration content (as array of text) with comments, e.g. ["#configuration:", "abc=4", "command.log='/var/log/dummy.log'"] + @return: the changed text, e.g. ["#configuration:", "abc=123", 'command.log = "/var/log/command.log"'] + ''' + if len(argv) == 0: + self.usage('missing <configuration-file>') + else: + configFile = argv[0] + argv = argv[1:] + if not os.path.exists(configFile): + self.usage('missing ' + configFile) + elif len(argv) == 0: + self.usage('missing <variables-files> or prefix=<prefix>') + else: + configuration = base.StringUtils.fromFile(configFile) + arg1 = argv[0] + argv = argv[1:] + if arg1.startswith('prefix='): + prefix = arg1[7:] + variables = self.findVariables(prefix) + else: + variablesFile = arg1 + if not os.path.exists(variablesFile): + self.usage('missing ' + variablesFile) + else: + variables = base.StringUtils.fromFile(variablesFile).split('\n') + self._out = self.adaptVariables(variables, configuration.split('\n')) + if self._hits > 0: + self._logger.log('{} variable(s) changed in {}'.format(self._hits, configFile), 1) + self.createBackup(configFile, base.FileHelper.pathToNode(os.path.dirname(configFile))) + base.StringUtils.toFile(configFile, '\n'.join(self._out)) + + def adaptVariables(self, variables, configuration, logSummary = False): + '''Replaces values of given variables in a configuration. + @post: self._hits: the number of changed variables + + @param variables: an array of strings with variable definitions + e.g. ['abc=123', 'command.log = "/var/log/command.log"'] + @param configuration: the configuration content (as array of text) with comments + e.g. ["#configuration:", "abc=4", "command.log='/var/log/dummy.log'"] + @param logSummary: True: the number of hits will be logged + @return: the changed text, e.g. ["#configuration:", "abc=123", 'command.log = "/var/log/command.log"'] + ''' + rc = [] + variables2 = dict() + self._hits = 0 + for line in variables: + parts = line.split('=', 2) + if len(parts) <= 1: + continue + key = parts[0].strip() + variables2[key] = line + keys = variables2.keys() + self._hasChanged = False + for line in configuration: + parts = line.split('=', 1) + if len(parts) <= 1: + rc.append(line) + continue + key = parts[0].strip() + if not key in keys: + rc.append(line) + else: + value = parts[1].strip() + value2 = variables2[key].split('=', 1)[1].strip() + if value == value2: + self._logger.log('already defined: {}: {}'.format(key, value), 3) + rc.append(line) + else: + self._hits += 1 + self._logger.log('{}: {} => {}'.format(key, value, value2), 2) + rc.append(variables2[key]) + self._hasChanged = True + if self._hits > 0 and self._verboseLevel > 1 and logSummary: + self._logger.log('{} variables changed'.format(self._hits)) + return rc + + def buildExamples(self): + '''Builds the filed used for the examples in the usage message. + ''' + def build(fn, content): + self._logger.log('creating ' + fn, 1) + base.StringUtils.toFile(fn, content) + + baseDir= '/usr/share/pyrshell/examples/' + configDir = self.ensureDirectory(baseDir + 'config') + dataDir = self.ensureDirectory(baseDir + 'data') + dataDir2 = self.ensureDirectory(baseDir + 'data/subdir') + safeDir = self.ensureDirectory(baseDir + 'safe') + build(configDir + os.sep + 'php_minimal.conf', '''memory_limit = 2048M +upload_max_filesize = 512M +max_file_uploads = 100 +post_max_size = 512M +max_execution_time = 600 +max_input_time = 600 +''') + fn = safeDir + os.sep + 'php.ini' + self._logger.log('copying ' + dataDir + os.sep + 'php.ini', 1) + shutil.copy2(fn, dataDir) + build(dataDir + os.sep + 'sum.txt', '''water: 7.32 EUR +cheese: 13.08 EUR +total: 20.40 EUR +''') + build(dataDir + os.sep + 'ip.out.txt', '''1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo + valid_lft forever preferred_lft forever + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever +2: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 + link/ether 8c:16:45:92:9a:c6 brd ff:ff:ff:ff:ff:ff + inet 192.168.2.50/24 brd 192.168.2.255 scope global enp2s0 + valid_lft forever preferred_lft forever + inet6 fe80::8e16:45ff:fe92:9ac6/64 scope link + valid_lft forever preferred_lft forever +3: wlp4s0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000 + link/ether 50:5b:c2:e9:38:01 brd ff:ff:ff:ff:ff:ff +''') + build(dataDir + os.sep + 'today.sh', '''#! /bin/bash +DATE=31.2.2100 +echo $DATE +''') + build(dataDir + os.sep + 'first.addr', '''jonny@gmx.de +Jonny.Hamburger@web.de +info@whow.com +''') + build(dataDir + os.sep + 'second.addr', '''info@whow.com +jonny@gmx.de +info@bla.com +''') + build(dataDir + os.sep + 'test.addr', '''info@whow.com +jonny@gmx.de +info@bla.com +''') + build(dataDir + os.sep + 'shop.addr', '''jonny@gmx.de +info@bla.com +''') + build(dataDir2 + os.sep + 'third.addr', '''jonny@gmx.de +info@bla.com +''') + build(configDir + os.sep + 'german.txt', '''No\tNr +First Name\tVorname +Last Name\tNachname +''') + build(dataDir + os.sep + 'addr.csv', '''No;First Name;Last Name +1;Henry;Miller +2;Berta;Eastern +''') + + def buildTargetname(self, nameInput): + '''Builds the targetname from a pattern and the input filename. + @param nameInput: the input filename + @return: the target filename + ''' + if self._target == '!full!': + rc = nameInput + '~' + else: + parts = base.FileHelper.splitFilename(nameInput) + rc = self._target + rc = rc.replace('!full!', nameInput) + rc = rc.replace('!path!', parts['path']) + rc = rc.replace('!node!', parts['node']) + rc = rc.replace('!fn!', parts['fn']) + rc = rc.replace('!ext!', parts['ext']) + return rc + + def currentFind(self, regExpr, ixStart=0, ixEnd=None): + '''Searches a regular expression in the current file. + @param regExpr: a regular expression (re.RegExpr) + @param ixStart: the index of the first line to inspect + @param ixEnd: None or the successor index of the last line to inspect (_lines[ixEnd] will not be inspected) + @return: None: not found otherwise: the index of the first hit + ''' + rc = None + if ixStart == None: + ixStart = 0 + if ixEnd == None: + ixEnd = len(self._lines) + for ix in range(ixStart, ixEnd): + line = self._lines[ix] + if regExpr.search(line): + rc = ix + break + return rc + + def currentFind2(self, regExpr, ixStart=0, ixEnd=None): + '''Searches a regular expression in the current file and return the line index and the matcher. + @param regExpr: a string or a regular expression (re.RegExpr) + @param ixStart: the index of the first line to inspect + @param ixEnd: None or the successor index of the last line to inspect (_lines[ixEnd] will not be inspected) + @return: tuple (index, matcher) the index line and the matcher. (None, None): not found + ''' + if type(regExpr) == str: + regExpr = self.regExprCompile(regExpr, 'currentFind2') + rc = (None, None) + if ixEnd == None: + ixEnd = len(self._lines) + for ix in range(ixStart, ixEnd): + line = self._lines[ix] + matcher = regExpr.search(line) + if regExpr.search(line): + rc = (ix, matcher) + break + return rc + + def currentInsertAnchored(self, line, anchor, below=True, start=None, end=None): + '''Inserts a line in front of or below a given anchor. + If the anchor is not found the line is added at the end. + + @param line: the line to insert + @param anchor: a regular expression to search: text or regexpr object + @param below: True the insert position is below the anchor line + @param start: None or the first index to inspect + @param end: None or the successor index to inspect (_lines[end] is not inspected) + @return: True: success (key found) + ''' + found = False + if start == None: + start = 0 + if end == None: + end = len(self._lines) + if anchor != None: + if type(anchor) == str: + anchor = self.regExprCompile(anchor, 'currentInsertAnchored') + ixFound = None + for ix in range(start, end): + current = self._lines[ix] + if anchor.search(current): + ixFound = ix + 1 if below else ix + break + if ixFound != None: + self._lines.insert(ixFound, line) + found = True + if not found: + self._lines.insert(end, line) + self._currentChanged = True + return found + + def currentReplace(self, regExpr, line, anchor=None, below=False, start=None, end=None): + '''Replaces a line containing given regular expression or inserts it. + If not found: + if an anchor is given: the anchor is searched. + If found, insertion is done. + If not found: nothing is done + if no anchor is given: + line is inserted at the end + @param regExpr: a pattern string or an re.regexpr object describing the key content of the line to replace + @param line: the line to replace (or insert) + @param anchor: None or a regular expression (string or re.RegExpr) describing an insert point (if not found) + @param below: True: the insert point is below the anchor + @param start: None or the first index to inspect + @param end: None or the successor index to inspect (_lines[end] is not inspected) + @return: True: success + False: anchor != None: anchor not found, nothing replaced + ''' + if type(regExpr) == str: + regExpr = re.compile(regExpr) + found = False + if start == None: + start = 0 + if end == None: + end = len(self._lines) + for ix in range(start, end): + current = self._lines[ix] + if regExpr.search(current) != None: + found = True + if self._lines[ix] != line: + self._currentChanged = True + self._logger.log('replaced: {} -> {}'.format(self._lines[ix], line), 3) + else: + self._logger.log('unchanged: {}'.format(self._lines[ix], line), 4) + self._lines[ix] = line + break + if not found and anchor != None: + if type(anchor) == str: + anchor = re.compile(anchor) + for ix in range(len(self._lines)): + current = self._lines[ix] + if anchor.search(current) != None: + found = True + ixNew = ix if below else ix + 1 + self._logger.log('insert into [{}: {}'.format(ixNew, line), 3) + self._lines.insert(ixNew, line) + self._currentChanged = True + break + if anchor == None and not found: + self._lines.insert(end, line) + self._currentChanged = True + found = True + return found + + def currentSimpleInsert(self, full, key, line, anchor, below=True, mustExist=False): + '''Inserts a line in front of or below a given anchor. + If the anchor is not found the line is added at the end. + + @param full: the filename + @param key: the key for the line to replace + @param line: the line to replace/insert + @param anchor: a regular expression to mark the insert position + @param below: True the insert position is below the anchor line + @param mustExist: True: the routine will be aborted if the file does not exist + @return: True: success (key found) + ''' + rc = False + if self.readCurrent(full, mustExist): + if not self.currentSetLine(key, line): + self.currentInsertAnchored(line, anchor, below) + self.writeCurrent(full) + rc = True + return rc + + def currentSetLine(self, key, value, count=1, start=None, end=None): + '''Sets a line marked by key. + @param key: a regular expression to search + @param line: line to replace + @param count: the key is search count times in the given range [start, end[ + @param start: None or the first index to inspect + @param end: None or the successor index to inspect (_lines[end] is not inspected) + @return: True: success (key found) + ''' + found = False + regKey = self.regExprCompile(key, 'currentSetLine') if type(key) == str else key + firstIx = start + if count == None: + count = 0x7fffffff + for current in range(count): + ix = self.currentFind(regKey, firstIx, end) + if ix == None: + break + if self._lines[ix] != value: + self._currentChanged = True + self._lines[ix] = value + firstIx = ix + 1 + found = True + return found + + def example(self): + example = '''# texttool example configuration +log.file=/var/log/local/texttool.log +php.upload_max_filesize=624M +php.max_file_uploads=102 +php.post_max_size=448M +php.max_execution_time=900 +php.max_input_time=630 +''' + self.storeExample(example) + + def extractExcluded(self, opt): + '''Sets the excluded option. + @param opt: the option value, e.g. ';*.txt;text*' + ''' + excluded = [] + if re.match(r'[*?\[\]]', opt[0]): + self.usage('wrong first char (meta character: ?*[]/) in: ' + opt) + patterns = opt[1:].split(opt[0]) + for pattern in patterns: + excluded.append((os.path.dirname(pattern), os.path.basename(pattern))) + return excluded + + def findRegion(self, regExprFirst, firstIncluding, regExprLast = None, lastIncluding = True, regExprBlock = None, onlyOneFirst = True): + '''Finds a sequence of lines with given first and last line. + @param regExprFirst: a regular expression (as string or re.RegExpr) to define the first line + @param firstIncluding: True: the first line is part of the region False: the line below the first line starts the region + @param regExprLast: a regular expression (as string or re.RegExpr) to define the last line + @param lastIncluding: True: the last line is part of the region False: the line above the last line starts the region + @param regExprBlock: None or a regular expression (as string or re.RegExpr) to define the block between start and last + regExprLast and regExprBlock are exclusive: only one should be not None + @param onlyOneFirst: False: if the regExprFirst is found it will tested not anymore + True: after finding the regExprFirst the line is tested again: if found it belongs to the block + @return: a tuple (ixStart, ixEnd) or (-1, x) if not found + ''' + if type(regExprFirst) == str: + regExprFirst = re.compile(regExprFirst) + if type(regExprLast) == str: + regExprLast = re.compile(regExprLast) + if type(regExprBlock) == str: + regExprBlock = re.compile(regExprBlock) + self._regionStart = -1 + self._regionEnd = 0 + for ix in range(len(self._lines)): + line = self._lines[ix] + if self._regionStart < 0 and regExprFirst.search(line): + self._regionStart = ix if firstIncluding else ix + 1 + elif self._regionStart >= 0: + if not onlyOneFirst and regExprFirst.search(line): + continue + if regExprLast != None and regExprLast.search(line): + self._regionEnd = ix + 1 if lastIncluding else ix + break + elif regExprBlock != None and regExprBlock.search(line) == None: + self._regionEnd = ix + 1 + break + if self._regionEnd <= 0: + self._regionEnd = len(self._lines) + self._logger.log('region {}..{}: [{},{}['.format(regExprFirst.pattern, regExprLast.pattern, self._regionStart, self._regionEnd), 3) + return self._regionStart, self._regionEnd + + def findVariables(self, prefix, configuration = None): + '''Return an array of variables defined in the internal configuration. + Example: the configuration contains 'php.max_memory=2M' + The prefix is 'php.', the result is [... 'max_memory=2M' ...] + @param prefix: the common prefix to filter the definitions in the configuration + @param configuration: None: the internal configuration is taken otherwise: the configuration to inspect + @return: an array (may be empty) with the specified variables + ''' + variables = [] + if configuration == None: + configuration = self._configuration + # mask meta characters: + # @doc re.sub(r'([\[\]{}().+*])', r'\\1', 'a.b) returns r'a\\1b' instead of r'a\.b' + pattern = re.sub(r'([\[\]{}().+*])', r'~!slash!~\1', prefix) + pattern = pattern.replace('~!slash!~', '\\') + keys = configuration.getKeys('^' + pattern) + for key in keys: + if key.startswith(prefix): + line = '{}={}'.format(key[len(prefix):], self._configuration.getString(key)) + variables.append(line) + return variables + + def grep(self, argv): + '''Search a regular expression in files matching a wildcard expression. + @param argv: the arguments, e.g. ['grep', 'jonny', *.txt' + ''' + (argv, options) = self.splitArgsAndOpts(argv) + pattern = argv[0] + filePattern = argv[1] + argv = argv[2:] + excluded = None + self._list = False + self._onlyMatching = False + self._format = None + self._out = [] + self._withLineNo = False + maxDepth = 999 + ignoreCase = False + ''' + -i or --ignore-case + ''' + for opt in options: + if opt.startswith('--excluded='): + excluded = self.extractExcluded(opt[11:]) + elif opt.startswith('--format='): + self._format = opt[9:] + if self._format == '': + self.usage('format may not be empty') + elif opt == '-i' or opt == '--ignore-case': + ignoreCase = True + elif opt == '--line-no': + self._withLineNo = True + elif opt == '--list' or opt == '-l': + self._list = True + elif opt.startswith('--max-depth='): + maxDepth = self.integerOption(opt) + elif opt == '--missing': + self._missing = True + elif opt == '--no-name': + self._noName = True + elif opt == '-o' or opt == '--only-matching': + self._onlyMatching = True + elif opt == '-v' or opt == '--reverse': + self._reverse = True + else: + self.usage('unknown option: ' + opt) + self._regSearch = self.regExprCompile(pattern, 'pattern', not ignoreCase) + if self._format != None: + self._formatGroups = re.findall(r'\$([0-9]+)', self._format) + self._formatGroups = list(map(lambda x: int(x), self._formatGroups)) + self._formatGroups.sort(reverse=True) + self.traverse(filePattern, 'grep', maxDepth, excluded) + msg = '\n'.join(self._out) + base.BaseTool.setResult(msg) + if msg != '': + print(msg) + if base.BaseTool.result() == '' and not self._exitWith0: + sys.exit(10) + + def grepOneFile(self, full): + '''Search in a given file. + @param full: the path and node + ''' + self._logger.log('inspecting {} ...'.format(full), 2) + with open(full, "rb") as fp: + lineNo = 0 + for rawline in fp: + lineNo += 1 + line = base.FileHelper.fromBytes(rawline.rstrip()) + matcher = self._regSearch.search(line) + if matcher == None: + if self._reverse: + if self._noName: + self._out.append(line) + elif self._withLineNo: + self._out.append('{}-{}: {}'.format(full, lineNo, line)) + else: + self._out.append('{}: {}'.format(full, line)) + elif self._missing: + self._out.append(full) + break + elif self._list: + self._out.append(full) + break + elif self._onlyMatching: + if self._noName: + self._out.append(line) + elif self._withLineNo: + self._out.append('{}-{}: {}'.format(full, lineNo, matcher.group(0))) + else: + self._out.append('{}: {}'.format(full, matcher.group(0))) + elif self._format != None: + theFormat = self._format.replace('!full!', full).replace('!node!', os.path.basename(full)).replace('!no!', str(lineNo)) + for groupNo in self._formatGroups: + if groupNo <= matcher.lastindex: + theFormat = theFormat.replace('$' + str(groupNo), matcher.group(groupNo)) + self._out.append(theFormat) + elif not self._missing and not self._reverse: + if self._noName: + self._out.append(line) + elif self._withLineNo: + self._out.append('{}-{}: {}'.format(full, lineNo, line)) + else: + self._out.append('{}: {}'.format(full, line)) + + def handleOneDir(self, directory, pattern, action, depth, maxDepth=999): + '''Search for files matching <pattern> and processes them depending on <action>. + @param directory: directory to process + @param pattern: only files matching this will be processed + @param action: the action, e.g. 'replace-or-insert' + @param depth: the current subdirectory nesting level + @param maxDepth: maximum subdirectory nesting level. < 0: not limited + @return: True: success False: stop the processing + ''' + rc = True + if depth == 0: + self._fileCount = 0 + self._logger.log('processing ' + directory + '/ ...', 2) + if not os.path.isdir(directory): + self.usage('not a directory: ' + directory) + elif maxDepth < 0 or depth <= maxDepth: + for node in os.listdir(directory): + full = directory + os.sep + node + try: + info = os.stat(full) + except FileNotFoundError: + self._logger.log('no permission: ' + directory + os.sep + node, 2) + continue + except OSError as exc2: + self._logger.log('os error on {}: {}'.format(directory + os.sep + node, str(exc2)), 2) + continue + if os.path.isdir(full): + if maxDepth < 0 or depth < maxDepth: + if self._dirMustBeWritable and base.LinuxUtils.isReadable(info, self._euid, self._egid): + self._logger.log('no dir write permission: ' + directory + os.sep + node, 2) + elif self.notExcluded(directory, node): + accepted = (base.LinuxUtils.isExecutable(info, self._euid, self._egid) + and base.LinuxUtils.isReadable(info, self._euid, self._egid)) + if accepted: + rc = self.handleOneDir(full, pattern, action, depth + 1, maxDepth) + self._logger.log('no permission: ' + directory + os.sep + node, 2) + elif fnmatch.fnmatch(node, pattern) and self.notExcluded(directory, node): + if not self._isRoot and self._fileMustBeWritable and not base.LinuxUtils.isWritable(info, self._euid, self._egid): + self._logger.log('no write permission: ' + directory + os.sep + node, 2) + continue + accepted = base.LinuxUtils.isReadable(info, self._euid, self._egid) + if not accepted: + self._logger.log('no permission: ' + directory + os.sep + node, 2) + else: + rc = self.handleOneFile(full, action) + if self._fileCount >= self._maxFileCount: + self._logger.log('maximal file count reached: ' + str(self._fileCount), 3) + rc = False + break + return rc + + def handleOneFile(self, filename, action): + '''Search for files matching <pattern> and processes them depending on <action>. + @param filename: the full name of the file to process + @param action: the action, e.g. 'replace-or-insert' + @return: True: success False: stop the processing + ''' + if action == 'replace-or-insert': + rc = self.replaceOrInsertOneFile(filename) + elif action == 'replace': + rc = self.replaceOneFile(filename) + elif action == 'replace-region': + rc = self.replaceRegionOneFile(filename) + elif action == 'grep': + rc = self.grepOneFile(filename) + else: + rc = False + raise Exception('handleOneFile(): unknown action ' + action) + return rc + + def notExcluded(self, path, node): + '''Tests whether a file or a directory is marked as "excluded". + @param node: the file's node + @param path: the path without node + @return True: the file is not excluded + ''' + rc = True + if self._excluded != None: + for pair in self._excluded: + rc = not (pair[0] != '' and fnmatch.fnmatch(path, pair[0]) or fnmatch.fnmatch(node, pair[1])) + if not rc: + break + if not rc: + self._logger.log('excluded: ' + path + os.sep + node, 2) + return rc + + def random(self, argv): + '''Prints a random number. + syntax: random { [ <maxExcluded> ] | <min> <maxExcluded> } <opts> + @param argv: the program arguments, e.g. ['100', '1000', '--seed=ThisIsMyPassword + ''' + argv, options = self.splitArgsAndOpts(argv) + if len(argv) == 0: + minValue, maxValue = '0', '2147483648' + elif len(argv) == 1: + minValue, maxValue = '0', argv[0] + elif len(argv) == 2: + minValue, maxValue = argv[0], argv[1] + if base.StringUtils.asInt(minValue) == None: + self._usage('minimum is not an integer: ' + argv[1]) + elif base.StringUtils.asInt(maxValue) == None: + self._usage('maximum is not an integer: ' + (argv[1] if len(argv) >= 2 else argv[0])) + random = base.CryptoEngine.CryptoEngine(self._logger) + statusFile = None + seedString = None + clearSeconds = 60 + for opt in options: + if opt.startswith('--seed='): + seedString = opt[7:] + elif opt.startswith('--status-file='): + statusFile = opt[14:] + elif opt.startswith('--clear-status-after='): + clearSeconds = self.integerOption(opt, 60) + else: + self.usage('unknown option: ' + opt) + if seedString == None: + seedString = str(time.time()) + if statusFile == None: + seedName = hex(random.hash('Hi' + random.saveSeed()))[2:] + statusFile = '{}{}texttool.{}.seed'.format(tempfile.gettempdir(), os.sep, seedName) + content = '' + if clearSeconds != 0 and os.path.exists(statusFile): + statInfo = os.stat(statusFile) + if time.time() - statInfo.st_mtime <= clearSeconds: + content = base.StringUtils.fromFile(statusFile) + if content != '': + random.restoreSeed(content) + random.setSeedFromString(seedString) + base.BaseTool.setResult(str(random.nextInt(int(maxValue), int(minValue)))) + + def readCurrent(self, full, mustExist=True): + '''Reads a given file into _lines[]. + @param full: the filename + @return: True: success + ''' + self._currentChanged = False + rc = True + self._lines = [] + self._currentFile = full + if not os.path.exists(full): + if mustExist: + self._logger.error('missing ' + full) + rc = False + else: + self._logger.log('reading {}...'.format(full), 3) + with open(full, 'rb') as fp: + self._table = [] + self._lines = [] + for rawLine in fp: + line = base.FileHelper.fromBytes(rawLine.rstrip(b'\n')) + self._lines.append(line) + return rc + + def removeRegion(self, first = None, last = None): + '''Removes the region. + @param first: None or the index of the first line None: self._regionFirst is taken + @param last: None or the index of the last line None: self._regionEnd is taken + ''' + if first == None: + first = self._regionStart + if last == None: + last = self._regionEnd + if first < 0: + first = 0 + if last >= len(self._lines): + last = len(self._lines) - 1 + if first <= last: + self._logger.log('removing lines [{}..{}[...'.format(first, last)) + self._lines = self._lines[0:first] + self._lines[last:] + + def replace(self, argv): + '''Search a regular expression in files matching a wildcard expression. + @param argv: the arguments, e.g. ['grep', 'jonny', *.txt'] + ''' + (argv, options) = self.splitArgsAndOpts(argv) + self._table = None + self._regSearch = None + self._replacement = None + self._patternSearch = None + excluded = None + maxDepth = 1024 + if len(argv) < 2: + self.usage('too few arguments') + elif argv[0].startswith('@'): + table = argv[0][1:] + self.readTable(table) + argv = argv[1:] + else: + if len(argv) < 3: + self.usage('too few arguments') + else: + self._patternSearch = argv[0] + self._regSearch = self.regExprCompile(self._patternSearch, 'pattern') + self._replacement = argv[1] + argv = argv[2:] + filePattern = argv[0] + argv = argv[1:] + if len(argv) == 0: + self._target = '!full!' + else: + self._target = argv[0] + argv = argv[1:] + for opt in options: + if opt.startswith('--excluded='): + excluded = self.extractExcluded(opt[11:]) + elif opt.startswith('--prefix-back-reference=') or opt.startswith('-p'): + cc = opt[24:] if opt[1] == '=' else opt[1:] + if cc == '': + self.usage('empty prefix is forbidden: ' + opt) + else: + self._patternReference = r'[{}]\d+'.format(cc) + elif opt.startswith('-p'): + self._regReference = re.compile(r'[{}]\d+'.format(opt[24:])) + elif opt.startswith('--count='): + self._countPerLine = self.integerOption(opt) + elif opt.startswith('--max-depth'): + maxDepth = self.integerOption(opt, 1024) + else: + self.usage('unknown option: ' + opt) + dirMustBeWritable = self._target.find('!full!') >= 0 or self._target.find('!path!') >= 0 + fileMustBeWritable = self._target == '!full!' + if self._replacement != None: + references = re.findall(self._patternReference, self._replacement) + if len(references) > 0: + self._references = list(set(map(lambda x: int(x[1:]), references))) + self._references.sort(reverse=True) + if len(self._references) == 0 or self._patternSearch.count('(') < self._references[0]: # ) + self.usage('missing "(" for groups in "{}". Highest back reference {} in "{}" has no group'.format( + self._patternSearch, 0 if len(self._references) == 0 else self._references[0], self._replacement)) + if filePattern.startswith('='): + for line in filePattern[1:].split('\n'): + self._out.append(self.replaceLine(line)) + else: + self.traverse(filePattern, 'replace', maxDepth, excluded, dirMustBeWritable, fileMustBeWritable) + msg = '\n'.join(self._out) + base.BaseTool.setResult(msg) + self._logger.log('hits: {}'.format(self._hits), 1) + if msg != '' and self._verboseLevel > 0: + print(base.BaseTool.result()) + if msg == '' and not self._exitWith0: + sys.exit(10) + + def replaceLine(self, line): + '''Handle replacement for one line. + @param line: the input line to inspect + @param fpOut: the filehandle of the target file + @return: the (possible changed) line + ''' + outline = line + count = 0 + pos = 0 + if self._regSearch != None: + again = True + while again: + again = False + matcher = self._regSearch.search(outline, pos) + if matcher != None: + self._hits += 1 + replacement = self._replacement + for group in self._references: + try: + pattern = '${}'.format(group) + except IndexError: + self.usage('too few "(" for groups in search pattern.') + repl = matcher.group(group) + replacement = replacement.replace(pattern, repl) + outline = outline[0:matcher.start()] + replacement + outline[matcher.end():] + pos = matcher.end() + count += 1 + again = self._countPerLine == None or count < self._countPerLine + else: + for (key, replacement) in self._table: + pos = 0 + count = 0 + again = True + while again: + again = False + ix = outline.find(key, pos) + if ix < 0: + again = False + else: + self._hits += 1 + outline = outline[0:ix] + replacement + outline[ix + len(key):] + pos = ix + len(replacement) + count += 1 + again = self._countPerLine == None or count < self._countPerLine + return outline + + def replaceOrInsert(self, argv): + '''Replaces or inserts a given line into files. + syntax: replace-or-insert <line-pattern> <line> <file-pattern> [<opts>] + @param argv: the command arguments, e.g. [r'count\s*=', 'count=1', '/etc/pyrshell/dirs.d/*.conf', r'--below-anchor=\[special\]'] + --count-files=<count> + ''' + argv, options = self.splitArgsAndOpts(argv) + if len(argv) < 3: + self.usage('missing arguments') + else: + pattern = self.regExprCompile(argv[0], '<line-pattern>') + line = argv[1] + filePattern = argv[2] + anchor = None + above = False + isOK = True + maxDepth = 999 + excluded = None + argv = argv[2:] + self._maxFileCount = 0x7ffffff + createIfMissing = False + for opt in options: + if opt.startswith('--above-anchor='): + anchor = self.regExprCompile(opt[15:], 'above anchor') + above = True + elif opt.startswith('--excluded='): + excluded = self.extractExcluded(opt[11:]) + elif opt.startswith('--below-anchor='): + anchor = self.regExprCompile(opt[15:], 'below anchor') + elif opt.startswith('--file-count'): + self._maxFileCount = self.integerOption(opt) + if self._maxFileCount < 0: + isOK = False + break + elif opt.startswith('--max-depth'): + maxDepth = self.integerOption(opt) + if maxDepth < 0: + isOK = False + break + elif opt == '--create-if-missing': + createIfMissing = True + else: + self.usage('unknown option: ' + opt) + self._replaceInfo = ReplaceInfo(pattern, line, anchor, above, None) + if isOK: + if not base.FileHelper.hasWildcards(filePattern): + self._maxDepth = 0 + if createIfMissing: + base.FileHelper.ensureFileExists(filePattern, '', self._logger) + self.traverse(filePattern, 'replace-or-insert', maxDepth, excluded, False, True) + + def replaceOrInsertOneFile(self, filename): + '''Handles one file for the command 'replace-or-insert'. + @param filename: the file to process + @return: True: success False: stop processing + ''' + rc = True + if self.currentSimpleInsert(filename, self._replaceInfo._pattern, self._replaceInfo._line, + self._replaceInfo._anchor, not self._replaceInfo._above): + rc = True + self._fileCount += 1 + return rc + + def replaceOneFile(self, full): + '''Handles one file for the command 'replace'. + @param full: the filename to process + @return True: success False: stop processing + ''' + rc = True + target = self.buildTargetname(full) + changed = False + asBinary = self._table != None + with open(full, 'rb') as fp, open(target, 'wb'if asBinary else 'w') as fpOut: + self._logger.log('inspecting {} ...'.format(full), 3) + lineNo = 0 + for rawLine in fp: + if asBinary: + line = rawLine + else: + line = base.FileHelper.fromBytes(rawLine) + line = line.rstrip(b'\n' if asBinary else '\n') + lineNo += 1 + out = self.replaceLine(line) + if line != out: + changed = True + fpOut.write(out + (b'\n' if asBinary else '\n')) + if not changed: + self._logger.log('leaving unchanged: ' + full, 4) + else: + self._logger.log('replaced {} hit(s) in {}'.format(self._hits, full), 4) + if self._target == '!full!': + os.unlink(full) + os.rename(target, full) + self._fileCount += 1 + return rc + + def replaceRegion(self, replacement, regExprAnchor = None): + '''Replaces the last found region with the given text. + @param replacement: the region to replace + @param regExprAnchor: None or a regular expression (text or re.RegExpr) for the line to insert if no region has been found + ''' + lines = replacement.split('\n') + if self._regionStart < 0: + ix = None + if type(regExprAnchor) == str: + regExprAnchor = re.compile(regExprAnchor) + ix = self.currentFind(regExprAnchor) + if ix == None: + self._lines += lines + else: + self._lines = self._lines[0:ix] + lines + self._lines[ix+1:] + self._currentChanged = True + elif replacement != '\n'.join(self._lines[self._regionStart:self._regionEnd]): + self._currentChanged = True + self._lines = self._lines[0:self._regionStart] + lines + self._lines[self._regionEnd:] + self._logger.log('replacing with {} line(s)'.format(replacement.count('\n') + 1), 3) + else: + self._logger.log('replacement is equal to region', 3) + + def replaceRegionOneFile(self, full): + '''Handles one file for the command 'replace'. + @param full: the filename to process + @return True: success False: stop processing + ''' + rc = True + info = self._regionReplaceInfo + self.readCurrent(full, True) + self.findRegion(info._regionStart, info._startIncluded, info._regionEnd, info._endExcluded, None, True) + self.replaceRegion(info._replacement) + self.writeCurrent(full) + self._fileCount += 1 + return rc + + def replaceRegionCmd(self, argv): + '''Replaces a region with a replacement. + syntax: replace-region <start> <end> <replacement> <file-pattern> [<opts>] + @param argv: the command arguments, e.g. ['<body>', '</body>', '<p>site locked</p>', 'index.php', '--start-excluded', ] + ''' + argv, options = self.splitArgsAndOpts(argv) + if len(argv) < 4: + self.usage('missing arguments') + else: + self._regionReplaceInfo = info = ReplaceRegionInfo() + info._regionStart = self.regExprCompile(argv[0], '<pattern-start>') + info._regionEnd = self.regExprCompile(argv[1], '<pattern-start>') + info._replacement = argv[2] + filePattern = argv[3] + excluded = None + maxDepth = None + for opt in options: + if opt == '--start-excluded': + info._startIncluded = False + elif opt == '--end-included': + info._endExcluded = True + elif opt.startswith('--max-depth='): + maxDepth = self.integerOption(opt) + elif opt.startswith('--excluded='): + excluded = self.extractExcluded(opt[11:]) + else: + self.usage('unknown option: ' + opt) + self.traverse(filePattern, 'replace-region', maxDepth, excluded, True, True) + + def readTable(self, full): + '''Reads the file with the key/replacement strings. + @param full: the filename + ''' + with open(full, 'rb') as fp: + self._table = [] + for line in fp: + line = line.rstrip(b'\n') + pair = line.split(b'\t', 1) + self._table.append(pair) + + def writeCurrent(self, full=None, forceWrite=False): + '''Reads a given file into _lines[]. + @param full: the filename + ''' + if forceWrite or self._currentChanged: + if full == None: + full = self._currentFile + with open(full, 'w') as fp: + self._table = [] + for line in self._lines: + fp.write(line + '\n') + self._logger.log('writing {}...'.format(full), 3) + + def traverse(self, fullPattern, action, maxDepth, excluded=None, dirMustBeWritable=False, fileMustBeWritable=False): + '''Traverses the directory tree and do the given action. + @param fullPattern: only files matching this will be processed. + If a path is given that is the start directory. Otherwise the current directory is used + @param action: the action, e.g. 'replace-or-insert' + @param depth: the current subdirectory nesting level + @param maxDepth: maximum subdirectory nesting level + @param excluded: None or an array of patterns for file exclusion + @param dirMustBeWritable: True the directory may be changed + @param fileMustBeWritable: True the file may be changed + ''' + self._excluded = excluded + self._dirMustBeWritable = dirMustBeWritable + self._fileMustBeWritable = fileMustBeWritable + startDir = os.path.dirname(fullPattern) + if maxDepth == None: + maxDepth = -1 + self.handleOneDir(startDir if startDir != '' else '.', os.path.basename(fullPattern), action, 0, maxDepth) + +def usage(): + '''Returns an info about usage. + ''' + return r"""usage: texttool [<global_opts>] <command> [<opts>] [<params>] + Text manipulation tool +<input>: file to manipulate +GLOBAL_OPTS +GLOBAL_MODES +<command>: + adapt[-configuration] <configuration-file> <variable-file> + Sets specified variables in a configuration file to given values. + The variables are defined in a separate file. + <configuration-file> + the file to change + <variable-file> + the file containing the values to change. One variable per line, e.g. 'verbose=True' + adapt[-configuration] <configuration-file> prefix=<prefix> + Sets specified variables in a configuration file to given values. + The variables are defined in the texttool configuration file. + <configuration-file> + the file to change + <prefix> + the prefix to filter the keys in the internal configuration file, e.g. 'php.' + build-examples + Populates the example directory: Than the below examples can be used unmodified. + e(xecute) <statements> [<input>] + Executes the statements on the file <input>. + If <input> is missed the statements must contain a "read <file>" statement. + If <input> is '-' the input is read from stdin. + grep <pattern> <file-pattern> [<opts>] + Search the <pattern> (a regular expression) in all files matching <file-pattern> + <file-pattern> + file name pattern, with wilcards *, ? [chars] and [!not chars] + <opts>: + --excluded=<pattern> + shell wildcard pattern for excluded files and directories, e.g. ":.git:./tmp/*.bak" + --format=<pattern> + prints the format: Wildcards: $1 first group (delimited by a parentesis pair). First from left + placeholders: !full! (path and node), !node! (node only) '!no! (line number), e.g. "!full!-!no!: $1 $3" + -i or --ignore-case + the search is case insensitive + --line-no + the line number is displayed (behind the filename) + -l --list + prints the filename only + --max-depth=<depth> + the maximum nesting level of the directory searched in + --missing + lists the full filenames not containing <pattern> + --no-name + the filename (and linenumber) is not displayed on top of the line + -o or --only-matching + prints the matching string only + -v or --reverse + prints lines not matching the given regular expression + exit code is 10 if no hit is found + python-to-c <python-source> <c-target> + Converts python source code into C like syntax (C++, C, TypeStript. This is a 80% tool: many corrections must be done manually. + <python-source>: the file with the python source code + <c-target>: the file with the C like syntax + replace {} <pattern> <replacement> | @<table> {} <input> [<target-pattern>] [<opts>] + Search the <pattern> (a regular expression) in all files matching <file-pattern> and replace it by a given string + <pattern> + a regular expression + <replacement> + may contain $<n> for back references ($1 replaces group 1...) + @<table> + a text file containing the strings and replacements separated by a TAB, one pair per line, e.g. "@table.txt" + <input> + '='<string>: the input is given as string, e.g. "=Needle" + file name pattern, with wilcards *, ? [chars] and [!not chars]. + If the file name pattern starts with '=', take '[=]' for distinction + <target-pattern> + if given the name of the target file, can contain macros !full!, !path!, !node!, !fn!, !ext! + e.g. "!path!/!fn!.replaced!ext!" + example: !full!: /tmp/abc.def !path!: /tmp/ !node!: abc.def !fn!: abc !ext!: .def + default: !full! (replaces the source file) + <opts>: + --excluded=<pattern> + shell wildcard pattern for excluded files and directories, e.g. ":.git:./tmp/*.bak" + --max-depth + 0: only the directory given by <file-pattern> will be processed. + otherwise: the maximal nesting level of subdirectories to be processed + exit code is 10 if no hit is found + replace-or-insert <line-pattern> <line> <file-pattern> [<opts>] + Search for the regular expression <line-pattern> in files. + If found the place is replaced by <line>. + Else the <anchor> is searched. If found the line is insert + <line-pattern>: + a regular expression defining the line. Should contain content from <line> + <line>: + precondition: this line is in the content of the files + <file-pattern>: + a file pattern with wildcards '*' and '?'. All files matching this pattern will be processed. In all subdirectories too + <opts>: + --above-anchor=<anchor> + if <line-pattern> is not found the line is inserted above the first line containing this regular expression + --below-anchor=<anchor> + if <line-pattern> is not found the line is inserted below the first line containing this regular expression + --create-if-missing + if <file-pattern> does not exists and the name does not contain wildcards it will be created + --excluded=<pattern> + shell wildcard pattern for excluded files and directories, e.g. ":.git:./tmp/*.bak" + --file-count=<count> + only <count> files will be processed + --max-depth + 0: only the directory given by <file-pattern> will be processed. + otherwise: the maximal nesting level of subdirectories to be processed + replace-region <pattern-start> <pattern-end> <replacement> <file-pattern> [<opts>] + Replace a region (some sequential lines) in a file + <pattern-start> + a regular expression describing the start of the region (this line is part of the region) + if not found nothing is replaced + <pattern-end> + a regular expression describing the end of the region (this line is not part of the region) + if this pattern is not found the file end is the region end + <replacement> + the replacement text: will be inserted instead of the old region + <file-pattern> + this files will be inspected + <opts>: + --start-excluded + <pattern-start> is not part of the region + --end-included + <pattern-end> is part of the region + --max-depth=<max-level> + maximum nesting level + --excluded=<pattern> + file name pattern to exclude, e.g.':*.txt:*.bak' + random { [ <maxExcluded> ] | <min> <maxExcluded> } <opts> + Prints a pseudo random number between 0 or <min> (inclusive) and <maxExcluded> or 2**31 (excluded). + If <seed> is not given a time related seed is taken. Otherwise the rand generator starts with the given seed. + <opts>: + --seed=<seed> + the rand generator starts with the given seed. + --status-file=<path> + the status of the random generator is stored in this file. Default: /tmp/texttool.<seed>.seed or /tmp/texttool.default.seed + script <script> [<input>] + Executes the statements in the file <script> on the file <input> + If <input> is missed the statements must contain a "read <file>" statement. + If <input> is '-' the input is read from stdin. + if <script> is '-' the script is read from stdin. +Example: +texttool build-examples +texttool -v3 adapt /usr/share/pyrshell/examples/data/php.ini /usr/share/pyrshell/examples/config/php_minimal.conf +texttool adapt-configuration /usr/share/pyrshell/examples/data/php.ini prefix=php. +texttool execute 'fb "[.]";p "host: ";p20' /etc/hostfile +texttool --log=/var/log/general.log script build_summary.txt calculation.csv +texttool grep '\bopen\b.*rb' '/usr/share/pyrshell/base/*.php' '--excluded=:*file*:*text*' +texttool grep 'total:\s+([0-9.]+)' /usr/share/pyrshell/examples/data/sum.txt --line-no '--format=Sum: $1' +texttool grep StringUtils '/usr/share/pyrshell/*.py' --missing --list +ip | texttool grep "^(\d: \w+)|(\s*inet [0-9.]+" --only-matching +texttool grep 'total:' /usr/share/pyrshell/data/sum.txt --reverse +texttool grep -i --no-name -v jonny /usr/share/pyrshell/examples/data/first.addr +texttool python-to-c /usr/share/pyrshell/base/ThreadLogger.py /tmp/stringutils.ts +texttool replace 'DATE=\S+' 'DATE=1.7.2019' /usr/share/pyrshell/examples/today.sh +texttool replace jonny@gmx.de mr.universe@gmx.de '/usr/share/pyrshell/examples/data/*.sql' '!path!universe_!node!.out' '--excluded=:shop.sql:test*.sql --max-depth=0' +texttool replace @table.changes /db/old.sql /db/new.sql +texttool replace hot cool '=should be hot' +texttool replace @table.txt "*.txt" "!full!.new" --excluded=.git;*draft* +texttool replace-or-insert '^directory\s*=' "directory=/etc" /usr/share/pyrshell/examples/inf.de.conf '--below-anchor=^\[files\]$' --create-if missing +texttool replace-region '<body>' '</body>' '<p>no access!</p>' index.html --start-excluded --max-depth=0 +texttool replace-region '<h[1-9]>' '</p>' '<p>no access!</p>' index.html --end-included +""" + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/texttool', 'run'] + ''' + appInfo = base.BaseTool.ApplicationInfo('texttool', 'appl/TextTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = TextTool(options) + rc = None + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'adapt-configuration' or cmd == 'adapt': + rc = tool.adaptConfiguration(argv) + elif cmd == 'build-examples' or cmd == 'adapt': + rc = tool.buildExamples() + elif cmd == 'execute' or cmd == 'e': + # rc = tool.execute(argv) + tool._logger.log('not implemented: ' + cmd) + elif cmd == 'grep': + if len(argv) < 2: + tool.usage('too few arguments') + rc = tool.grep(argv) + elif cmd == 'python-to-c': + if len(argv) < 2: + tool.usage('missing arguments') + else: + converter = base.PythonToTypeScript.PythonToTypeScript(tool, TextTool(options)) + tool.readCurrent(argv[0], True) + converter.convert() + converter._typeScript.writeCurrent(argv[1], True) + elif cmd == 'random': + tool.random(argv) + elif cmd == 'replace': + rc = tool.replace(argv) + elif cmd == 'replace-or-insert': + rc = tool.replaceOrInsert(argv) + elif cmd == 'replace-region': + rc = tool.replaceRegionCmd(argv) + elif cmd == 'script': + rc = tool.script(argv) + elif cmd == 'manual': + rc = tool.manual(argv) + else: + tool.usage("unknown command: " + cmd) + base.BaseTool.setLatestTool(tool) + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/UrlChecker.py b/appl/UrlChecker.py new file mode 100755 index 0000000..a855293 --- /dev/null +++ b/appl/UrlChecker.py @@ -0,0 +1,357 @@ +#! /usr/bin/python3 +''' +Created on 07.05.2019 + +@author: hm +''' +import sys +import os.path +import time +import re +import datetime +sys.path.insert(0, '/usr/share/pyrshell') +import base.JavaConfig +import net.HttpClient +import base.BaseTool +import net.EMail + +class UrlChecker (base.BaseTool.BaseTool): + + def __init__(self, globalOptions): + '''Constructor. + @param logger: the logger + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'urlchecker.conf', False, 'urls.d') + self._client = net.HttpClient.HttpClient(self._verboseLevel, self._logger) + self._urlInfo = [] + self._indexUrlInfo = 0 + self._data = None + self._urlError = [] + self._rexprEmail = re.compile(r'^[-.\w]+@[-.\w]+?\.[a-zA-Z]+$') + + def check(self): + '''Reads one URL configuration file. + @param filename: the name of the file, e.g. '/etc/pyrshell/urls.d/wiki.url.conf' + @return: [url, indication] url: the URL which has been checked + indication: None: site online otherwise: the offline indication message + ''' + currentIndex = self._indexUrlInfo + current = self._urlInfo[currentIndex] + self._indexUrlInfo = (currentIndex + 1) % len(self._urlInfo) + url = current.getString('url') + self._data = None + indication = None + data = self._client.getContent(url) + if data == None: + indication = 'not reachable' + no = 1 + while indication == None and no > 0: + needle = current.getString('needle.{}'.format(no)) + if needle == None: + no = 0 + else: + if data != None and not data.decode('utf-8').find(needle) >= 0: + indication = 'missing marker in content: ' + needle + else: + no += 1 + + no = 1 + while indication == None and no > 0: + needle = current.getString('regexpr.{}'.format(no)) + if needle == None: + no = 0 + else: + if data != None and re.match(needle, data.decode('utf-8')) == None: + indication = 'missing regular expression in content: ' + needle + else: + no += 1 + return [url, indication] + + def close(self): + '''Frees the resources. + ''' + if self._client != None: + self._client.close(self) + self._client = None + + def configFromNginx(self, argv): + '''Builds configuration files from NGINX configuration. + @param argv: arguments, e.g. '[ '/etc/nginx/sites-enabled', 'bigtoy', /opt/new-config'] + ''' + if len(argv) < 2: + self.usage('missing arguments') + else: + srcDir = argv[0] + trgDir = argv[2] if len(argv) > 2 else self._additionalConfigDir + if not os.path.exists(srcDir) or not os.path.isdir(srcDir): + self.usage('not a directory: ' + srcDir) + elif not os.path.exists(trgDir) or not os.path.isdir(trgDir): + usage('not a directory: ' + trgDir) + else: + remoteHost = argv[1] + rexprHost = re.compile(r'^\s*server_name\s+([-.\w]+)') + for node in os.listdir(srcDir): + full = srcDir + os.sep + node + if os.path.isdir(full): + continue + lines = base.BaseTool.BasicStatics.grep('server_name', full) + matcher = rexprHost.match(lines[0]) + if matcher != None: + domain = matcher.group(1) + self.createConfig(domain, remoteHost, trgDir) + + def createConfig(self, domain, remoteHost, trgDir): + '''Creates a configuration file. + @param domain: the domain of the website + @param remoteHost: this server hosts the website + @param trgDir: target directory + ''' + full = trgDir + os.sep + domain + '-' + remoteHost + '.conf' + url = self._client.getRealUrl('http://{}'.format(domain)) + content = self._client.getContent(url, 1) + with open(full, "w") as fp: + fp.write('# created by urlchecker\n') + fp.write('server={}\n'.format(remoteHost)) + fp.write('domain={}\n'.format(domain)) + fp.write('url={}\n'.format(url)) + fp.write('#needle.1=<body>\n') + fp.write('#rexpr.1=[a-z]+\n') + if content != None: + fp.write('content={}\n'.format(content.decode('utf-8').replace('\\n', '\n'))) + + def example(self): + text = '''# webdashserver example configuration +log.file=/var/log/local/urlchecker.log +# used as sender info in the email +service.host={} +# all URL checks will be done in interval (in seconds) +checker.interval=3600 +# == Email data for error report: == +send.always=False +# Receiver of the error messages: may be a blank separated list of email addresses +admin.email=hm.neutral@gmx.de +smtp.host=smtp.gmx.de +smtp.port=587 +smtp.sender=hm.neutral@gmx.de +smtp.user=hm.neutral@gmx.de +smtp.code=sEcReT +smtp.tls=True +'''.format(base.BaseTool.BasicStatics.hostname(True)) + self.storeExample(text) + + def isEmail(self, email): + '''Tests whether a string is an email address. + @param email: string to test + @return True: email is correct + ''' + rc = self._rexprEmail.match(email) != None + return rc + + def readConfigs(self): + '''Reads all configuration files. + ''' + def sumLengths(array): + rc = 0 + for ix in range(len(array)): + rc += len(array[ix]) + return rc + path = self._configuration.getString('configuration.path', '/etc/pyrshell/urls.d') + if not os.path.exists(path): + self._usage('configuration path not found: ' + path) + else: + files = os.listdir(path) + hosts = [] + arrays = [] + rexprFile = re.compile(r'.*-(\w+)\.conf$') + for aFile in files: + if aFile.endswith('.conf'): + matcher = rexprFile.match(aFile) + host = 'default' if matcher == None else matcher.group(1) + try: + ix = hosts.index(host) + except ValueError as exc: + ix = len(hosts) + hosts.append(host) + arrays.append([]) + + full = path + os.sep + aFile + config = base.JavaConfig.JavaConfig(full, self._logger) + if config.getString('url') == None: + self._logger.error('{}: missing entry url=<url>. File ignored'.format()) + else: + arrays[ix].append(config) + # self._urlError.append(None) + counts = [] + for ix in range(len(arrays)): + counts.append(len(arrays[ix])) + totalCount = sumLengths(arrays) + if totalCount == 0: + self.usage('no valid configuration found: no URL specified') + else: + countHosts = len(hosts) + while sumLengths(arrays) > 0: + # from all hosts the first entry + for ix in range(countHosts): + if len(arrays[ix]) > 0: + self._urlInfo.append(arrays[ix][0]) + del arrays[ix][0] + self._urlError.append(None) + # from all hosts the rest of the entries + for ix in range(countHosts): + for ix2 in range(totalCount // countHosts - 1): + if len(arrays[ix]) > 0: + self._urlInfo.append(arrays[ix][0]) + del arrays[ix][0] + self._urlError.append(None) + for info in self._urlInfo: + self._logger.debug('{}: {} {}'.format(ix, info.getString('server', '?'), info.getString('url'))) + + def sendStatusEmail(self, subject, message): + '''Sends one or many emails. + @param message: the message to send + @param subjectSuffix: None or the end of the email subject + @param isProblem: True: the errors have been started False: the errors have been finished + ''' + text = message + myHost = self._configuration.getString('service.host') + email = net.EMail.EMail(myHost + ': ' + subject, text) + receipients = self._configuration.getString('admin.email').split(' ') + host = self._configuration.getString('smtp.host') + port = self._configuration.getInt('smtp.port', 0) + sender = self._configuration.getString('smtp.sender') + user = self._configuration.getString('smtp.user') + code = self._configuration.getString('smtp.code') + withTls = self._configuration.getString('smtp.tls') == 'True' + if not (self.isEmail(receipients[0]) and host and port > 0 and port < 0xffff and self.isEmail(sender) and code): + self._logger.error('wrong smtp data: receiver: {} host: {} port: {} user: {} sender: {}'.format(receipients[0], host, port, user, sender)) + else: + email.setSmtpLogin(host, port, user, code, withTls) + cc = None if len(receipients) < 2 else receipients[1:] + email.sendTo(sender, receipients[0], cc) + self._logger.debug('email sent to ' + receipients[0]) + + def sendEmailStatusChange(self, url, indication, timeOffline = None): + '''Sends one or many emails. + @param url: the error status of this url has been changed + @param indication: None: the status has changed to online + Otherwise: the indication of the offline state + @param timeOffline: None: the status has changed to offline + Otherwise: the status has changed to online. The offline status has started at this time + ''' + if timeOffline == None: + subject = 'website offline ' + url + message = 'Indication: ' + indication + else: + subject = 'website again online ' + url + message = 'The website {} was offline since {}'.format(url, timeOffline.format('%Y.%m.%d %H:%M')) + self.sendStatusEmail(subject, message) + + def sendEmailOnline(self, url, timeOffline): + '''Sends an email because the website is online again. + @param url: the error status of this url has been changed + @param timeOffline: the offline status has started at this time + ''' + + subject = 'website again online ' + url + message = 'The website {} is online again.\nOffline: {} - {}'.format( + url, timeOffline.strftime('%Y.%m.%d %H:%M'), datetime.datetime.now().strftime('%Y.%m.%d %H:%M')) + self.sendStatusEmail(subject, message) + + def sendEmailOffline(self, url, indication): + '''Sends one or many emails. + @param url: the error status of this url has been changed + @param indication: None: the status has changed to online + Otherwise: the indication of the offline state + @param timeOffline: None: the status has changed to offline + Otherwise: the status has changed to online. The offline status has started at this time + ''' + subject = 'website offline ' + url + message = 'Indication: ' + indication + self.sendStatusEmail(subject, message) + + def service(self): + """Starts the TCP server and wait for tasks. + """ + interval = self._configuration.getInt('checker.interval', 3600) + slice = interval // len(self._urlInfo) + self._logger.debug("service: domains: {} interval: {} sec slice: {} sec".format(len(self._urlInfo), interval, slice)) + while True: + currentIndex = self._indexUrlInfo + offlineStart = self._urlError[currentIndex] + [url, indication] = self.check() + if indication == None: + if offlineStart != None: + self.sendEmailOnline(url, offlineStart) + elif offlineStart == None: + if indication == 'not reachable': + self.sendEmailOffline(url, indication) + else: + self.sendStatusEmail('website content changed: ' + url, indication) + self._urlError[currentIndex] = datetime.datetime.now() + time.sleep(interval) + + self.close() + def testEmail(self): + '''Sends a test email. + ''' + self.sendStatusEmail("test email", "Success") + +def usage(): + '''Returns an info about usage + ''' + return """usage: urlchecker [<global_opts>] <command> + Checks whether a amount of URLs are reachable by http(s). +GLOBAL_OPTS +GLOBAL_MODES +<command>: + test + send a test email + service or daemon + starts the monitoring daemon + config-from-nginx <nginx-dir> [<target-dir>] + create configuration files from all sites defined in a nginx configuration, e.g. /etc/nginx/sites-enabled + <target-dir>the created files will be stored there. Default: /etc/pyrshell/urls.d + Note: <nginx-dir> should be imported from the observed website + example + Creates an example configuration + service + Starts the TCP server +Examples: +urlchecker -q install +urlchecker -v3 service +""" + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/urlchecker', 'service'] + ''' + serviceInfo = base.BaseTool.ServiceInfo('urlchecker', 'www-data', 'www-data', + 'A monitor observing websites', 'CHECKER', None) + appInfo = base.BaseTool.ApplicationInfo('urlchecker', 'appl/UrlChecker.py', usage, serviceInfo) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = UrlChecker(options) + (cmd, argv) = tool.handleStandardCommands(argv) + rc = None + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + if cmd == 'service' or cmd == 'daemon': + tool.readConfigs() + rc = tool.service() + elif cmd == 'test': + rc = tool.testEmail() + elif cmd == 'config-from-nginx': + rc = tool.configFromNginx(argv) + elif cmd == 'reload': + # rc = tool.service() + pass + else: + tool.usage('unknown command: ' + cmd) + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(['dummy', 'daemon']) + #main(['dummy', 'config-from-nginx', '/opt/nginx', 'bear']) diff --git a/appl/WebDashClient.py b/appl/WebDashClient.py new file mode 100755 index 0000000..e4c9d0f --- /dev/null +++ b/appl/WebDashClient.py @@ -0,0 +1,426 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' + +import time +import sys +import os +import re +import datetime +import random + +sys.path.insert(0, '/usr/share/pyrshell') +import base.StringUtils +import base.FileHelper +import base.BaseTool +import net.TcpClient +import base.LinuxUtils + +class TaskInfo: + + def __init__(self, name, interval, parent): + '''Constructor. + @param name: the task's name + @param interval: the repeating interval, will be multiplied by the time slice + ''' + self._name = name + self._parent = parent + if interval <= 0 or interval > 3600: + self._parent._logger.error('invalid interval in {}: {}'.format(name, interval)) + self._interval = interval + self._current = random.randint(0, interval) + + def next(self): + '''Tests whether the task should be triggered. + @return True: the task should be triggered + ''' + rc = False + self._current += 1 + if self._current >= self._interval: + self._current = 0 + rc = True + self._parent._logger.log('{}: current/interval: {}/{} rc: {}'.format(self._name, self._current, self._interval, 'T' if rc else 'F'), 4) + return rc + +class WebDashClient (base.BaseTool.BaseTool): + + def __init__(self, globalOptions): + '''Constructor. + @param globalOptions: the basic configurations + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'webdashclient.conf') + self._connection = None + self._lastStress = None + self._loopNo = 0 + # msec since epoch + self._timeLastStress = None + self._hostName = self._configuration.getString('host.name', '?') + if self._hostName == '?': + self._logger.error('missing host.name in configuration: ' + self._configuration._filename) + tasks = self._configuration.getString('tasks').strip().split(':') + rexprTasks = re.compile(r'^(filesystems?|stress(?:es)?|users?|clouds?)-(\d+)$') + self._tasks = [] + for task in tasks: + matcher = rexprTasks.match(task) + if matcher == None: + self._logger.error('unknown task or syntax error (<name>-<interval>): ' + task) + else: + info = TaskInfo(matcher.group(1), int(matcher.group(2)), self) + self._logger.log('adding task {} ({} sec)...'.format(info._name, info._interval), 2) + self._tasks.append(info) + + def close(self): + '''Frees the resources. + ''' + self.disconnect() + + def connect(self): + '''Connect to the server. + @return: None: success otherwise: error message + ''' + rc = None + host = self._configuration.getString('server.host', '127.0.0.2') + port = self._configuration.getInt('server.port', 58200) + self._logger.log('connecting to {}:{}'.format(host, port), 2) + again = True + start = time.time() + while again: + again = False + try: + self._connection = net.TcpClient.TcpClient(host, port, self._logger) + except: + again = True + time.sleep(5) + self._logger.log('connection found after {:.1f} second(s)'.format(time.time() - start), 2) + return rc + + def createConfigExample(self): + '''Creates a example configuration file + ''' + self.ensureDirectory(self._configDir) + filename = self._configDir + os.sep + 'webdashclient.conf' + # to avoid errors a dummy could be created earlier: + if base.StringUtils.hasContent(filename, '# dummy created'): + filename = self._configDir + os.sep + 'WebDashClient.example' + base.StringUtils.toFile(filename, '''# WebDashClient example configuration +log.file=/var/log/local/WebDashClient.log +server.port=58201 +server.host=127.0.0.1 +io.pattern=^(sda|sdb)$ +net.pattern=^(eth0|wlan0)$ +#host.name=caribou +# time slice in seconds +slice=60 +# possible tasks: filesystems processes users clouds +tasks=filesystems-5:stress-2 +#fs.exclude=/media/dummy:/media/dummy2 +''') + self._logger.log('created: ' + filename) + + def disconnect(self): + '''Frees the resources. + ''' + if self._connection != None: + self._connection.close() + self._logger.log('disconnected', 2) + self._connection = None + + def example(self): + text = '''# WebDashClient example configuration +log.file=/var/log/local/WebDashClient.log +server.port=58201 +server.host=127.0.0.1 +#host.name=caribou +io.pattern=^sd[ab]$ +net.pattern=^ens +#cloud.basedir=/media/clouds +#cloud.excluded=/:dummy +# time slice in seconds: the base unit of the task handling intervals +slice=60 +# possible tasks: filesystems processes users clouds +tasks=filesystems-5:stress-2 +#fs.excluded=/media/dummy:/media/dummy2 +''' + self.storeExample(text) + + def task(self, task, inLoop=False): + '''Executes the tasks. + @param task: name of the task to execute + @param inLoop: True: the task is executed multiple times + @return: None: success otherwise: error message + ''' + rc = None + if task == 'filesystems': + self.taskFileSystem() + elif task == 'stress': + if not inLoop: + self.taskStress() + time.sleep(1) + self.taskStress() + elif task == 'users': + self.taskUsers() + elif task == 'processes': + self.taskProcesses() + elif task == 'clouds': + self.taskClouds() + else: + rc = "unknown task: " + task + return rc + + def taskFileSystem(self): + '''Find out the data of the filesystems and send it. + @return: None: success otherwise: error message + ''' + self._logger.log('taskFileSystem()...', 4) + rc = None + infoList = base.LinuxUtils.diskFree() + excludes = ':' + self._configuration.getString('fs.excluded', '') + ':' + self._logger.log('taskFileSystem(): found: {} items excluded: {}'.format(len(infoList), excludes), 2) + for info in infoList: + # [path, stat.f_blocks*blocksize, stat.f_bfree*blocksize, stat.f_favail*blocksize] + ix = excludes.find(':' + info[0] + ':') + if ix < 0: + total = int(info[1]) + free = int(info[3]) + used = total - int(info[2]) + args = [self._hostName, info[0], str(total), str(used), str(free)] + self._logger.log('taskFileSystem(): ' + '\n'.join(args), 3) + answer = self._connection.sendAndReceive('filesystems', args) + self._logger.log('answer: ' + answer, 3) + else: + self._logger.log('taskFileSystem(): excluded: ' + info[0], 3) + return rc + + def cloudStatistic(self, path): + '''Detects the number of files/dirs, the youngest and the largest files of the cloud. + @param path: the full path of the directory containing the "data" directory + @return an array [files, dirs, size, trashsize, the statistic info as text] + ''' + info = self._processHelper.executeInput(['hmdu', path, 'data', 'files_trashbin'], False) + parts = info.split(' ') + files = int(parts[2]) + dirs = int(parts[4]) + size = int(parts[5].replace('.', '')) + trashSize = int(parts[7].replace('.')) + info = 'Dateien: {} / {} Platzbedarf: {} / {} Verzeichnisse: {} / {}'.format(files, parts[4], size, + trashSize, parts[10], parts[12]); + return [files, dirs, size, trashSize, info] + + def cloudTrash(self, path): + '''Detects the used spaces of all trashes of the cloud. + @param path: the full path of the directory containing the "data" directory + @return [size, dirs, files] + ''' + rc = None + path += os.sep + 'data' + if not os.path.isdir(path): + self._logger.error('is not a directory: ' + path) + else: + # data/kawi/files_trashbin + users = os.listdir(path) + for user in users: + directory = path + os.sep + user + os.sep + 'files_trashbin' + if os.path.isdir(directory): + rc = base.FileHelper.directoryInfo(directory, None, None, -1, rc) + if rc == None: + rc = [0, 0, 0] + else: + rc = [rc._fileSizes, rc._dirCount, rc._fileCount] + return rc + + def cloudLogs(self, path, maxCount=10): + '''Returns the last lines from the logfile. + @param path: the full path of the directory containing the "data" directory + @return the found logfiles, string encoded: '\n' => '\\n'', '\t' => '\\t'' + ''' + lines = '' + path += os.sep + 'data' + os.sep + 'nextcloud.log' + if not os.path.exists(path): + self._logger.error('missing logfile: ' + path) + else: + # data/kawi/files_trashbin + lines = base.FileHelper.tail(path, maxCount, True) + if lines != None: + lines = base.StringUtils.escChars('.'.join(lines)) + else: + lines = '' + return lines + + def cloudUsers(self, path): + '''Detects the usernames of the cloud. + @param path: the full path of the directory containing the "data" directory + @return a list of '|' separated users + ''' + users = '' + path += os.sep + 'data' + if not os.path.isdir(path): + self._logger.error('is not a directory: ' + path) + else: + # data/kawi/files_trashbin + nodes = os.listdir(path) + for node in nodes: + if node != 'files_external' and os.path.isdir(path + os.sep + node) and not node.startswith('appdata_'): + users += '|' + node + return users[1:] + + def run(self): + '''Executes the configured tasks in an endless loop. + @return: None: success otherwise: error message + ''' + sliceSeconds = self._configuration.getInt('slice', 60) + if sliceSeconds < 5: + self._logger.error('wrong slice in configuration, changed to 5: ' + str(sliceSeconds)) + sliceSeconds = 5 + self._logger.log('starting endless task loop with slice {} and {} tasks'.format(sliceSeconds, len(self._tasks)), 1) + while True: + isConnected = False + for task in self._tasks: + if task.next(): + if not isConnected: + rc = self.connect() + if rc != None: + self._logger.error(rc) + break + isConnected = True + self.task(task._name, True) + if isConnected: + self.disconnect() + self._logger.log('sleeping {} sec...'.format(sliceSeconds), 4) + time.sleep(sliceSeconds) + self._loopNo += 1 + return None + + def taskClouds(self): + '''Find out the data of the clouds and send it. + @return: None: success otherwise: error message + ''' + rc = None + baseDir = self._configuration.getString('cloud.basedir') + excluded = self._configuration.getString('cloud.excluded', '').split(':') + cloudDirs = os.listdir(baseDir) + for node in cloudDirs: + if node in excluded: + pass + elif node in cloudDirs: + used = 0 + free = 0 + total = 0 + info = '-' + pathData = baseDir + os.sep + 'data' + [trashSize, dirs, files] = [-1, 0, 0] + if self._loopNo % 1 == 0: + [files, dirs, used, trashSize, info] = base.StringUtils.escChars(self.cloudStatistic(pathData)) + logs = '' + if self._loopNo % 3 == 0: + logs = self.cloudLogs(pathData)[0:12000] + users = '' + if self._loopNo % 3 == 0: + users = self.cloudUsers(pathData)[0:4000] + args = [self._hostName, pathData, str(total), str(used), str(free), str(trashSize), str(files), str(dirs), users, info, logs] + self._logger.log('taskClouds(): ' + '\t'.join(base.StringUtils.limitItemLength(args, 40)), 3) + answer = 'WAIT' + while answer == 'WAIT': + answer = self._connection.sendAndReceive('clouds', args) + self._logger.log('answer: ' + str(answer)[0:80], 3) + time.sleep(10) + elif node not in excluded: + self._logger.log('taskClouds(): excluded: {} [{}]'.format(info[0], ','.join(cloudDirs)[0:80]), 3) + return rc + + def taskProcesses(self): + '''find out the data about the processes and memory and send it. + @return: None: success otherwise: error message + ''' + rc = None + return rc + + def taskUsers(self): + '''Find out the data about the users and send it. + @return: None: success otherwise: error message + ''' + rc = None + return rc + + def taskStress(self): + '''Find out the stress data and send it. + @return: None: success otherwise: error message + ''' + rc = None + now = datetime.datetime.now().strftime('%s') + patternIo = self._configuration.getString('io.pattern', '^~$') + patternNet = self._configuration.getString('net.pattern', '^~$') + info = base.LinuxUtils.stress(patternIo, patternNet) + if self._lastStress == None: + self._logger.log('first stress data ignored', 1) + self._timeLastStress = now + self._lastStress = info + else: + args = [self._configuration.getString('host.name'), + str(int(now) - int(self._timeLastStress)), + str(int(info[0]) - int(self._lastStress[0])), str(int(info[1]) - int(self._lastStress[1])), + str(int(info[2]) - int(self._lastStress[2])), str(int(info[3]) - int(self._lastStress[3])), + "{:.1f}".format(info[4]), str(info[5]), str(info[6])] + self._lastStress = info + self._logger.log('taskStress() host time rdio wrio rdnet wrnet load memav swapav: ' + '\n'.join(args), 3) + answer = self._connection.sendAndReceive('stress', args) + self._logger.log('answer: ' + answer, 3) + return rc + +def usage(): + '''Returns an info about usage. + ''' + return """usage: WebDashClient [<global_opts>] <command> [<args>] + Starts a TCP server to collect data from other hosts to store in the database +GLOBAL_OPTS +GLOBAL_MODES +<command>: +{} + example + Creates an example configuration + run + Executes the configured tasks + task <task> + Execute the <task>: filesystems users processes + +Examples: +webdashclient -q install +webdashclient -v3 task filesystems + """ + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/wdcollect', 'run'] + ''' + serviceInfo = base.BaseTool.ServiceInfo('webdashclient', 'root', 'www-data', + 'A TCP client assembling data and sending it to the webdash server', 'WEBDASHCLIENT', None) + appInfo = base.BaseTool.ApplicationInfo('webdashclient', 'appl/WebDashClient.py', usage, serviceInfo) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = WebDashClient(options) + rc = None + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'run' or cmd == 'daemon': + rc = tool.run() + elif cmd == 'task': + if len(argv) < 1: + task = 'filesystems' + else: + task = ':'.join(argv) + rc = tool.connect() + if rc == None: + rc = tool.task(task) + else: + tool.usage('unknown command: ' + cmd) + tool.close() + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/WebDashServer.py b/appl/WebDashServer.py new file mode 100755 index 0000000..9e16e02 --- /dev/null +++ b/appl/WebDashServer.py @@ -0,0 +1,336 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' + +import time +import sys +import os.path +import MySQLdb +#import base.BaseTool + +sys.path.insert(0, '/usr/share/pyrshell') +#import base.Logger +#import base.MemoryLogger +#import base.StringUtils +import base.BaseTool +import net.TcpTaskHandler +import net.TcpServer + +class WebDashStorageHandler (net.TcpTaskHandler.TcpTaskHandler): + def __init__(self, configuration, logger, verboseLevel = 0): + '''Constructor. + @param configuration: the configuration file + @param logger: the logger + ''' + net.TcpTaskHandler.TcpTaskHandler.__init__(self, logger) + self._configuration = configuration + # each thread has its own db connection: self._db[threadId] = connection() + self._db = dict() + self._verboseLevel = verboseLevel + + def connectDB(self): + rc = None + host = self._configuration.getString('mysql.host', 'localhost') + db = self._configuration.getString('mysql.db') + user = self._configuration.getString('mysql.user') + code = self._configuration.getString('mysql.code') + if db == None or user == None or code == None: + self._logger.error("missing mysql.db/mysql.user etc. Cannot connect database.") + else: + rc = MySQLdb.connect(host=host, # your host + user=user, # username + passwd=code, # password + db=db) # name of the database + return rc + + def close(self): + net.TcpTaskHandler.TcpTaskHandler.close(self) + + def completeSql(self, sql, params): + '''Returns the sql statement with expanded placeholders + @param sql: the sql statement with placeholders '%s' for the parameters + @param params: an array with the (positional) parameters + @return the sql with expanded placeholders + ''' + for item in params: + sql = sql.replace('%s', item[0:80], 1) + sql = sql.replace('\n', ' ') + return sql.replace(' ', ' ') + + def example(self): + text = '''# webdashserver example configuration +log.file=/var/log/local/webdashserver.log +service.port=58201 +service.host=127.0.0.1 +mysql.user=webdash +mysql.code=Secret.Phrase +mysql.db=appwebdash +# in seconds. all older records (of table stresses) will be deleted +max.age.stresses=86400 +# Email data for error report: +# Receiver of the error messages: may be a blank separated list of email addresses +send.always=False +admin.email=hm.neutral@gmx.de +smtp.host=smtp.gmx.de +smtp.port=587 +smtp.sender=hm.neutral@gmx.de +smtp.user=hm.neutral@gmx.de +smtp.code=sEcReT +smtp.tls=True +''' + self.storeExample(text) + + def fulfill(self, args, serverThread): + '''FulFills the task like "filesystem". + @param args: the command arguments. args[0] contains the command + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + @return: True: request is handled False: request is unknown, next chain member should be asked + ''' + rc = True + threadId = serverThread._id + if args[0] == 'threadstart': + self._db[threadId] = self.connectDB() + elif args[0] == 'threadend': + self._db[threadId].close() + del self._db[threadId] + self._logger.log('=== thread {}: db closed. open connections ({}): {}'.format(threadId, len(self._db), base.StringUtils.join(' ', self._db.keys())), 2) + else: + keys = [] + for key in self._db.keys(): + keys.append(key) + for key in keys: + if int(key) < int(threadId) - 10: + self._db[key].close() + del self._db[key] + self._logger.log('connection lately closed: {}'.format(key), 1) + if len(self._db) >= 3: + self._logger.log("too many connection, blocking...") + serverThread.send('WAIT') + time.sleep(10) + else: + db = self._db[threadId] + if args[0] == 'filesystems': + self.storeFilesystemData(db, args[1], args[2], args[3], args[4], args[5]) + serverThread.send('OK') + elif args[0] == 'clouds': + self.storeCloudData(db, args[1:]) + serverThread.send('OK') + elif args[0] == 'stress': + # server, deltaTime, ioRead, ioWrite, netRead, netWrite, load, memoryAvailable, swapAvailable + self.storeStressData(db, args[1], args[2], args[3], args[4], args[5], args[6], args[7], args[8], args[9]) + serverThread.send('OK') + else: + rc = False + return rc + + def storeCloudData(self, db, args): + """Stores the data of a cloud into the database. + @param db: database handle + @param args: the data delivered from the client + """ + # args = [self._hostName, path, str(total), str(used), str(free), str(trashSize), str(files), str(dirs), users, info, logs] + if len(args) == 9: + [server, name, total, used, free, trashSize, trashFiles, trashDirs, users] = args + logs = '' + info = '' + if len(args) == 10: + [server, name, total, used, free, trashSize, trashFiles, trashDirs, users, info] = args + logs = '' + else: + [server, name, total, used, free, trashSize, trashFiles, trashDirs, users, info, logs] = args + info = base.StringUtils.unescChars(info) + name = os.path.basename(name) + if name == '': + name = 'root' + sql = '''UPDATE clouds +SET + cloud_total=%s, + cloud_used=%s, + cloud_free=%s,$ + changed=NOW() +WHERE + cloud_name=%s + and cloud_host = (SELECT host_id FROM hosts WHERE host_name=%s)''' + with db.cursor() as cursor: + params = [total, used, free] + # trashSize, trashFiles, trashDirs, logs, users, info, + sql2 = '' + if int(trashSize) >= 0 and int(trashDirs) > 0: + sql2 = 'cloud_trash=%s,cloud_trashfiles=%s,cloud_trashdirs=%s,' + + params.append(trashSize) + params.append(trashFiles) + params.append(trashDirs) + if len(logs) > 0: + sql2 += 'cloud_log=%s,' + params.append(logs) + if len(users) > 0: + sql2 += 'cloud_users=%s,' + params.append(users) + if len(info) > 0: + sql2 += 'cloud_info=%s,' + params.append(info) + sql = sql.replace('$', sql2) + params.append(name) + params.append(server) + count = cursor.execute(sql, params) + if count == 0: + sql = '''INSERT INTO clouds +(cloud_name, cloud_total, cloud_used, cloud_free, cloud_trash,cloud_trashfiles,cloud_trashdirs,cloud_log,cloud_users,cloud_info,changed, changedby, created, createdby, cloud_host) +VALUES +(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), 'crawler', NOW(), 'crawler', (SELECT host_id FROM hosts where host_name=%s)); +''' + params = [name, total, used, free, trashSize, trashFiles, trashDirs, logs, users, info, server] + rc = cursor.execute(sql, params) + if rc == None: + self._logger.log('storecloudData() [{}] {}'.format(count, self.completeSql(sql, params)), 3) + else: + self._logger.log('storecloudData() {}'.format(count, self.completeSql(sql, params)), 3) + else: + self._logger.log('storecloudData(): {} records changed: {}'.format(self.completeSql(sql, params), count), 3) + db.commit() + + def storeFilesystemData(self, db, server, fsName, total, used, free): + """Stores the data of a filesystem into the database. + @param server: the sending server + @param fsName: the name of the filesystem + @param total: the total space of the filesystem in bytes + @param used: the used space of the filesystem in bytes + @param free: the available space of the filesystem + """ + sql = '''UPDATE filesystems +SET + filesystem_total=%s, + filesystem_used=%s, + filesystem_free=%s, + changed=NOW() +WHERE + filesystem_mount=%s + and filesystem_host = (SELECT host_id FROM hosts WHERE host_name=%s)''' + with db.cursor() as cursor: + params = [total, used, free, fsName, server] + count = cursor.execute(sql, params) + if count == 0: + sql = '''INSERT INTO filesystems +(filesystem_name, filesystem_total, filesystem_used, filesystem_free, changed, changedby, created, createdby, filesystem_host) +VALUES +(%s, %s, %s, %s, NOW(), 'crawler', NOW(), 'crawler', (SELECT host_id FROM hosts where host_name=%s)); +''' + if fsName == '/': + name = 'root' + else: + name = fsName.replace('/', '_')[1:] + params = [name, total, used, free, server] + rc = cursor.execute(sql, params) + if rc == None: + self._logger.log('storeFilesystemData() [{}] {}'.format(count, self.completeSql(sql, params)), 3) + else: + self._logger.log('storeFilesystemData() {}'.format(count, self.completeSql(sql, params)), 3) + else: + self._logger.log('storeFilesystemData(): {} records changed: {}'.format(self.completeSql(sql, params), count), 3) + db.commit() + + def storeStressData(self, db, server, deltaTime, ioRead, ioWrite, netRead, netWrite, load, memoryAvailable, swapAvailable): + """Stores the data of a filesystem into the database. + @param server: the sending server + @param deltaTime: the byte data are sampled since this time in (msec) + @param ioRead: the read amount in bytes (disk io) + @param ioWrite: the write amount in bytes (disk io) + @param netRead: the read amount in bytes (network io) + @param netWrite: the write amount in bytes (network io) + @param load: the 1 minute load + @param memoryAvailable: the available ram memory + @param swapAvailable: the available swap memory + """ + with db.cursor() as cursor: + maxAge = self._configuration.getInt('max.age.stresses', 0); + if maxAge > 0: + sql = 'delete from stresses where changed < timestampadd(second, -{}, NOW());'.format(maxAge) + cursor.execute(sql) + sql = '''INSERT INTO stresses +(stress_time, stress_readio, stress_writeio, stress_readnet, stress_writenet, stress_load, stress_memoryavailable, stress_swapavailable, created, createdby, stress_host) +VALUES +(%s, %s, %s, %s, %s, %s, %s, %s, NOW(), 'webdash', (SELECT host_id FROM hosts where host_name=%s)); +''' + params = (deltaTime, + ioRead, + ioWrite, + netRead, + netWrite, + load, + memoryAvailable, + swapAvailable, + server + ) + recId = cursor.execute(sql, params) + if recId != None: + self._logger.log('storeFilesystemData(): {} {}'.format(recId, self.completeSql(sql, params)), 3) + db.commit() + +class WebDashServer (base.BaseTool.BaseTool): + + def __init__(self, globalOptions): + '''Constructor. + @param logger: the logger + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'webdashserver.conf', True) + + def service(self): + """Starts the TCP server and wait for tasks. + """ + port = self._configuration.getInt('service.port', 58201) + host = self._configuration.getString('service.host', '127.0.0.1') + handler = WebDashStorageHandler(self._configuration, self._logger, self._verboseLevel) + handler.setVerboseLevel(self._verboseLevel) + # def __init__(self, threadId, ip, port, clientSocket, server, verboseLevel = 0): + server = net.TcpServer.TcpServer(port, self._logger, handler, host, self._verboseLevel) + self._logger.log("listening on {}:{}...".format(host, port), 1) + server.listen() + handler.close() + +def usage(): + '''Returns an info about usage. + ''' + return """usage: webdashserver [<global_opts>] <command> + Starts a TCP server to collect data from other hosts to store in the database +GLOBAL_OPTS +GLOBAL_MODES +<command>: + service or daemon + Starts the TCP server +Examples: +webdashserver -q install +webdashserver -v3 service +""" + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/wdserver', 'service'] + ''' + serviceInfo = base.BaseTool.ServiceInfo('webdashservice', 'webdash', 'webdash', + 'A TCP server storing data into a database', 'WEBDASH', None) + appInfo = base.BaseTool.ApplicationInfo('webdashserver', 'appl/WebDashServer.py', usage, serviceInfo) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = WebDashServer(options) + rc = None + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'service' or cmd == 'daemon': + rc = tool.service() + elif cmd == 'reload': + # rc = tool.service() + pass + else: + tool.usage('unknown command: ' + cmd) + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/Webserver.py b/appl/Webserver.py new file mode 100755 index 0000000..08c9a47 --- /dev/null +++ b/appl/Webserver.py @@ -0,0 +1,107 @@ +#! /usr/bin/python3 +''' +Webserver: Scalable Vector Graphics tool + +@author: hm +''' +import os.path +import sys +import http.server +import socketserver + + +sys.path.insert(0, '/usr/share/pyrshell') +import base.BaseTool +import base.ProcessHelper + +class RequestHandler (http.server.SimpleHTTPRequestHandler): + def do_GET(self): + '''Customizes the handling of a GET request. + ''' + #if self.server._verboseLevel >= 2: + # self.server._logger.log('GET: ' + self.requestline) + http.server.SimpleHTTPRequestHandler.do_GET(self) + + def log_message(self, format, *restArgs): + '''Customizes logging of the requests. + @param format: the format of the log message + ''' + self.server._logger.log('"{}" {} {}'.format(restArgs[0], restArgs[1], restArgs[2]), 2) + +class Webserver (base.BaseTool.BaseTool): + '''Translates from one wiki syntax into another. + ''' + def __init__(self, globalOptions): + '''Constructor. + @param globalOptions: the basic configurations + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'pywebserver.conf') + self._processTool = base.ProcessHelper.ProcessHelper(self._verboseLevel, self._logger) + # True: delete user/db without confirmation: use only in unittests! + self._forceDeleting = False + + def example(self): + '''Creates a example configuration. + ''' + example = '''# Webserver example configuration +log.file=/var/log/local/pywebserver.log +port=20180 +document.root=/var/www/pywebserver +''' + self.storeExample(example) + + def run(self): + '''Runs the webserver service. + ''' + port = self._configuration.getInt('port', 20180) + root = self._configuration.getString('document.root', '/var/www/pywebserver') + if not os.path.isdir(root): + self.usage('missing document root: ' + root) + os.chdir(root) + self._logger.log('document root: ' + os.path.realpath(os.curdir), 1) + self._logger.log('listening to port {}...'.format(port), 1) + httpd = socketserver.TCPServer(("", port), RequestHandler) + httpd._verboseLevel = self._verboseLevel + httpd._logger = self._logger + httpd.serve_forever() + httpd.server_close() + +def usage(): + '''Returns an info about usage. + ''' + return """webserver [<global-opts>] <command> + A simple webserver used for transporting infos and files. +GLOBAL_OPTS +GLOBAL_MODES +<command>: + daemon + start the service +example: + webserver -v3 daemon +""" + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/pywebserver', 'run'] + ''' + serviceInfo = base.BaseTool.ServiceInfo('pywebserver', 'pywebserver', 'pywebserver', + 'A HTTP webserver', 'WEBSERVER', None) + appInfo = base.BaseTool.ApplicationInfo('pywebserver', 'appl/Webserver.py', usage, serviceInfo) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = Webserver(options) + rc = None + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'daemon' or cmd == 'run': + tool.run() + else: + tool.usage("unknown command: " + cmd) + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/WikiTool.py b/appl/WikiTool.py new file mode 100755 index 0000000..3157558 --- /dev/null +++ b/appl/WikiTool.py @@ -0,0 +1,205 @@ +#! /usr/bin/python3 +''' +wikitool: Scalable Vector Graphics tool + +@author: hm +''' +import os.path +import sys +import re +import time +import math + +sys.path.insert(0, '/usr/share/pyrshell') +import base.BaseTool + +class WikiStatus: + def __init__(self): + self._openScript = None + self._inTable = False + self._regTable = None + self._regHeadline = None + self._regUnorderedList = None + self._regUnorderedSubList = None + +class WikiTool (base.BaseTool.BaseTool): + '''Translates from one wiki syntax into another. + ''' + def __init__(self, globalOptions): + '''Constructor. + @param globalOptions: the basic configurations + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'wikitool.conf') + + def example(self): + example = '''# wikitool example configuration +log.file=/var/log/local/wikitool.log +''' + self.storeExample(example) + + def gitlabToMediaWiki(self, argv): + '''Translates from gitlab into media wiki. + @param argv: arguments + @return rc: None or error message + ''' + rc = None + source = argv[0] + target = argv[1] + argv = argv[2:] + status = WikiStatus() + if not os.path.exists(source): + rc = "input file {} does not exist".format(source) + else: + status._regHeadline = re.compile(r'^(#+)\s*(.*)') + status._regUnorderedList = re.compile(r'^ ?[-+*]\s*(.*)') + status._regUnorderedSubList = re.compile(r'^ ?[-+*]?\s*(.*)') + status._regTable = re.compile(r'^\|') + status._openScript = None + status._inTable = False + out = '' + with open(source, 'rb') as fpInput: + self._logger.log('open success: ' + source, 3) + try: + for rawLine in fpInput: + line = rawLine.decode('UTF-8') + out = self.handleLine(line, status, out) + except Exception as exc: + self._logger.error('gitlabToMediaWiki: ' + str(exc)) + if status._inTable: + # remove last "|-\n": + out = out[0:-3] + out += "|}\n" + if target == '-': + print(out) + else: + self._logger.log('writing: {} bytes'.format(len(out)), 3) + try: + fpOut = open(target, 'wb') + rawLine = out.encode('UTF-8') + fpOut.write(rawLine) + fpOut.close() + except OSError as exc3: + self._logger.error('Exception ' + str(exc3)) + except Exception as exc4: + self._logger.error('Exception ' + str(exc4)) + if fpOut != None: + os.chmod(target, 0o666) + return rc + + def handleLine(self, line, status, out): + '''Translate a line. + @param line: the line to translate + @param status: IN/OUT: the translator state + @param out: IN: the generated text + @result: the translated line + ''' + ready = False + if status._openScript != None: + if line.startswith('```'): + out += status._openScript + "\n" + status._openScript = None + ready = True + else: + out += line + ready = True + elif line.startswith('```'): + language = line.strip()[3:] + if language == '': + out += '<pre>' + status._openScript = '</pre>' + else: + status._openScript = '</syntaxhighlight>' + out += '<syntaxhighlight lang="{}" "line=\'line\'>\n'.format(language) + ready = True + if not ready: + matcher = status._regTable.match(line) + if matcher != None: + cols = line.strip().split('|') + cols = cols[1:-1] + if not status._inTable: + out += "{|\n" + for col in cols: + out += '! ' + self.lineMarkupGitToMedia(col.strip()) + "\n" + status._inTable = True + else: + for col in cols: + out += '| ' + self.lineMarkupGitToMedia(col.strip()) + "\n" + out += '|-' + "\n" + ready = True + if not ready: + if status._inTable: + # remove last "|-\n": + out = out[0:-3] + out += "|}\n" + status._inTable = False + matcher = status._regHeadline.match(line) + if matcher != None: + countText = matcher.group(1) + prefix = '=' * len(countText) + out += '{} {} {}\n'.format(prefix, self.lineMarkupGitToMedia(matcher.group(2)), prefix) + ready = True + if not ready: + matcher = status._regUnorderedList.match(line) + if matcher != None: + out += '* {}\n'.format(self.lineMarkupGitToMedia(matcher.group(1))) + ready = True + if not ready: + matcher = status._regUnorderedSubList.match(line) + if matcher != None: + out += '** {}\n'.format(self.lineMarkupGitToMedia(matcher.group(1))) + else: + out += self.lineMarkupGitToMedia(line) + return out + + def lineMarkupGitToMedia(self, line): + '''Transforms the markup inside the line (fat attributes ...) + @param line: line to transform + @return: the transformed line + ''' + line = re.sub(r'\b__(.+?)__\b', r"'''\1'''", line) + line = re.sub(r'\*\*(.+?)\*\*', r"'''\1'''", line) + line = re.sub(r'\b([*_])(.+?)\1\b', r"''\2''", line) + line = re.sub(r'\[(.*?)\]\((.*?)\)', r'[[\2|\1]]', line) + return line + +def usage(): + '''Returns an info about usage. + ''' + return """wikitool [<opts>] <command> + translate from one wiki dialect to another +GLOBAL_OPTS +GLOBAL_MODES +<command>: +{} + gitlab-to-mediawiki <input> <output> +example: + wikitool -v2 gitlab-to-mediawiki /tmp/gitlab.txt /tmp/mediawiki.txt" +""" + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/wikitool', 'run'] + ''' + appInfo = base.BaseTool.ApplicationInfo('wikitool', 'appl/WikiTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = WikiTool(options) + tool._verboseLevel = 4 + rc = None + (cmd, argv) = tool.handleStandardCommands(argv) + if cmd == None: + pass + elif cmd == '': + tool.usage('missing subcommand') + elif cmd == 'gitlab-to-mediawiki': + if len(argv) < 2: + tool.usage('too few arguments') + else: + rc = tool.gitlabToMediaWiki(argv) + else: + tool.usage('unknown command: ' + cmd) + if rc != None: + tool.usage(rc) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/ZipTool.py b/appl/ZipTool.py new file mode 100755 index 0000000..e4bc30a --- /dev/null +++ b/appl/ZipTool.py @@ -0,0 +1,181 @@ +#! /usr/bin/python3 +''' +Created on 08.06.2018 + +@author: hm +''' + +import sys +import os.path + +sys.path.insert(0,'/usr/share/pyrshell') + +import base.Zipper +import base.BaseTool + + +class ZipTool (base.BaseTool.BaseTool): + + def __init__(self, globalOptions): + '''Constructor. + @param logger: the logger + ''' + base.BaseTool.BaseTool.__init__(self, globalOptions, 'ziptool.conf') + + def readArgFile(self, name): + '''Reads a file containing the arguments. + @param name: the file's name + @return: an array of arguments + ''' + argv = [] + with open(name) as fp: + for line in fp: + line = line.strip() + if not line.startswith('#') and line != '': + argv.append(line) + return argv + + def zip(self, cmd, argv): + (argv, options) = self.splitArgsAndOpts(argv) + argv2 = None + for opt in options: + if opt.startswith('-a') or opt.startswith('--argv=') or opt.startswith('--argument-file='): + if len(options) > 1: + print('arguments behind {:s} will be ignored: {:s}'.format(opt, ' '.join(options[1:]))) + if opt.startswith('-a'): + argFile = opt[2:] + else: + argFile = opt[opt.find('=') + 1:] + if not os.path.exists(argFile): + self.usage('optument file does not exist: ' + argFile) + else: + argv2 = self.readArgFile(argFile) + break + if argv2 != None: + argv += argv2 + if len(argv) < 2: + self.usage('missing arguments') + archive = argv[0] + argv = argv[1:] + rc = None + if cmd == 'create': + tool = base.Zipper.Zipper(archive, 'w', self._logger) + if tool._zip != None: + rc = tool.create(argv) + elif cmd == 'extract': + tool = base.Zipper.Zipper(archive, 'r', self._logger) + if tool._zip != None: + rc = tool.extract(argv) + elif cmd == 'info': + tool = base.Zipper.Zipper(archive, 'r', self._logger) + if tool._zip != None: + rc = tool.info(argv) + else: + rc = 'unknown subcommand: ' + cmd + if rc != None: + self.usage(rc) + + def example(self): + example = '''# ziptool example configuration +log.file=/var/log/local/ziptool.log +''' + self.storeExample(example) + return example + +def usage(): + '''Returns an info about usage. + ''' + return """usage: ziptool [<global_opts>] <command> [<opts>] [<params>] + Zip archive management +<command>: + create <archive> [<opts>] <input1> [<opts>] <input2> ...] + Create a zip archive. Note: <opts> are relevant for all following <inputX> + <inputX>: a file or a directory + <opt>: + --shortest-path + The path name of the stored files is maximal shorted: + <input> may be /abc/def. The file /abc/def/x/a.txt has a stored name x/a.txt + --store-node-only + The path of the stored files is shorted to the last node of the base: + <input> may be /abc/def. The file /abc/def/x/a.txt has a stored name def/x/a.txt + --ignored-files=<reg_expr> + The matching files will not be stored + --ignored-dirs=<reg_expr> + The matching files will not be stored + --already-compressed-dirs=<reg_expr> + The matching directories will not be compressed + info <archive> <opts> [<pattern>] + <pattern>: + Only files matching the pattern are displayed. + If no pattern option (-r or -w) is given, the pattern is interpreted as substring of the full name + <opt>: + -1 or --name-only + Displays the full filename only + -r or --reg-expression + The pattern is interpreted as regular expression + -w or --wildcard(s) + The pattern is treated as a shell pattern: + '*': any string '?': one character '['<chars_or_ranges>']', e.g. "*.mp[34] + -s or --substring + The pattern is treated as a substring. This is the default + -p or --path + The pattern describes the path + -n or --node + the pattern describes the node + -f or --full + The pattern describes the full name (path and node) + extract <archive> <opts> [<pattern1> [ <opts> <pattern2> ... ]] + <opt>: + -o or --overwrite + Existing files will be overwritten. Default + -u or --update + Existing files will be overwritten if the stored file is younger + -f or --freshen + Replace existing files, do not create ones + -t or --not-overwrite + Ignore existing files + -r or --reg-expression + The pattern is interpreted as regular expression + -w or --wildcard(s) + The pattern is treated as a shell pattern: + '*': any string '?': one character '['<chars_or_ranges>']', e.g. "*.mp[34] + -s or --substring + The pattern is treated as a substring. This is the default + -p or --path + The pattern describes the path + -n or --node + the pattern describes the node + -f or --full + The pattern describes the full name (path and node) + backup <archive> [<opts>] <directory> + <directory> + the directory to store + <opt>: + + restore <archive> [opts] [<target-directory>] + <opt>: + --delete + Each file which is in target directory but not in archive will be deleted +Example: +ziptool create --store-node-only /tmp/test.zip /home/adam /home/berta +zt --args=/opt/backup/backup.args.txt +""" + +def main(argv): + '''The main routine. + @param argv: the program arguments, e.g. ['/usr/local/bin/texttool', 'script', '-', 'data.csv'] + @return: None: ok otherwise: error message + ''' + appInfo = base.BaseTool.ApplicationInfo('ziptool', 'appl/ZipTool.py', usage) + (options, argv) = base.BaseTool.getGlobalOptions(argv, appInfo) + tool = ZipTool(options) + (cmd, argv) = tool.handleStandardCommands(argv) + rc = None + if cmd == None: + pass + else: + tool.zip(cmd, argv) + tool._finish() + +if __name__ == '__main__': + main(sys.argv) diff --git a/appl/__init__.py b/appl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/base/BaseLogger.py b/base/BaseLogger.py new file mode 100644 index 0000000..583b55f --- /dev/null +++ b/base/BaseLogger.py @@ -0,0 +1,98 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +class BaseLogger: + '''Base class of the loggers. + The derived class must implement the method log(message) + ''' + def __init__(self, verboseLevel): + '''Constructor. + @param verboseLevel: logging is done only if minLevel < verboseLevel. minLevel is a parameter of log() + ''' + self._verboseLevel = verboseLevel + self._logDebug = True + self._logInfo = True + self._errors = 0 + self._maxErrors = 20 + self._firstErrors = [] + self._errorFilter = None + self._mirrorLogger = None + self._inUse = False + + def debug(self, message): + '''Logs a debugging message. + @param message: the message to log + @return: True + ''' + if self._mirrorLogger != None: + self._mirrorLogger.debug(message) + if self._logDebug: + self._inUse = True + self.log(message) + self._inUse = False + return True + + def error(self, message): + '''Logs a message. + @param message: the error message to log + @return: False + ''' + if self._mirrorLogger != None: + self._mirrorLogger.error(message) + filtered = self._errorFilter != None + if filtered: + if type(self._errorFilter) == str: + filtered = message.find(self._errorFilter) >= 0 + else: + filtered = self._errorFilter.search(message) != None + if not filtered: + self._inUse = True + self.log('+++ ' + message) + self._errors += 1 + if self._errors < self._maxErrors: + self._firstErrors.append(message) + self._inUse = False + return False + + def info(self, message): + '''Logs an info message. + @param message: the message to log + @return: True + ''' + if self._mirrorLogger != None: + self._mirrorLogger.info(message) + if self._logInfo: + self._inUse = True + self.log(message) + self._inUse = False + return True + + def setMirror(self, logger): + '''Sets a "mirror" logger: all messages are logged to the mirror too + @param logger: the mirror logger + ''' + if self._mirrorLogger != None: + logger.setLogger(self._mirrorLogger) + self._mirrorLogger = logger + + def setErrorFilter(self, excluded, mirrorsToo = True): + '''Sets the error filter: if the pattern matches the error is ignored (not logged) + @param excluded: string: a substring of the ignored error + re.RegExpression: a compiled regular expression of the ignored errors + @param mirrorsToo: True: the filter is used for the mirror loggers too + ''' + self._errorFilter = excluded + if mirrorsToo and self._mirrorLogger != None: + self._mirrorLogger.setErrorFilter(excluded) + + def transferErrors(self, logger): + '''Transfers the error from another logger. + @param logger: the source of the errors to transfer + ''' + self._errors += logger._errors + self._firstErrors += logger._firstErrors + +if __name__ == '__main__': + pass diff --git a/base/BaseTool.py b/base/BaseTool.py new file mode 100644 index 0000000..d8b06de --- /dev/null +++ b/base/BaseTool.py @@ -0,0 +1,1454 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +import tempfile +import re +import sys +import os.path +import subprocess +import fnmatch +import datetime +import time +import platform +import shutil +import grp +import pwd +import posix + +sys.path.insert(0, '/usr/share/pyrshell') + +import base.StringUtils +import base.JavaConfig +import base.Logger +import base.MemoryLogger +import base.ProcessHelper +import base.FileHelper + +baseToolLatestTool = None +baseToolResult = None + +def latestTool(): + global baseToolLatestTool + return baseToolLatestTool + +def result(): + global baseToolResult + return baseToolResult + +def setLatestTool(tool): + global baseToolLatestTool + baseToolLatestTool = tool + +def setResult(result): + global baseToolResult + baseToolResult = result + +glob_logger = None +glob_errors = 0 +baseToolUnitTestActive = False + +def unitTestIsActive(): + global baseToolUnitTestActive + return baseToolUnitTestActive + +class ServiceInfo: + '''Stores the default values of a systemd service. + ''' + def __init__(self, starter, user, group, description, prefixEnvVar, additionalEnvVariables): + '''Constructor. + @param starter: name of the script which is started from the service + @param user: None or: the process is started as this user + @param group: None or: the process is started as this group + @param description: this info is showed from the "systemctl status" command + @param prefixEnvVar: prefix of the environment variables, e.g. 'WEBDASH' + @param additionalEnvVariables: None or the definition of non standard environment variables, e.g. 'A=B\nC=D' + ''' + self._starter = starter + self._user = user + self._group = group + self._description = description + self._prefixEnvVar = prefixEnvVar + self._additionalEnvVariables = additionalEnvVariables + +class ApplicationInfo: + '''Stores the default values of an application. + ''' + def __init__(self, application, module, usage, serviceInfo = None): + '''Construction. + @param application: name of the application + @param module: name of the module relative to base, e.g. 'appl/FtpTool.py' + @param usage: a function which shows the usage message and exits the process + @param serviceInfo: None or the info about the systemd service + ''' + self._usage = usage + self._serviceInfo = serviceInfo + self._applicationName = application + self._module = module + self.anotherInit(application, module, usage, serviceInfo) + + def anotherInit(self, application, module, usage, serviceInfo = None): + '''Basic initialization. + @param application: name of the application + @param module: name of the module relative to base, e.g. 'appl/FtpTool.py' + @param usage: a function which shows the usage message and exits the process + @param serviceInfo: None or the info about the systemd service + ''' + self._serviceInfo = serviceInfo + self._usage = usage + self._applicationName = application + self._module = module + +def globOnError(function, path, exceptionInfo): + '''Used for logging errors in shutil services like rmtree() + @param function: the "owner" of the exception, platform dependent + @param path: the full filename raising the error + @param exceptionInfo: @see sys.exc_info() + ''' + glob_errors += 1 + if glob_logger != None: + (aType, value, traceback) = exceptionInfo + glob_logger.error('cannot remove {}: [{s} {s}]'.format(path, str(aType), value)) + +class GlobalOptions: + def __init__(self, appInfo, verboseLevel=None): + '''Constructor. + ''' + self._verboseLevel = verboseLevel if verboseLevel != None else 1 + self._exitWith0 = unitTestIsActive() + self._runtime = False + self._logFiles = [] + self._configDir = '/etc/pyrshell' + self._configFile = None + self._errorFilter = None + self._count = None + self._appInfo = appInfo + self._testSourceDir = None + self._testTargetDir = None + self._host = None + +class BaseTool: + '''Base class of some other tools. + ''' + + def __init__(self, globalOptions, defaultConfigFile, useThreads=False, additionalConfigDir=None): + '''Constructor. + @param globalOptions: an instance of GlobalOptions + @param defaultConfigFile: name of the configuration file if not set in globalOptions + @param useThreads: True: a threadsafe logger will be used + @param additionalConfigDir: None or a directory containing many config files. Full path or a node (in /etc/pyrshell) + ''' + setLatestTool(self) + self._start = time.process_time() + self._startReal = time.time() + self._userId = posix.getuid() + self._isRoot = self._userId == 0 + self._globalOptions = globalOptions + self._logger = None + if globalOptions._appInfo._serviceInfo != None: + self._initFromServiceInfo(globalOptions._appInfo._serviceInfo) + self._verboseLevel = globalOptions._verboseLevel + self.installLoggers(globalOptions) + self._exitWith0 = globalOptions._exitWith0 + self._configDir = globalOptions._configDir + version = base.StringUtils.grepInFile('/etc/lsb-release', r'DISTRIB_RELEASE=([0-9.]+)', None, 1) + self._ubuntuVersion = None if len(version) == 0 else version[0] + version = base.StringUtils.fromFile('/etc/debian_version') + self._debianVersion = None if len(version) == None else version.split('/')[0] + if self._debianVersion != None: + if self._debianVersion.startswith('9.'): + self._debianVersion = 'stretch/stable' + elif self._debianVersion.startswith('8.'): + self._debianVersion = 'jessie/stable' + elif self._debianVersion.startswith('7.'): + self._debianVersion = 'whezzy/stable' + elif self._debianVersion.startswith('10.'): + self._debianVersion = 'buster/stable' + if additionalConfigDir != None and additionalConfigDir.find(os.sep) < 0: + self._additionalConfigDir = self._configDir + os.sep + additionalConfigDir + else: + self._additionalConfigDir = additionalConfigDir + if globalOptions._configFile == None: + globalOptions._configFile = defaultConfigFile + loggerName = None if len(globalOptions._logFiles) == 0 else globalOptions._logFiles[0] + if loggerName == None: + logger = base.MemoryLogger.MemoryLogger(self._verboseLevel) + elif loggerName == '-': + logger = base.MemoryLogger.MemoryLogger(self._verboseLevel) + elif useThreads: + logger = base.ThreadLogger.ThreadLogger(loggerName, self._verboseLevel) + else: + logger = base.Logger.Logger(loggerName, self._verboseLevel) + if loggerName == None: + self._logger = logger + else: + self.ensureDirectory(self._configDir) + fn = self._configDir + os.sep + globalOptions._configFile + if not os.path.exists(fn): + self.example() + self._configuration = base.JavaConfig.JavaConfig(fn, logger) + name = self._configuration.getString('log.file', tempfile.gettempdir() + os.sep + 'pyrshell.log') + base.FileHelper.ensureFileExists(name, '') + try: + os.chmod(name, 0o666) + except OSError as exc: + pass + logger2 = base.Logger.Logger(name, globalOptions._verboseLevel) + logger2.transferErrors(logger) + self._logger = logger2 + self._beingRoot = os.geteuid() == 0 + self._verboseLevel = globalOptions._verboseLevel + if globalOptions._errorFilter != None: + self._logger.setErrorFilter(globalOptions._errorFilter, True) + if self._additionalConfigDir != None: + base.FileHelper.ensureDirectory(self._additionalConfigDir, self._logger) + base.FileHelper.ensureDirectory('/var/log/local') + if hasattr(globalOptions._appInfo, '_usage'): + self._usage = globalOptions._appInfo._usage + self._processHelper = base.ProcessHelper.ProcessHelper(self._verboseLevel, self._logger) + + def _finish(self): + '''Does the final work. + ''' + end = time.process_time() + endReal = time.time() + if self._globalOptions._runtime: + self._logger.log('realtime: {:.3f} sec'.format( + endReal - self._startReal) + ' runtime: {:.3f} sec'.format(end - self._start)) + + def _initFromServiceInfo(self, serviceInfo): + '''Transfers the service info into the global options. + @param serviceInfo: an instance of ServiceInfo + ''' + prefix = serviceInfo._prefixEnvVar + envVar = prefix + '_CONFIG' + if envVar in os.environ: + self._globalOptions._configDir = os.environ[envVar] + envVar = prefix + '_HOST' + if envVar in os.environ: + self._globalOptions._host = os.environ[envVar] + else: + self._globalOptions._host = BasicStatics.hostname(False) + envVar = prefix + '_APPL' + if envVar in os.environ: + self._globalOptions._appInfo._applicationNameName = os.environ[envVar] + + def clearDirectory(self, directory): + '''Deletes all files and directories from a given directories. + ''' + glob_errors = 0 + glob_logger = self._logger + if not os.path.isdir(directory): + self._logger.error('missing directory: ' + directory) + else: + try: + for node in os.listdir(directory): + if node != '.' and node != '..': + full = directory + os.sep + node + if os.path.isdir(full): + shutil.rmtree(full, True, globOnError) + else: + try: + os.unlink(full) + except OSError as exc: + self._logger.error('cannot delete {}: {}'.format(full, str(exc))) + errors = glob_errors + if errors > 0: + self._logger.error('clearDirectory(): {d} errors'.format(errors)) + finally: + glob_logger = None + + def createBackup(self, full, suffix = ''): + '''Creates a backup file in /var/tmp. + @param full: the file to save + ''' + node = os.path.basename(full) + suffix = node + '.' + base.FileHelper.pathToNode(os.path.dirname(full)) + datetime.datetime.now().strftime('-%Y.%m.%d_%H_%M_%S') + suffix + trg = '/var/tmp/' + node + suffix + if os.path.isdir(full): + self._processHelper.execute(['/bin/tar', 'czf', trg + '.tgz', '.'], True, '!shell', None, full) + else: + shutil.copy(full, trg) + self._logger.log('backup of {} created: {}'.format(full, trg), 2) + + def ensureDirectory(self, path, mode = 0o777): + '''Ensures that a directory exists. + @param path: the full name of the directory + @param mode: the rights, e.g. 0o777 for all access for everyone + @return: None: cannot create directory + otherwise: path + ''' + rc = base.FileHelper.ensureDirectory(path, self._logger, mode) + return rc + + def ensureFileDoesNotExist(self, filename, pattern=None): + '''Ensures that a file does not exist. + @param filename: the file to delete if it exists. + @param pattern: None or a regular expression for nodes to delete, e.g. r'.*\.z\d\d$' + Note: re.match() is used to find. Do not forget '.*' at the top + ''' + base.FileHelper.ensureFileDoesNotExist(filename, self._logger) + if pattern != None: + regExpr = self.regExprCompile(pattern, 'file pattern') + path = os.path.dirname(filename) + if path != '': + nodes = os.listdir(path) + for node in nodes: + if regExpr.match(node): + try: + full = path + os.sep + node + os.unlink(full) + except OSError as exp: + self._logger.error('cannot delete {:s}: {:s}'.format(full, str(exp))) + + def ensureSymbolicLink(self, source, target, createTarget=True): + '''Ensures that a directory exists. + @param source: the full name of the link source, e.g. '../sibling' + @param target: full name of the file of type 'link' + @param createTarget: creates the target if it does not exist + @return: True: the link exists + ''' + rc = base.FileHelper.ensureSymbolicLink(source, target, createTarget, self._logger) + return rc + + def debianName(self): + return None if self._debianVersion == None else self._debianVersion.split('/')[0] + + def errorFileToLog(self, filename, intro=None): + '''Dumps an error file into the log as errors. + @param filename: full path of file + @param intro: None or a preceding message + ''' + if os.path.exists(filename): + with open(filename, 'r') as fp: + for line in fp: + if line != '': + if intro != None: + line = intro + ' ' + line + intro = None + self._logger.error(line) + + def floatArgument(self, arg, defaultValue = None): + '''Gets an float argument. If an error is detected usage is called + @param arg: the argument, e.g. 'factor=1.23' + @return the number behind the first '=' + ''' + rc = defaultValue + index = arg.find('=') + if index < 0: + if defaultValue == None: + self.usage('missing float in ' + arg) + else: + try: + rc = float(arg[index+1:]) + except ValueError: + self.usage('not a float behind "=" in ' + arg) + return rc + + def fullPathToName(self, path): + '''Converts a path name into a node name: slashes are changed to '=' + @param path: full path + @return: a node name (separator replaced by '=') + ''' + rc = path.replace(os.sep, '=').lstrip('=') + return rc + + def getFilenameOrCopy(self, directory, node): + '''Get a configuration file, a copy for unittests if needed: + If _testTargetDir is defined, the file is copied from directory to _testTargetDir and the filename with _testTargetDir dir is returned. + Otherwise the filename with directory is returned + @param directory: the directory used outside of unittests + @return the full filename in directory or in _targetDir + ''' + if self._globalOptions._testTargetDir == None: + rc = directory + os.sep + node + else: + source = directory + os.sep + node + rc = self._globalOptions._testTargetDir + os.sep + node + if os.path.exists(source): + self._logger.log('copying {} -> {}'.format(source, rc), 2) + shutil.copy2(source, rc) + else: + self._logger.log('missing {}'.format(source), 3) + return rc + + def getSource(self, directory, node = None): + '''Returns the source directory specified by directory. + For unit tests the result is taken from the global options (_testSourceDir) + @param directory: the directory's name + @param node: None or the filename without path + @return: the directory name (node == None) or the filename (directory + os.sep + node) + ''' + if self._globalOptions._testSourceDir == None: + rc = directory + else: + rc = self._globalOptions._testSourceDir + if node != None: + if node.endswith(os.sep): + rc += node + else: + rc += os.sep + node + if rc.startswith('//'): + rc = rc[1:] + return rc + + def getTarget(self, directory, node = None): + '''Returns the source directory specified by directory. + For unit tests the result is taken from the global options (_testTargetDir) + @param directory: the directory's name + @param node: None or the filename without path + @return: the directory name (node == None) or the filename (directory + os.sep + node) + ''' + if self._globalOptions._testTargetDir == None: + rc = directory + else: + rc = self._globalOptions._testTargetDir + if node != None: + if node.endswith(os.sep): + rc += node + else: + rc += os.sep + node + return rc + + @staticmethod + def handleCommonCommands(appInfo, argv, options): + '''Handles the non specific commands of the tool, e.g. 'install'. + @param appInfo: the description of the application + @param argv: the program arguments starting with the command, e.g. ['install'] + @param options: the global options (extracted from the program arguments) + ''' + rc = False + if len(argv) > 0: + cmd = argv[0] + if cmd == 'install': + tool = InstallTool(appInfo, options) + tool.install(appInfo._applicationNameName, appInfo._module, options) + if appInfo._serviceInfo != None: + info = appInfo._serviceInfo + tool.installAsService(argv[1:], info._user, info._group, info._prefixEnvVar, info._description) + rc = True + elif cmd == 'uninstall': + tool = InstallTool(appInfo, options) + if appInfo._serviceInfo != None: + tool.uninstallService(argv[1:]) + tool.uninstall(appInfo._applicationNameName, argv[1:]) + rc = True + return rc + + def handleStandardCommands(self, argv): + '''Executes the common commands when the instance is initialized. + At this time the only common command is 'example'. + @param argv: the program arguments, e.g. ['info', '--short'] + @return: a tuple (cmd, argv) cmd is the first argument, argv the rest of argv. + cmd==None: a standard command has been found + cmd=='': no subcommand has been found + ''' + if len(argv) == 0: + cmd = '' + else: + cmd = argv[0] + argv = argv[1:] + options = self._globalOptions + appInfo = options._appInfo + if cmd == 'example': + self.example() + cmd = None + elif cmd == 'install': + tool = InstallTool(appInfo, options) + tool.install(appInfo._applicationName, appInfo._module, options) + if appInfo._serviceInfo != None: + info = appInfo._serviceInfo + tool.installAsService(argv, info._user, info._group, info._prefixEnvVar, info._description) + cmd = None + elif cmd == 'uninstall': + tool = InstallTool(appInfo, options) + if appInfo._serviceInfo != None: + tool.uninstallService(argv[1:]) + tool.uninstall(appInfo._applicationName, argv[1:]) + cmd = None + return (cmd, argv) + + def installLoggers(self, options): + '''Initializes the loggers described in the options._logFiles: multiple loggers are mirrored + @param options: the program options + ''' + BasicStatics.ensureDirectory('/var/log/local') + baseLogger = None + logger = None + for name in options._logFiles: + if name == '' or name == '-': + logger = base.MemoryLogger.MemoryLogger(options._verboseLevel) + else: + logger = base.Logger.Logger(name, options._verboseLevel) + if baseLogger == None: + baseLogger = logger + else: + # only the base logger should cry: + logger._verbose = False + baseLogger.setMirror(logger) + if logger == None: + logger = base.MemoryLogger.MemoryLogger(options._verboseLevel) + self._logger = logger + + def integerOption(self, arg, defaultValue = None): + '''Gets an integer argument. If an error is detected usage is called + @param arg: the argument, e.g. 'count=123' + @return the number behind the first '=' + ''' + rc = defaultValue + index = arg.find('=') + if index < 0: + ix = 0 + while ix < len(arg) and arg[ix] < '0' and arg[ix] > '9': + ix += 1 + if ix < len(arg): + index = ix - 1 + if index < 0: + if defaultValue == None: + self.usage('missing integer in ' + arg) + else: + try: + rc = int(arg[index+1:]) + except ValueError: + self.usage('not an integer {} in {}'.format(arg[index+1:], arg)) + return rc + + def integerArgument(self, arg, defaultValue = None): + '''Gets an integer argument. If an error is detected usage is called + @param arg: the argument, e.g. 'count=123' + @return the number behind the first '=' + ''' + rc = defaultValue + try: + rc = int(arg) + except ValueError: + self.usage('not an integer {}'.format(arg)) + return rc + + def nearBackup(self, filename, moveNotCopy = False): + '''Makes a backup of a file in the same parent directory. + @param filename: the source file + @param moveNotCopy: True: the source will be renamed False: the file will be copyied + ''' + if os.path.exists(filename): + target = filename + '.{}'.format(int(time.time())) + self._logger.log('creating backup file: ' + target, 2) + try: + if moveNotCopy: + os.rename(filename, target) + else: + shutil.copy2(filename, target) + except OSError as exc: + self._error('cannot create backup file {}: {}'.format(target, str(exc))) + + def logFile(self, filename, messagePattern, start): + '''Writes metadata file size and runtime into the log. + @param filename: file to log + @param messagePattern: log message with makros %f (filename) %s (filesize) %t (filetime) and %r (runtime) + @param start: None or the start of the operation (for calculating runtime) + ''' + stat = os.stat(filename) + size = base.StringUtils.formatSize(stat.st_size) + fdate = datetime.datetime.fromtimestamp(stat.st_mtime) + dateString = fdate.strftime("%Y.%m.%d %H:%M:%S") + runtime = '?' if start == None else '{:d} sec'.format(int(time.time() - start)) + msg = messagePattern.replace('%f', filename).replace('%s', size).replace('%t', dateString).replace('%r', runtime) + self._logger.log(msg) + return msg + + def publicIp(self): + '''Tries to detect the public IP of the host. + @return: None: nothing found othewise: the best known public IP + ''' + self._processHelper.executeInputOutput(['ip', 'addr', 'show'], None) + lines = self._processHelper._rawOutput.decode().split('\n') + device = None + lastIpType = '0-none' + lastDevType = '0-none' + for line in lines: + matcher = re.match(r'\d: (\w+):', line) + if matcher != None: + device = matcher.group(1) + if device.startswith('e'): + devType = '9-ethernet' + elif device.startswith('w'): + devType = '7-wlan' + elif device == 'lo': + devType = '1-loopback' + else: + devType = '3-unknown' + continue + matcher = re.match(r'\s+inet (\S+)/', line) + if matcher != None: + ip = matcher.group(1) + if ip.startswith('127.'): + ipType = '1-localhost' + elif len(ip) > 6 and (ip.startswith('169.254.') or ip.startswith('192.168.') or ip.startswith('10.') + or (ip >= '172.16.' and ip <= '172.32.' and ip[6] == '.')): + ipType = '3-private' + else: + ipType = '9-public' + self._logger.log('device: {} ip: {} devType: {} ipType: {}'.format(device, ip, devType, ipType), 3) + if (devType > lastDevType) or (ipType > lastIpType): + lastIp = ip + lastIpType = ipType + lastDevType = devType + rc = lastIp if ipType > '1-localhost' else None + self._logger.log('public ip: {}'.format(str(rc)), 2) + return rc + + def popd(self, directory): + '''Changes the current direcory (if needed and possible). + @param directory: None or the new current directory + @return None: directory = None + '': changing directory failed + otherwise: the current directory (before changing) + ''' + if directory != None and directory != '': + os.chdir(directory) + if os.path.realpath(os.curdir) != os.path.realpath(directory): + self._logger.error('cannot change to directory ' + directory) + + def pushd(self, directory): + '''Changes the current direcory (if needed and possible). + @param directory: None or the new current directory + @return None: directory = None + '': changing directory failed + otherwise: the current directory (before changing) + ''' + if directory == None: + rc = None + else: + rc = os.curdir + os.chdir(directory) + if os.path.realpath(os.curdir) != os.path.realpath(directory): + os.chdir(rc) + self._logger.error('cannot change to directory ' + directory) + rc = '' + return rc + + def regExprCompile(self, pattern, location, isCaseSensitive = False): + '''Compiles a regular expression. + @param pattern: a regular expression. + @param isCaseSensitive: true: the case is relevant + @return: the re.RegExpr instance + ''' + rc = base.StringUtils.regExprCompile(pattern, location, self._logger, isCaseSensitive) + if rc == None: + msg = self._logger._firstErrors[-1] if len(self._logger._firstErrors) > 0 else 'wrong regular expression in ' + location + self.usage(msg) + return rc + + def tableOfContent(self, archive): + '''Returns the table of content of a tar archive. + @param archive: the tar archive to inspect + @returns: the list of files stored into the archive + ''' + lines = self._processHelper.executeInputOutput(['/bin/tar', 'tzf', archive], None, True) + return lines + + def saveDirectoryByTar(self, name, source, target, opts): + '''Saves a directory into a tar archive. + @param name: archive name without extension + @param source: source directory + @param target: target directory (of the tar archive) + @param opts: None or an array of options like '-exclude=<pattern>' + ''' + if not os.path.isdir(source): + self._logger.error('tar: unknown source {:s} for {:s}'.format(source, name)) + elif not os.path.isdir(target): + self._logger.error('unknown target: ' + target) + else: + start = time.time() + # oldDir = os.getcwd() + os.chdir(source) + archive = target + os.sep + name + '.tgz' + self.ensureFileDoesNotExist(archive) + argv = ['/bin/tar', 'czf', archive] + if opts != None: + argv += opts + argv.append('.') + self._processHelper.execute(argv, True) + # os.chdir(oldDir) + self.logFile(archive, '%f: %s %t created in %r', start) + + def saveDirectoryByZip(self, name, source, target, opts, interval = None): + '''Saves a directory into a zip archive. + @param name: will be used for the node name of the archive + @param source: the directory to save + @param target: the archive will be stored here + @param opts: None or an array of options. inclusions, exclusions... + @param interval: None or number of days. Only files younger than this will be stored + ''' + if not os.path.isdir(source): + self._logger.error('zip: unknown source {:s} for {:s}'.format(source, name)) + elif not os.path.isdir(target): + self._logger.error('unknown target: ' + target) + else: + start = time.time() + oldDir = os.getcwd() + os.chdir(source) + archive = target + os.sep + name + ('.zip' if interval == None else '.latest.zip') + self.ensureFileDoesNotExist(archive, name + r'\.z\d\d$') + no = 1 + while True: + no += 1 + fn = archive.replace('.zip', 'z{:2d}'.format(no)) + if os.path.exists(fn): + self.ensureFileDoesNotExist(fn) + else: + break + dontCompress = '.zip:.tgz:.gzip:.gz:.bz2:.lz:.lzma:.xz:.7z:.deb:.rpm:.cab:.rar:.jar:.war:.ear' + \ + '.jpg:.png:.gif:.tif:.tiff:.jpeg:.gzip:.mp3:.mpeg:.mp4:.m4p:.flv:.f4a:.f4b:.f4v:.vob:.ogv:.avi:.mov' + \ + ':.odt:.ott:.ods:.odg:.otg:.odp:.odb:.doc:.docx:.xls:.xlsx:.xltx:.xlw' + argv = ['/usr/bin/zip', '-r5qyo', '-n', dontCompress] + volSize = self._configuration.getString('zip.volume.size') + if volSize != None: + if re.match(r'\d+[kmg]$', volSize) == None: + self._logger('wrong value in zip.volume.size') + else: + argv.append('-s') + argv.append(volSize) + if interval != None: + aDate = datetime.date.fromtimestamp(time.time() - interval * 86400).strftime('%Y-%m-%d') + argv.append('-t') + argv.append(aDate) + # should be exist at least one young file: + argv.append(archive) + argv.append('.') + if interval != None: + fnDummy = tempfile.gettempdir() + os.sep + '.saveDirectoryByZip.{:.0f}.mrk'.format(time.time()) + base.StringUtils.toFile(fnDummy, '') + argv.append(fnDummy) + if opts != None: + argv += opts + self._processHelper.execute(argv, True) + os.chdir(oldDir) + if interval != None: + os.unlink(fnDummy) + self.logFile(archive, '%f: %s %t created in %r', start) + + def splitArgsAndOpts(self, argv): + '''Splits the argument vector into arguments and options (starting with '-'). + @param argv: the arguments, e.g. ['random', '--seed=Wow'] + @return: a tuple (argv, options) + ''' + argv2 = [] + options = [] + for arg in argv: + if arg.startswith('-'): + options.append(arg) + else: + argv2.append(arg) + return (argv2, options) + + def storeExample(self, text, filename=None, path=None): + '''Stores the text in an configiguration file (if that does not exist) or in an example file. + @param text: the example file content + @param filename: None: the application name will be used. Otherwise: the name of the configuration file + ''' + if filename == None: + filename = self._globalOptions._appInfo._applicationName + '.conf' + if path == None: + path = self._configDir + full = path + os.sep + filename + if os.path.exists(full): + parts = filename.split('.') + if len(parts) == 1: + filename = filename + '.example' + else: + filename = '.'.join(parts[0:-1]) + '.example' + full = path + os.sep + filename + self._logger.log('creating ' + full) + base.StringUtils.toFile(full, text) + + def usage(self, message): + '''Shows a usage message. + @param message: the reason of the usage call (an error message) + ''' + self._logger.error(message) + msg = self._usage() + if msg != None: + msg = msg.replace('GLOBAL_OPTS', BasicStatics.usageGlobals(self._configuration._filename)) + appInfo = self._globalOptions._appInfo + installOpts = defaultUser = defaultGroup = environLogfile = '' + if appInfo._serviceInfo != None: + serviceInfo = appInfo._serviceInfo + defaultUser = serviceInfo._user + defaultGroup = serviceInfo._group + environLogfile = None + for name in os.environ: + if name.endswith('_LOG'): + environLogfile = name + break + if environLogfile == None: + environLogfile = appInfo._applicationName.upper() + '_LOG' + msg = msg.replace('GLOBAL_MODES', BasicStatics.usageInstallUninstall(appInfo._applicationName, + defaultUser, defaultGroup, installOpts, environLogfile)).rstrip() + if self._verboseLevel > 0: + print(msg) + print('+++ ' + message) + if not self._exitWith0: + sys.exit(1) + +class BasicStatics: + + @staticmethod + def argsToDictionary(argv, usage): + '''Tests the arguments and put them into a dictionary. + ''' + rc = dict() + rc['autostart'] = True + while len(argv) > 0: + if argv[0].startswith('--user='): + user = argv[0][7:] + if not re.match(r'^[\w-]+$', user): + usage('invalid characters in <user>:' + user) + else: + rc['user'] = user + elif argv[0].startswith('--group='): + group = argv[0][8:] + if not re.match(r'^[\w-]+$', group): + usage('invalid characters in <group>:' + group) + else: + rc['group'] = group + elif argv[0].startswith('--application='): + application = argv[0][14:] + if not re.match(r'^[\w-]+$', application): + usage('invalid characters in <application>:' + application) + else: + rc['application'] = application + elif argv[0].startswith('--host='): + host = argv[0][7:] + if not re.match(r'^[\w-]+$', host): + usage('invalid characters in <host>:' + host) + else: + rc['host'] = host + elif argv[0] == '--no-auto-start': + rc['autostart'] = False + elif argv[0].startswith('-l'): + rc['logfile'] = argv[0][2:] + elif argv[0].startswith('--log='): + rc['logfile'] = argv[0][6:] + else: + usage('unknown option: ' + argv[0]) + argv = argv[1:] + return rc + + @staticmethod + def ensureDirectory(path, logger=None, rights=0o777): + '''Ensures that a directory exists. + @param path: the full name of the directory + @param rights: the access rights. + @param logger: None or the logger + @return: None: cannot create directory + otherwise: path + ''' + if not os.path.isdir(path): + if os.path.exists(path): + BasicStatics.error('{:s} is not a directory but a file'.format(path), logger) + os.makedirs(path, rights) + if os.path.isdir(path): + BasicStatics.log('created: ' + path, logger) + else: + path = None + return path + + @staticmethod + def ensureFileDoesNotExist(filename, pattern=None, logger = None): + '''Ensures that a file does not exist. + @param filename: the file to delete if it exists. + If pattern is defined this name is used to detect the directory for pattern matching + @param pattern: None or a regular expression for nodes to delete, e.g. r'.*\.z\d\d$' + Note: re.match() is used to find. Do not forget '.*' at the top + ''' + if os.path.exists(filename): + try: + os.unlink(filename) + BasicStatics.log('removed: ' + filename, logger) + except OSError as exp: + BasicStatics.error('cannot delete {:s}: {:s}'.format(filename, str(exp)), logger) + if pattern != None: + regExpr = base.StringUtils.regExprCompile(pattern, 'file pattern', logger) + path = os.path.dirname(filename) + if path != '': + nodes = os.listdir(path) + for node in nodes: + if regExpr.match(node): + try: + full = path + os.sep + node + os.unlink(full) + except OSError as exp: + BasicStatics.error('cannot delete {:s}: {:s}'.format(full, str(exp)), logger) + + @staticmethod + def ensureUserAndGroup(user, group, logger, userId = None, groupId = None): + '''Tests whether a given user and group exists. If not they will be created. + @param user: None or the username + @param group: None or the group name + @param logger: None error will be didplayed via print() + @param userId: None or the user id (only used while creating) + @param groupId: None or the group id (only used while creating) + ''' + if user != None: + if BasicStatics.userExists(user): + BasicStatics.log('user exists: ' + user, logger) + else: + subprocess.call(['/usr/sbin/useradd', '-s', '/bin/false', user]) + if not BasicStatics.userExists(user): + BasicStatics.error('cannot create user ' + user, logger) + else: + BasicStatics.log('user created: ' + user, logger) + if group != None: + if BasicStatics.groupExists(group): + BasicStatics.log('group exists: ' + group, logger) + else: + # create group if not exists: + subprocess.call(['/usr/sbin/groupadd', '-f', group]) + if BasicStatics.groupExists(group): + BasicStatics.log ('group created: ' + group, logger) + else: + BasicStatics.error('cannot create group: ' + group, logger) + + @staticmethod + def error(msg, logger): + '''Logs an error. + @param msg: error message + @param logger: None: print will be used. Otherwise: the logger + ''' + if logger != None: + logger.error(msg) + else: + print('+++ ' + msg) + + @staticmethod + def findFiles(path, pattern, useFullName=True): + '''Finds all files of a directory given by a pattern. + @param path: the directory to search + @param pattern: a pattern with unix shell wildcards: *: any string ?: one char [<ranges>] + @param useFullName: True: the result contain the path too False: only the nodes are returned + @return: an array with the found names + ''' + rc = [] + for node in os.listdir(path): + if fnmatch.fnmatch(node, pattern): + if useFullName: + rc.append(path + os.sep + node) + else: + rc.append(node) + return rc + + @staticmethod + def firstFile(path, pattern, useFullName=True): + '''Finds the first file matching a given pattern. + @param path: the directory to search + @param pattern: a pattern with unix shell wildcards: *: any string ?: one char [<ranges>] + @param useFullName: True: the result contain the path too False: only the nodes are returned + @return: None or the found name + ''' + rc = None + for node in os.listdir(path): + if fnmatch.fnmatch(node, pattern): + if useFullName: + rc = path + os.sep + node + else: + rc = node + break + return rc + + @staticmethod + def grep(pattern, filename): + '''Finds all lines matching a given pattern. + @param pattern: the string to search, if starting with '/' it is a regular expr. + @param filename: the file to search + @return: an array of the found lines. May be empty. + ''' + rc = [] + hasWildcards = pattern.find('*') >= 0 or pattern.find('?') >= 0 + if hasWildcards: + pattern = ('*' + pattern + '*').replace('**', '*').replace('**', '*') + with open(filename, 'rb') as fp: + for line in fp: + again = False + try: + line = line.decode('UTF-8') + except UnicodeDecodeError: + again = True + if again: + line = line.decode('latin1') + if hasWildcards and fnmatch.fnmatch(line, pattern) or not hasWildcards and line.find(pattern) >= 0: + rc.append(line) + return rc + + @staticmethod + def groupExists(name): + '''Tests whether a given group exists. + @param name: the group's name + ''' + try: + info = grp.getgrnam(name) + except KeyError: + info = None + return info != None + + @staticmethod + def hostname(fullQualified=False): + '''Returns the local hostname. + @param fullQualified: True: the hostname with domain will be returned + @return: the hostname + ''' + rc = base.StringUtils.fromFile('/etc/hostname').strip() + if not fullQualified: + ix = rc.find('.') + if ix > 0: + rc = rc[0:ix] + return rc + + @staticmethod + def log(message, logger=None): + '''Logs an message. + @param message: the message to show + @param logger: None or the logger + ''' + if logger != None: + logger.log(message) + else: + print(message) + + @staticmethod + def usageGlobals(defaultConfigFile = ''): + return '''<global_opt>: + -0 or --exit-with-0 + exit code is 0 not depending on any condition + -c<dir> or --configuration-directory=<dir> + Configuration directory: contains the configuration files. Default: '/etc/pyrshell' + -j<file> or --job-file=<file> + The main configuration file inside the configuration directory. Default: "{}" + -l<file> or --log=<file> + The logfile. If empty no logfile will be created. + -q or --quiet + No logs to stdout + -r or --runtime + Writes the runtime to the logger + --test-source-dir=<path> + Used in unit tests + --test-target-dir=<path> + Used in unit tests + -v<level> or --verbose-level=<level> + Verbose level: 1: summary 2: details start 3: fine Default: 1'''.format( + defaultConfigFile.replace('\n', ' ')) + + @staticmethod + def usageInstallUninstall(defaultApplication, defaultUser = None, defaultGroup = None, installOpts = None, environLogfile = None): + if defaultUser == None or defaultUser == '': + rc = '''<global-command>: + example + creates an example configuration + install <opts> + installs the tool as command + <opts>: + --application=<application> + the name of the application. Default: {} + -l<file> or --log=<log> + file for logging output (of the daemon). + Default: /var/log/local/<application>.log + uninstall + uninstalls the application + <opts>: + --purge + remove configuration files too + --application=<application> + the name of the application. Default: '{}' +'''.format(defaultApplication, defaultApplication) + else: + rc = '''<global-command> +install <opts> + installs the daemon as a systemd service + <opts>: + --application=<application> + the name of the application. Default: {} + --user=<user> + the daemon runs as this user. Default: <application> or '{}' + --group=<group> + the daemon runs under this group. Default: <application> or '{}' + --no-auto-start + the service does not start at boot time (systemctl disable <application>) + -l<file> or --log=<log> + file for logging output (of the daemon). + Default: os.environ['{}'] or /var/log/local/<application>.log + uninstall <opts> + --purge + remove configuration files too + --application=<application> + the name of the application. Default: '{}' + --hold-user + the user will not be deleted. + Note: the user will be deleted only if its name contains the application + --hold-group + the user will not be deleted + Note: the group will be deleted only if its name contains the application'''.format( + defaultApplication, defaultUser, defaultGroup, environLogfile, defaultApplication) + return rc + + @staticmethod + def userExists(name): + '''Tests whether a given user exists. + @param name: the user's name + ''' + try: + info = pwd.getpwnam(name) + except KeyError: + info = None + return info != None + +class InstallTool: + '''Implements an installer. + ''' + def __init__(self, appInfo, globalOptions): + '''Constructor: + @param appInfo: the basic infos + @param globalOptions: the basic initialization values + ''' + self._appInfo = appInfo + if hasattr(globalOptions, '_appInfo'): + self._usage = globalOptions._appInfo._usage + self._program = os.path.abspath(sys.argv[0]) + self._systemDPath = '/etc/systemd/system/' + self._configPath = globalOptions._configDir + self._configFile = globalOptions._configFile if globalOptions._configFile != None else appInfo._applicationName + '.conf' + self._logger = base.MemoryLogger.MemoryLogger(1) + + def createSystemDScript(self, serviceName, starter, user, group, description): + '''Creates the file controlling a systemd service. + @param serviceName: used for syslog and environment file + @param starter: name of the starter script without path, e.g. 'pymonitor' + @param user: the service is started with this user + @param group: the service is started with this group + @param description: this string is showed when the status is requestes. + ''' + systemdFile = '{}{}.service'.format(self._systemDPath, self._appInfo._applicationName) + script = '''[Unit] +Description={}. +After=syslog.target +[Service] +Type=simple +User={} +Group={} +WorkingDirectory=/etc/pyrshell +EnvironmentFile=-/etc/pyrshell/{}.env +ExecStart=/usr/local/bin/{} daemon +ExecReload=/usr/local/bin/{} reload +SyslogIdentifier={} +StandardOutput=syslog +StandardError=syslog +Restart=always +RestartSec=3 +[Install] +WantedBy=multi-user.target +'''.format(description, user, group, serviceName, starter, starter, serviceName) + with open(systemdFile, "w") as fp: + fp.write(script) + print('systemd script created: ' + systemdFile) + + def ensureEnvironmentFile(self, envVarPrefix, configPath, logfile, envVariables=None, logger=None): + '''Tests whether a environmentment variable exists. If not it will be created. + @param envVarPrefix: a prefix of all variable names, e.g. 'MONITOR' + @param configPath: the path of the directory containing the configuration file, e.g. '/etc/pyrshell' + @param logFile: the log file, e.g. '/var/log/local/pymonitor.log' + @param envVariables: None or additional entries of the environment file, e.g. 'MONITOR_HOST=localhost\n' + ''' + envFile = self._configPath + os.sep + self._appInfo._applicationName + '.env' + if os.path.exists(envFile): + BasicStatics.log('environment file exists: ' + envFile, logger) + else: + path = ':'.join(sys.path) + ':' + os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) + with open(envFile, "w") as fp: + fp.write('PYTHONPATH={}\n'.format(path)) + fp.write('{}_CONFIG={}\n'.format(envVarPrefix, configPath)) + fp.write('{}_APPL={}\n'.format(envVarPrefix, self._appInfo._applicationName)) + fp.write('{}_LOGFILE={}\n'.format(envVarPrefix, logfile)) + if envVariables != None: + fp.write(envVariables); + if not os.path.exists(envFile): + BasicStatics.error('cannot create environment file ' + envFile, logger) + else: + BasicStatics.error('environment file created: ' + envFile, logger) + + def ensureStartScript(self): + fn = '/usr/local/bin/' + self._appInfo._applicationName + if os.path.islink(fn): + os.unlink(fn) + if os.path.exists(fn): + self._logger.error('cannot remove link: ' + fn) + else: + self._logger.log('link removed: ' + fn) + if not os.path.exists(fn): + prog = os.path.abspath(sys.argv[0]) + baseDir = os.path.dirname(os.path.dirname(prog)) + with open(fn, "w") as fp: + fp.write('#! /bin/bash\n') + fp.write('export PYTHONPATH={:s}:$PYTHONPATH\n'.format(baseDir)) + fp.write(prog + ' $*\n') + self._logger.log('created: ' + fn) + os.chmod(fn, 0o755) + + def evalOption(self, option): + '''Evaluates a non standard option. + Should be overwritten in deriveded classes. + @param option: the option to inspect + @return: True: option recognized False: option not recognized + ''' + return False + + def install(self, scriptName, module, options): + '''Installs a given Python module as external linux script. + @param scriptName name of the script in /usr/local/bin, e.g. 'ftptool' + @param module name of the module relative to base, e.g. 'appl/FtpTool.py' + ''' + user = os.geteuid() + if user != 0: + self.usage('you must be root!') + else: + src = '/usr/share/pyrshell/' + module + trg = '/usr/local/bin/' + scriptName + if not os.path.exists(src): + self.usage('missing module ' + src) + if os.path.exists(trg): + trg2 = trg + datetime.datetime.now().strftime('%s') + print(trg + ' already exists. Renameing to ' + trg2) + os.rename(trg, trg2) + os.symlink(src, trg) + print(src + ' installed into ' + trg) + # BasicStatics.ensureConfiguration(options, self._configFile, 'install', self._usage, self._example) + + def installAsService(self, argv, defaultUser, defaultGroup, prefixEnvVar, description): + '''Installs the monitor as a systemd service. + @param argv: program arguments, e.g. ['--user=service'] + @param defaultUser: the service ist started under this user + @param defaultGroup: the service ist started under this group + @param prefixEnvVar: the prefix of the environment variables, e.g. 'MONITOR' + @param description: a string shown by the 'systemctl status <service>' + command + ''' + user = os.geteuid() + if user != 0: + self.usage('you must be root!') + user = None + group = None + application = None + logfile = None + autoStart = True + starter = None + while len(argv) > 0: + if argv[0].startswith('--user='): + user = argv[0][7:] + if not re.match(r'^[\w-]+$', user): + self.usage('invalid characters in <user>:' + user) + elif argv[0].startswith('--group='): + group = argv[0][8:] + if not re.match(r'^[\w-]+$', group): + self.usage('invalid characters in <group>:' + group) + elif argv[0].startswith('--application='): + application = argv[0][14:] + if not re.match(r'^[\w-]+$', application): + self.usage('invalid characters in <application>:' + application) + elif argv[0].startswith('--starter='): + starter = argv[0][10:] + if not re.match(r'^[\w-]+$', application): + self.usage('invalid characters in <starter>:' + starter) + elif argv[0] == '--no-auto-start': + autoStart = False + elif argv[0].startswith('-l'): + logfile = argv[0][2:] + elif argv[0].startswith('--log='): + logfile = argv[0][10:] + elif self.evalOption(argv[0]): + pass + else: + self.usage('unknown option: ' + argv[0]) + argv = argv[1:] + if application != None and len(application) > 0: + appInfo = ApplicationInfo(application, None, usage) + self._appInfo = appInfo + if starter == None: + starter = self._appInfo._applicationName + if user == None or len(user) == 0: + user = defaultUser if defaultUser != None else self._appInfo._applicationName + if group == None or len(group) == 0: + group = defaultGroup if defaultGroup != None else self._appInfo._applicationName + if logfile == None: + logfile = '/var/log/local/' + self._appInfo._applicationName + '.log' + self.createSystemDScript(self._appInfo, starter, user, group, description) + BasicStatics.ensureUserAndGroup(user, group, self._logger) + # envVarPrefix, configPath, logfile, envVariables=None + self.ensureEnvironmentFile(prefixEnvVar, self._configPath, logfile) + self.ensureStartScript() + if autoStart: + subprocess.call(['/bin/systemctl', 'enable', self._appInfo._applicationName]) + print('+++ check configuration in {}/{} and start with:\nsystemctl start {}'.format(self._configPath, self._configFile, starter)) + + def uninstall(self, scriptName, argv): + '''Uninstalls the systemd service. + @param scriptName name of the script in /usr/local/bin, e.g. 'ftptool' + ''' + user = os.geteuid() + if user != 0: + self.usage('you must be root!') + else: + trg = '/usr/local/bin/' + scriptName + purge = False + for arg in argv: + if arg == '--purge': + purge = True + else: + self.usage('unknown option: ' + arg) + BasicStatics.ensureFileDoesNotExist(trg) + if purge: + fn = self._configPath + os.sep + self._configFile + BasicStatics.ensureFileDoesNotExist(fn) + + def uninstallService(self, argv): + '''Uninstalls the systemd service. + @param argv: program arguments + ''' + user = os.geteuid() + if user != 0: + self.usage('you must be root!') + deleteUser = True + deleteGroup = True + purge = False + application = self._appInfo._applicationName + while len(argv) > 0: + if argv[0] == '--purge': + purge = True + elif argv[0] == '--hold-user': + deleteUser = False + elif argv[0] == '--hold-group': + deleteGroup = False + elif argv[0].startswith('--application='): + application = argv[0][14:] + else: + self.usage('unknown option: ' + argv[0]) + argv = argv[1:] + subprocess.call(['/bin/systemctl', 'disable', application]) + systemdFile = '/etc/systemd/system/{}.service'.format(application) + ini = None + if os.path.exists(systemdFile): + ini = base.JavaConfig.JavaConfig(systemdFile, self._logger, True) + os.unlink(systemdFile) + if os.path.exists(systemdFile): + print('+++ cannot delete ' + systemdFile) + else: + print('removed: ' + systemdFile) + user = ini.getString('User', application) if ini != None else application + if deleteUser and BasicStatics.userExists(user): + subprocess.call(['/usr/sbin/userdel', user]) + if BasicStatics.userExists(application): + print('+++ cannot delete user ' + user) + else: + print('user deleted: ' + user) + + group = ini.getString('Group', application) if ini != None else application + if deleteGroup and BasicStatics.groupExists(group): + subprocess.call(['/usr/sbin/groupdel', group]) + if BasicStatics.groupExists(group): + print('+++ cannot delete group ' + group) + else: + print('group deleted: ' + group) + fn = '/usr/local/bin/' + application + if os.path.exists(fn): + os.unlink(fn) + if os.path.exists(fn): + print('+++ cannot remove ' + fn) + else: + print('removed: ' + fn) + def usage(self, message): + '''Display a simplified usage message. + @param message: the error message + ''' + self._logger.error(message) + + def writeExampleConfiguration(self): + '''Writes an example configuration file. + ''' + configFile = self._configPath + os.sep + self._configFile + if os.path.exists(configFile): + configFile = self._configPath + os.sep + self._appInfo._applicationName + '.example.conf' + if type(self._example) == str: + example = self._example + name = None + else: + [example, name, content] = self._example + base.StringUtils.toFile(configFile, example) + print("example configuration written: " + configFile) + if name != None: + base.StringUtils.toFile(name, content) + print("example configuration written: " + name) + +def getGlobalOptions(argv, appInfo, additionalOptions=None): + '''Evaluates the global options. + @param argv: the program arguments + @param appInfo: None or the application info + @param additionalOptions: None or an array of allowed options, e.g. ['m', 'n'] + @return: a tuple (globalOptions, argv) + ''' + def usage(message, options): + tool = BaseTool(options, options._appInfo._applicationName + '.conf') + if len(argv) < 1 or argv[0] != '-q': + print(tool._usage()) + else: + options._verboseLevel = 0 + print('+++ ', message) + + argv = argv[1:] + rc = GlobalOptions(appInfo) + verboseLevel = 1 + if additionalOptions == None: + additionalOptions = [] + while len(argv) > 0 and argv[0].startswith('-'): + arg = argv[0] + if arg.startswith('-l'): + rc._logFiles.append(arg[2:]) + elif arg.startswith('--log='): + rc._logFiles.append(arg[6:]) + elif arg.startswith('-c'): + rc._configDir = arg[2:] + elif arg.startswith('--configuration-directory='): + rc._configDir = arg[26:] + elif arg.startswith('-f'): + rc._configDir = arg[2:] + elif arg.startswith('--configuration-file='): + rc._configFile = arg[11:] + elif arg == '-0' or arg == '--exit-with-0': + rc._exitWith0 = True + elif arg == '-q' or arg == '--quiet': + verboseLevel = 0 + elif arg == '-r' or arg == '--runtime': + rc._runtime = True + elif arg.startswith('--test-source-dir='): + rc._testSourceDir = arg[18:] + elif arg.startswith('--test-target-dir='): + rc._testTargetDir = arg[18:] + elif arg.startswith('-v') or arg.startswith('--verbose-level='): + try: + start = 2 if arg[1] == 'v' else 16 + verboseLevel = int (arg[start:]) + except ValueError: + appInfo._usage('not an integer in ' + arg) + else: + usage('unknown global option ' + arg, rc) + argv = argv[1:] + if verboseLevel not in [0, 1, 2, 3, 4, 5]: + usage('illegal verbose level: ' + str(verboseLevel), rc) + verboseLevel = 0 + rc._verboseLevel = verboseLevel + if len(rc._logFiles) == 0: + rc._logFiles.append('/var/log/local/{}.log'.format(appInfo._applicationName)) + version = platform.python_version() + if not version.startswith('3'): + usage('Wrong python version. We need version 3 and have ' + version) + return (rc, argv) + +if __name__ == '__main__': + pass diff --git a/base/CryptoEngine.py b/base/CryptoEngine.py new file mode 100644 index 0000000..66a95b4 --- /dev/null +++ b/base/CryptoEngine.py @@ -0,0 +1,400 @@ +''' +Created on 05.11.2018 + +@author: hm +''' +import random +import base64 +import math +import time + +class CryptoEngine: + '''Implements a Pseudo Random Generator with the KISS algorithm. + We want an algorithm which can be implemented in any programming language, e.g. in JavaScript or Java. + JavaScript (at this moment) only contains floating point calculation. + Java knows only signed integers or floating point numbers. + Therefore we use IEEE 754 (64 bit floating point). + ''' + + def __init__(self, logger): + '''Constructor. + @param logger: the logger + ''' + self._counter = 0 + self._base64Trailer = '!#$%&()*'; + self._uBoundBase64Tail = '*'; + self._x = 372194.0 + # @cond _y != 0 + self._y = 339219.0 + # @cond z | c != 0 + self._z = 470811222.0 + self._c = 1.0 + self._logger = logger + + def bytesToString(self, aBytes): + '''Converts a string into a byte array without encoding. + @param aBytes: byte array to convert + @return a string + ''' + try: + rc = aBytes.decode('ascii') + except UnicodeDecodeError as exc: + rc = -1 + raise exc + return rc + + def decode(self, string, charSet): + '''Decodes a string encoded by encode(). + Format of the string: version salt encrypted + '0' (version string) + 4 characters salt + rest: the encrypted string + @param string: string to encode + @param charSet: the character set of the string and the result, e.g. 'word' + @return: the decoded string (clear text) + ''' + self._counter += 1 + aSet = self.getCharSet(charSet) + aSize = len(aSet) + rc = '' + if string.startswith('0'): + prefix = string[1:5] + string = string[5:] + aHash = self.hash(prefix) + self.setSeed(aHash, 0x20111958, 0x4711, 1); + length = len(string) + for ix in range(length): + ix3 = aSet.find(string[ix]) + ix2 = (aSize + ix3 - self.nextInt(aSize - 1)) % aSize + rc += aSet[ix2] + return rc + + def decodeBinary(self, string): + '''Decodes a string encrypted by encryptBinary(). + @param string: string to decode + @return: the decoded string (clear text) + ''' + aSet = self.getCharSet('base64') + aSize = len(aSet) + rc = '' + if string.startswith('0'): + prefix = string[1:5] + string = string[5:] + aHash = self.hash(prefix) + self.setSeed(aHash, 0x20111958, 0x4711, 1); + aLen = len(string) + buffer = '' + # replace the trailing '=' "randomly" with a char outside the character set: + if aLen > 0 and string[aLen - 1] == '=': + string[aLen - 1] = self._base64Trailer[self._counter * 7 % len(self._base64Trailer)] + if aLen > 1 and string[aLen - 2] == '=': + string[aLen - 2] = self._base64Trailer[self._counter * 13 % len(self._base64Trailer)] + for ix in range(aLen): + ix3 = aSet.find(string[ix]) + ix2 = (aSize + ix3 - self.nextInt(aSize - 1)) % aSize + buffer += aSet[ix2] + binBuffer = self.stringToBytes(buffer + '\n') + try: + binBuffer2 = base64.decodebytes(binBuffer) + except Exception as exc: + if str(exc) == 'Incorrect padding': + try: + binBuffer = binBuffer[0:-1] + binBuffer2 = base64.decodebytes(binBuffer) + except Exception: + binBuffer = binBuffer[0:-1] + binBuffer2 = base64.decodebytes(binBuffer) + ix = binBuffer2.find(b'\n') + if ix >= 0: + binBuffer2 = binBuffer2[0:ix] + rc = self.bytesToString(binBuffer2) + return rc + + def encode(self, string, charSet): + '''Encodes a string with a randomly generated salt. + Format of the string: version salt encoded + '0' (version string) + 4 characters salt + rest: the encoded string + @param string: string to encode + @param charSet: the character set of the string and the result, e.g. 'word' + @return: the encrypted string + ''' + self._counter += 1 + self.setSeedRandomly() + rc = self.nextString(4, charSet) + aSet = self.getCharSet(charSet) + aSize = len(aSet) + aHash = self.hash(rc) + self.setSeed(aHash, 0x20111958, 0x4711, 1) + length = len(string) + for ix in range(length): + ix3 = aSet.find(string[ix]) + ix2 = (ix3 + self.nextInt(aSize - 1)) % aSize + rc += aSet[ix2] + return '0' + rc + + def encodeBinary(self, string): + '''Encrypts a string with a randomly generated salt. + The string can be based on any char set. It will be base64 encoded before encryption. + Format of the result: version salt encrypted + '0' (version string) + 4 characters salt + rest: the encrypted string + @param string: the string or bytes to encrypt + @return: the encoded string + ''' + self.setSeedRandomly() + if type(string) == str: + string = self.stringToBytes(string) + # convert it to a ascii usable string + string += b'\n' + buffer = base64.encodebytes(string) + string = self.bytesToString(buffer).rstrip() + rc = self.nextString(4, 'base64') + aSet = self.getCharSet('base64') + aSize = len(aSet) + aHash = self.hash(rc) + self.setSeed(aHash, 0x20111958, 0x4711, 1) + length = len(string) + for ix in range(length): + ix3 = aSet.find(string[ix]) + ix2 = (ix3 + self.nextInt(aSize - 1)) % aSize + rc += aSet[ix2] + return '0' + rc + + def getCharSet(self, name): + '''Returns a string with all characters of the charset given by name. + @param name: the name of the charset + @return: None: unknown charset + otherwise: the charset as string + ''' + if name == 'dec': + rc = '0123456789' + elif name == 'hex': + rc = '0123456789abcdef' + elif name == 'upper': + rc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + elif name == 'lower': + rc = 'abcdefghijklmnopqrstuvwxyz' + elif name == 'alfa': + rc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + elif name == 'word': + rc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_' + elif name == 'ascii94': + rc = r'''!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''' + elif name == 'ascii95': + rc = r''' !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''' + elif name == 'ascii': + rc = r''' !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''' + chr(127) + elif name == 'base64': + rc = r'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + else: + self._logger.error('unknown character set: ' + name) + rc = '' + + return rc + + def getCharSetNames(self): + '''Returns the list of the known charset names. + @return the list of the known charset names + ''' + rc = [ + 'dec', + 'hex', + 'upper', + 'lower', + 'alfa', + 'word', + 'ascii94', + 'ascii95', + 'ascii', + 'base64'] + return rc + + def hash(self, string): + '''Converts a string into an integer. + @param string: the string to convert + @return: the hash value + ''' + rc = len(string) + count = rc + for ix in range(count): + rc = (rc * (ix + 1) + (ord(string[ix]) << (ix % 4 * 7))) & 0x7fffffff + return rc + + def nextChar(self, charSet='ascii'): + '''Returns a pseudo random character. + @param charSet: the result is a character from this string + @return: a pseudo random character + ''' + aSet = self.getCharSet(charSet) + ix = self.nextInt(0, len(aSet) - 1) + rc = aSet[ix] + return rc + + def nextInt(self, maxValue=0x7fffffff, minValue = 0): + '''Returns a pseudo random 31 bit integer. + @param maxValue: the maximal return value (inclusive) + @param minValue: the minimal return value (inclusive) + @return: a number from [minValue..maxValue] + ''' + if maxValue == minValue: + rc = minValue + else: + if minValue > maxValue: + temp = maxValue + maxValue = minValue + minValue = temp + rc = self.nextSeed() + rc = rc % (maxValue - minValue) + minValue + return rc + + def nextString(self, length, charSet): + '''Returns a pseudo random string. + @param length: the length of the result + @param charSet: all characters of the result are from this string + @return: a pseudo random string with the given charset and length + ''' + aSet = self.getCharSet(charSet) + aSize = len(aSet) + rc = '' + aRandom = None + for ix in range(length): + if ix % 4 == 0: + aRandom = self.nextSeed() + else: + aRandom >>= 8 + rc += aSet[aRandom % aSize] + return rc + + def nextSeed(self): + '''Sets the next seed and returns a 32 bit random value. + @return: a pseudo random number with 0 <= rc <= 0xffffffff + ''' + # linear congruential generator (LCG): + self._x = math.fmod(69069.0 * self._x + 473219.0, 4294967296) + # Xorshift + #self._y ^= int(self._y) << 13 + self._y = math.fmod(int(self._y) ^ int(self._y) << 13, 4294967296) + #self._y ^= self._y >> 17 + self._y = math.fmod(int(self._y) ^ int(self._y) >> 17, 4294967296) + #self._y ^= self._y << 5 + self._y = math.fmod(int(self._y) ^ int(self._y) << 5, 4294967296) + # multiply with carry: + t = 698769069.0 * self._z + self._c + #self._c = math.fmod(t >> 32, 2) + self._c = math.fmod(int(t) >> 32, 2) + self._z = math.fmod(t, 4294967296) + return int(math.fmod(self._x + self._y + self._z, 4294967296)) + + def oneTimePad(self, user, data): + '''Builds a one time pad. + @param user: the user id + @param data: None or additional data: allowed char set: word + @return: char set: word + ''' + if data != None and self.testCharSet(data, 'word') >= 0: + rc = '' + else: + padData = '{:08x}{:04x}'.format(int(round(time.time())), user) + data + rc = self.encode(padData, 'word'); + return rc + + def restoreSeed(self, seed): + '''Returns the current seed as string. + @return the seed as string + ''' + parts = seed.split(':') + self.setSeed(float(parts[0]), float(parts[1]), float(parts[2]), float(parts[3])) + + def saveSeed(self): + '''Returns the current seed as string. + @return the seed as string + ''' + rc = '{}:{}:{}:{}'.format(repr(self._x), repr(self._y), repr(self._z), repr(self._c)) + return rc + + def setSeed(self, x, y, z, c): + '''Sets the parameter of the KISS algorithm. + @param x: + @param y: + @param z: + @param c: + ''' + self._x = math.fmod(x, 4294967296) + self._y = 1234321.0 if y == 0 else math.fmod(y, 4294967296) + if z == 0 and c == 0: + c = 1.0 + self._c = math.fmod(c, 2) + self._z = math.fmod(z, 4294967296) + + def setSeedFromString(self, seedString): + '''Converts a string, e.g. a password, into a seed. + @param seedString: the string value to convert + ''' + if seedString == '': + seedString = 'Big-Brother2.0IsWatching!You' + while len(seedString) < 8: + seedString += seedString + x = self.hash(seedString[0:len(seedString) - 3]) + y = self.hash(seedString[1:8]) + z = self.hash(seedString[3:5]) + c = self.hash(seedString[1:]) + self.setSeed(x, y, z, c) + + def setSeedRandomly(self): + '''Brings "true" random to the seed + ''' + utime = time.time() + rand1 = int(math.fmod(1000*1000*utime, 1000000000.0)) + rand2 = int(math.fmod(utime*1000, 1000000000.0)) + self.setSeed(rand1, rand2, int(random.random() * 0x7fffffff), 1) + + def stringToBytes(self, string): + '''Converts a string into a byte array without encoding. + @param string: string to convert + @return a bytes array + ''' + rc = string.encode('ascii') + return rc + + def testCharSet(self, string, charSet): + '''Tests whether all char of a string belong to a given charSet. + @param string: string to test + @param charSet: the char set to test + @return: -1: success + otherwise: the index of the first invalid char + ''' + aSet = self.getCharSet(charSet) + rc = -1 + for ix in range(len(string)): + if aSet.find(string[ix]) < 0: + rc = ix + break + return rc + + def unpackOneTimePad(self, pad, maxDiff = 60): + '''Decodes a one time pad. + @param pad: the encoded one time pad + @param maxDiff: maximal difference (in seconds) between time of the pad and now + @return: None: invalid pad + otherwise: a tuple (time, user, data) + ''' + padData = self.decode(pad, 'word') + length = len(padData) + if length < 12 or self.testCharSet(padData[0:12], 'hex') >= 0 or self.testCharSet(padData[12:], 'word') >= 0: + rc = None + else: + padTime = int(padData[0:8], 16) + now = time.time() + if abs(now - padTime) >= maxDiff: + rc = None + else: + user = int(padData[8:12], 16) + data = None if len(padData) == 12 else padData[12:] + rc = (padTime, user, data) + return rc + +if __name__ == '__main__': + pass diff --git a/base/FileHelper.py b/base/FileHelper.py new file mode 100644 index 0000000..1045452 --- /dev/null +++ b/base/FileHelper.py @@ -0,0 +1,678 @@ +''' +Created on 25.04.2018 + +@author: hm +''' + +import os +import stat +import datetime +import time +import shutil +import re +import tarfile +import zipfile +import tempfile + +import base.StringUtils +import base.LinuxUtils + +fileHelperRegExprWildcards = re.compile(r'[*?\[\]]') + +class DirInfo: + '''Stores the directory info + ''' + def __init__(self, maxYoungest=5, maxLargest=5, maxOldest=5, maxSmallest=5, minSize=1, dirsOnly=False, filesOnly=False, trace=0): + '''Constructor. + @param maxYoungest: the maximal number of entries in self._youngest + @param maxLargest: the maximal number of entries in self._largest + @param maxOldest: the maximal number of entries in self._oldest + @param maxLargest: the maximal number of entries in self._smallest + @param minSize: the minimum size of the entries in self._smallest + @param dirsOnly: True: only directories will be processed + @param filesOnly: True: only files (not dirs) will be processed + @param trace: if > 0: after processing this amount of nodes a statistic is logged + ''' + self._fileCount = 0 + self._fileSizes = 0 + self._dirCount = 0 + self._dirPattern = None + self._filePattern = None + self._ignoredDirs = 0 + self._ignoredFiles = 0 + self._youngest = [] + self._largest = [] + self._smallest = [] + self._oldest = [] + self._maxYoungest = maxYoungest + self._maxLargest = maxLargest + self._maxLargest = maxOldest + self._maxSmallest = maxSmallest + self._minSize = minSize + self._timeYoungest = 0 + self._timeOldest = 0 + self._sizeLargest = 0 + self._dirsOnly = dirsOnly + self._filesOnly = filesOnly + self._trace = trace + self._nextTracePoint = trace + +def _error(message, logger = None): + '''Prints an error message. + @param message: error message + @param logger: None or the logger + ''' + if logger == None: + print('+++ ' + message) + else: + logger.error(message) + +def _log(message, logger = None): + '''Prints a message. + @param message: error message + @param logger: None or the logger + ''' + if logger == None: + print(message) + else: + logger.log(message) + +def clearDirectory(path, logger=None): + '''Deletes (recursivly) all files and subdirectories of a given path. + Note: if the path is not a directory (or it does not exists) it will not be handled as an error + @param path: the directory to clear + @param logger: None or an error reporter + ''' + if os.path.exists(path): + for node in os.listdir(path): + full = path + os.sep + node + if os.path.isdir(full): + shutil.rmtree(full, True) + else: + os.unlink(full) + if os.path.exists(full) and logger != None: + _error('cannot remove: ' + full, logger) + +def copyDirectory(source, target, option = None, logger=None, verboseLevel = 0): + '''Copies all files (and dirs) from source to target directory. + @param source: the base source directory + @param target: the base target directoy() + @param option: None, 'clear' or 'update' + 'clear': all files (and subdirs) of target will be deleted + 'update': only younger or not existing files will be copied False: all files will be copied + @param logger: None or the logger + ''' + if option == 'clear': + if verboseLevel >= 2: + _log('clearing ' + target, logger) + clearDirectory(target, logger) + for node in os.listdir(source): + src = source + os.sep + node + trg = target + os.sep + node + if os.path.islink(src): + if not option == 'update' or not os.path.exists(trg): + ref = os.readlink(src) + if verboseLevel >= 2: + _log('symlink: {} [{}]'.format(trg, ref)) + try: + os.symlink(ref, trg) + except OSError as exc: + _error('cannot create a symlink: {} -> {}'.format(ref, trg), logger) + elif os.path.isdir(src): + if option != 'update' or not os.path.exists(trg): + if verboseLevel >= 2: + _log('directory: {} -> {}'.format(src, trg)) + shutil.copytree(src, trg, True) + else: + copyDirectory(src, trg, option, logger) + else: + if not os.path.exists(trg) or option == 'update' and os.path.getmtime(src) > os.path.getmtime(trg): + try: + if verboseLevel >= 2: + _log('{} -> {}'.format(src, trg)) + shutil.copy2(src, trg) + except OSError as exc: + _error('cannot copy {}: {}'.format(trg, str(exc))) + +def copyIfExists(source, target, option = None, logger=None): + '''Copies all files (and dirs) from source to target directory. + @param source: the base source directory + @param target: the base target directoy() + @param options: None + @param logger: error logger + @param verboseLevel: True: do logging + ''' + if os.path.exists(source): + if logger != None: + logger.log('copying {} => {} ...'.format(source, target), 2) + shutil.copy2(source, target) + +def directoryInfo(path, filePattern = None, dirPattern = None, maxDepth = -1, fileInfo = None, + maxYoungest=5, maxLargest=5, maxOldest=5, maxSmallest=5, minSize=1, dirsOnly=False, + filesOnly=False, trace=0): + '''Returns the directory info of the given path. + @param path: the full path of the directory to inspect + @param filePattern: None or a regular expression (as text) describing the file names to inspect + @param dirPattern: None or a regular expression (as text) describing the directory names to inspect + @param maxDepth: maximal depth of recursion. < 0: unlimited 0: only the start directory + @param dirInfo: None or a DirectoryInfo instance which will be completed + @param maxYoungest: the maximal number of entries in DirInfo._youngest[] + @param maxLargest: the maximal number of entries in DirInfo._largest[] + @param maxSmallest: the maximal number of entries in DirInfo._smallest[] + @param minSize: the minimum size of the entries in DirInfo._smallest[] + @param dirsOnly: only directories will be part of the result + @param filessOnly: only files (not directories) will be part of the result + @param trace: if > 0: a statistic is printed if this amount of nodes (files or nodes) is processed + @return: a DirInfo instance + ''' + def infoOneDir(path, depth, fileInfo): + def showStatistic(info): + print('{}: dirs: {} files: {} ignored dirs: {} ignored files: {}'.format(path, info._dirCount, info._fileCount, info._ignoredDirs, info._ignoredFiles)) + info._nextTracePoint += info._trace + if not isinstance(fileInfo, DirInfo): + depth = 0 + fileInfo._dirCount += 1 + if fileInfo._trace > 0 and fileInfo._dirCount + fileInfo._fileCount + fileInfo._ignoredDirs + fileInfo._ignoredFiles > fileInfo._nextTracePoint: + showStatistic(fileInfo) + try: + nodes = os.listdir(path) + except PermissionError: + fileInfo._ignoredDirs += 1 + return + if fileInfo._trace > 0 and len(nodes) == 0 and fileInfo._dirCount + fileInfo._fileCount + fileInfo._ignoredDirs + fileInfo._ignoredFiles % fileInfo._trace == 0: + showStatistic(fileInfo) + for node in nodes: + if fileInfo._trace > 0 and fileInfo._dirCount + fileInfo._fileCount + fileInfo._ignoredDirs + fileInfo._ignoredFiles > fileInfo._nextTracePoint: + showStatistic(fileInfo) + full = path + os.sep + node + stats = os.lstat(full) + isDir = stat.S_ISDIR(stats.st_mode) + if isDir: + if not fileInfo._filesOnly: + length = len(fileInfo._youngest) + if fileInfo._maxYoungest > 0 and (length < fileInfo._maxYoungest or stats.st_mtime > fileInfo._timeYoungest): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxYoungest: + del fileInfo._youngest[0] + fileInfo._youngest.append(str(stats.st_mtime) + ':' + path + os.sep + node) + fileInfo._youngest.sort(key=lambda x: float(x.split(':')[0])) + fileInfo._timeYoungest = float(fileInfo._youngest[0].split(':')[0]) + length = len(fileInfo._oldest) + if fileInfo._maxOldest > 0 and (length < fileInfo._maxOldest or stats.st_mtime < fileInfo._timeOldest): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxOldest: + del fileInfo._oldest[-1] + fileInfo._oldest.insert(0, str(stats.st_mtime) + ':' + path + os.sep + node) + fileInfo._oldest.sort(key=lambda x: float(x.split(':')[0])) + fileInfo._timeOldest = float(fileInfo._oldest[-1].split(':')[0]) + if (fileInfo._dirPattern == None or fileInfo._dirPattern.match(node) == None) and (maxDepth == None or maxDepth < 0 or depth < maxDepth): + infoOneDir(path + os.sep + node, depth + 1, fileInfo) + else: + fileInfo._ignoredDirs += 1 + else: # not isDir + if fileInfo._dirsOnly: + fileInfo._ignoredFiles += 1 + continue + if (fileInfo._filePattern == None or fileInfo._filePattern.match(node) != None): + fileInfo._fileSizes += stats.st_size + fileInfo._fileCount += 1 + length = len(fileInfo._largest) + if fileInfo._maxLargest > 0 and (length < fileInfo._maxLargest or stats.st_size > fileInfo._sizeLargest): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxLargest: + del fileInfo._largest[0] + fileInfo._largest.append(str(stats.st_size) + ':' + path + os.sep + node) + fileInfo._largest.sort(key=lambda x: float(x.split(':')[0])) + fileInfo._sizeLargest = float(fileInfo._largest[-1].split(':')[0]) + length = len(fileInfo._smallest) + if fileInfo._maxSmallest > 0 and (stats.st_size >= fileInfo._minSize and (length < fileInfo._maxSmallest or stats.st_size > fileInfo._sizeSmallest)): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxSmallest: + del fileInfo._smallest[-1] + fileInfo._smallest.insert(0, str(stats.st_size) + ':' + path + os.sep + node) + fileInfo._smallest.sort(key=lambda x: float(x.split(':')[0])) + fileInfo._sizeSmallest = float(fileInfo._smallest[-1].split(':')[0]) + length = len(fileInfo._youngest) + if fileInfo._maxYoungest > 0 and (length < fileInfo._maxYoungest or stats.st_mtime > fileInfo._timeYoungest): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxYoungest: + del fileInfo._youngest[0] + fileInfo._youngest.append(str(stats.st_mtime) + ':' + path + os.sep + node) + fileInfo._youngest.sort(key=lambda x: float(x.split(':')[0])) + fileInfo._timeYoungest = float(fileInfo._youngest[0].split(':')[0]) + length = len(fileInfo._oldest) + if fileInfo._maxOldest > 0 and (length < fileInfo._maxOldest or stats.st_mtime < fileInfo._timeOldest): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxOldest: + del fileInfo._oldest[-1] + fileInfo._oldest.insert(0, str(stats.st_mtime) + ':' + path + os.sep + node) + fileInfo._oldest.sort(key=lambda x: float(x.split(':')[0])) + fileInfo._timeOldest = float(fileInfo._oldest[-1].split(':')[0]) + else: + fileInfo._ignoredFiles += 1 + continue + # end of infoOneDir() + if fileInfo == None: + fileInfo = DirInfo(maxYoungest, maxLargest, maxOldest, maxSmallest, minSize, dirsOnly, filesOnly, trace) + if filePattern != None: + fileInfo._filePattern = base.StringUtils.regExprCompile(filePattern, 'file pattern') + if dirPattern != None: + fileInfo._filePattern = base.StringUtils.regExprCompile(dirPattern, 'dir pattern') + + euid = os.geteuid() + egid = os.getegid() + fileInfo._maxYoungest = maxYoungest + fileInfo._maxLargest = maxLargest + fileInfo._maxOldest = maxOldest + fileInfo._maxDepth = maxDepth + infoOneDir(path, 0, fileInfo) + return fileInfo + +def distinctPaths(path1, path2): + '''Tests whether two paths are not part of each other. + @param path1: first path to test + @param path2: 2nd path to test + @return: True: path1 is not parent of path2 and path2 is not parent of path1 + ''' + dir1 = os.path.realpath(path1) + dir2 = os.path.realpath(path2) + return not dir1.startswith(dir2) and not dir2.startswith(dir1) + +def ensureDirectory(directory, logger = None, mode=0o777): + '''Ensures that the given directory exists. + @param directory: the complete name + @param logger: None or the logger + @return: None: could not create the directory + otherwise: the directory's name + ''' + if not os.path.isdir(directory): + try: + os.lstat(directory) + os.unlink(directory) + except FileNotFoundError: + pass + _log('creating {}{} ...'.format(directory, os.sep), logger) + try: + os.makedirs(directory, mode) + except OSError as exc: + _error('cannot create dir {}: {}'.format(directory, str(exc), logger)) + if not os.path.isdir(directory): + directory = None + return directory + +def ensureFileDoesNotExist(filename, logger = None): + '''Ensures that a file does not exist. + @param filename: the file to delete if it exists. + @param logger: None or the logger + ''' + try: + info = os.lstat(filename) + try: + if os.path.isdir(filename): + _log('removing {}{} ...'.format(filename, os.sep), logger) + shutil.rmtree(filename, False) + else: + _log('removing {} ...'.format(filename), logger) + os.unlink(filename) + except OSError as exp: + _error('cannot delete {:s}: {:s}'.format(filename, str(exp)), logger) + except FileNotFoundError: + pass + +def ensureFileExists(filename, content = '', logger = None): + '''Ensures that a file does not exist. + @param filename: the file to create if it does not exist + @param content: this text will be stored for a new created file + @param logger: None or the logger + ''' + try: + if os.path.exists(filename): + if os.path.isdir(filename): + _log('is a directory: {}'.format(filename), logger) + else: + _log('creating {} ...'.format(filename), logger) + base.StringUtils.toFile(filename, content) + except OSError as exc: + _error('problems with {}: {}'.format(filename, str(exc))) + +def ensureSymbolicLink(source, target, createTarget=True, logger = None): + '''Ensures that a directory exists. + @param source: the full name of the link source, e.g. '../sibling' + @param target: full name of the file of type 'link' + @param createTarget: creates the target if it does not exist + @param logger: None or the logger + @return: True: the link exists + ''' + info = None + try: + info = os.lstat(target) + except FileNotFoundError: + pass + if info != None: + if os.path.islink(target): + oldLink = os.readlink(target) + if oldLink != source: + _log('changing link from {} to {}'.format(oldLink, source), logger) + os.unlink(target) + elif os.path.isdir(target): + _error('target {} is already a directory (not a link)'.format(target), logger) + else: + _log('removing the non link file ' + target, logger) + os.unlink(target) + if not os.path.exists(target): + baseDir = os.path.dirname(target) + if not os.path.isdir(baseDir): + if createTarget: + ensureDirectory(baseDir, logger) + hasParent = os.path.isdir(baseDir) + if not hasParent: + _error('parent of target is not a directory: ' + baseDir, logger) + absSource = base.FileHelper.joinPaths(source, target, logger) if source.startswith('..') else source + if not os.path.exists(absSource): + _error('missing source {} [= {}]'.format(source, absSource), logger) + elif hasParent: + _log('creating symbol link {} -> {}'.format(source, target), logger) + os.symlink(source, target) + rc = os.path.islink(target) and os.readlink(target) == source + return rc + +def fileClass(path): + '''Returns the file class of the file. + @param path: the full filename + @return: a tuple (class, subclass): class: 'container', 'text', 'binary', 'unknown' + subclass of 'container': 'dir', 'tar', 'tgz', 'zip' + subclass of 'text': 'xml', 'shell' + ''' + def isBinaryByte(bb): + rc = bb < 0x09 or bb > 0x0d and bb < 0x20 + return rc + def isBinary(byteArray): + found = 0 + rc = False + for ix in range(len(byteArray)): + bb = byteArray[ix] + if bb == b'\x00': + rc = True + break; + elif isBinaryByte(bb): + found += 1 + if found > 100 or found > len(byteArray) / 10: + rc = True + break + return rc + def isNullString(byteArray): + '''Tests whether the byteArray is a text delimited with 0. + @param byteArray: array to test + @return True: only text and '\0' is part of byteArray + ''' + ix = 0 + rc = True + hasNull = False + while ix < len(byteArray): + if byteArray[ix] == 0: + hasNull = True + elif isBinaryByte(byteArray[ix]): + rc = False + break + ix += 1 + return rc and hasNull + def isNullNumber(byteArray): + '''Tests whether the byteArray are digits delimited with 0. + @param byteArray: array to test + @return True: only decimal digits and '\0' is part of byteArray + ''' + ix = 0 + rc = True + hasNull = False + while ix < len(byteArray): + if byteArray[ix] == 0: + hasNull = True + elif not (byteArray[ix] >= 0x30 and byteArray[ix] <= 0x39): # TAB + rc = False + break + ix += 1 + return rc and hasNull + if os.path.isdir(path): + (theClass, subClass) = ('container', 'dir') + else: + with open(path, 'rb') as fp: + start = fp.read(4096) + if start.startswith(b'\x1f\x8b\x08'): + (theClass, subClass) = ('container', 'tar') + elif start.startswith(b'BZ') and isBinary(start[8:80]): + (theClass, subClass) = ('container', 'tar') + elif start.startswith(b'PK') and isBinary(start[2:32]): + (theClass, subClass) = ('container', 'zip') + elif isNullString(start[0:100]) and isNullNumber(start[100:0x98]): + (theClass, subClass) = ('container', 'tar') + elif (start[0:100].lower().find(b'<xml>') >= 0 or start[0:100].lower().find(b'<html') >= 0) and not isBinary(start): + (theClass, subClass) = ('text', 'xml') + elif len(start) > 5 and start.startswith(b'#!') and not isBinary(start): + (theClass, subClass) = ('text', 'shell') + elif isBinary(start): + (theClass, subClass) = ('binary', 'binary') + else: + (theClass, subClass) = ('text', 'text') + return (theClass, subClass) + +def fileType(path): + '''Returns the file type: 'file', 'dir', 'link', 'block' + @param path: the full filename + @return: the filetype: 'file', 'dir', 'link', 'block', 'char' + ''' + if os.path.islink(path): + rc = 'link' + elif os.path.isdir(path): + rc = 'dir' + else: + rc = 'file' + return rc + +def fromBytes(line): + '''Converts a line with type bytes into type str. + @param line: line to convert + ''' + try: + rc = line.decode() + except UnicodeDecodeError: + try: + rc = line.decode('latin-1') + except UnicodeDecodeError: + rc = line.decode('ascii', 'ignore') + return rc + +def hasWildcards(filename): + '''Tests whether a filename has wildcards. + @param filename: filename to test + @return: True: the filename contains wildcard like '*', '?' or '[...]' + ''' + global fileHelperRegExprWildcards + rc = fileHelperRegExprWildcards.search(filename) != None + return rc + +def joinPaths(relPath, start = None, logger = None): + '''Joins a relative path and a start path to a non relative path. + Example: joinPath('../brother', '/parent/sister') is '/parent/brother' + @param relPath: the relative path, e.g. '../sister' + @param start: the start point for joining, e.g. 'family/sister'. If None: the current directory + @param logger: for error logging + @returns the non relative path, e.g. 'family/brother' + ''' + rc = None + relParts = relPath.split(os.sep) + if start == None: + start = os.curdir + startParts = start.split(os.sep) + if len(relParts) == 0 or relParts[0] != '..': + if logger != None: + logger.error('not a relative path: ' + relPath) + else: + rc = '' + while len(relParts) > 0 and relParts[0] == '..': + if len(startParts) == 0: + if logger != None: + logger.error('too many backsteps in relpath {} for start {}'.format(relPath, start)) + rc = None + break + relParts = relParts[1:] + startParts = startParts[0:-1] + if rc != None: + rc = os.sep.join(startParts) + if len(relParts) > 0: + if rc == '': + rc = os.sep.join(relParts) + else: + rc += os.sep + os.sep.join(relParts) + return rc + +def listFile(statInfo, full, orderDateSize = True, humanReadable = True): + '''Builds the info for one file (or directory) + @param statInfo: the info returned by os.(l)stat() + @param full: the filename + @param orderDateSize: True: order is date left of size False: order is size leftof date + @param humanReadable: True: better for reading (matching unit), e.g. "10.7 GByte" or "3 kByte" + ''' + if stat.S_ISDIR(statInfo.st_mode): + size = '<dir>' + elif humanReadable: + size = "{:>8s}".format(base.StringUtils.formatSize(statInfo.st_size)) + else: + size = '{:13.6f} MB'.format(statInfo.st_size / 1000000) + fdate = datetime.datetime.fromtimestamp(statInfo.st_mtime) + dateString = fdate.strftime("%Y.%m.%d %H:%M:%S") + if orderDateSize: + rc = '{:s} {:>12s} {:s}'.format(dateString, size, full) + else: + rc = '{:>12s} {:s} {:s}'.format(size, dateString, full) + return rc + +def pathToNode(path): + '''Changed a path into a name which can be used as node (of a filename). + @param path: the path to convert + @return: path with replaced path separators + ''' + rc = path.replace(os.sep, '_').replace(':', '_') + return rc + +def splitFilename(full): + '''Splits a filename into its parts. + @param full: the filename with path + @return: a dictionary with the keys 'full', 'path', 'node', 'fn', 'ext' + example: { 'full': '/home/jonny.txt', 'path': '/home/', 'node' : 'jonny.txt', 'fn': 'jonny' , 'ext': '.txt' } + ''' + rc = dict() + rc['full'] = full + ix = full.rfind(os.sep) + if ix < 0: + rc['path'] = '' + node = rc['node'] = full + else: + rc['path'] = full[0:ix+1] + node = rc['node'] = full[ix+1:] + ix = node.rfind('.', 1) + if ix < 0: + rc['fn'] = node + rc['ext'] = '' + else: + rc['fn'] = node[0:ix] + rc['ext'] = node[ix:] + return rc + +def setModified(path, timeUnix, date = None): + '''Sets the file modification time. + @precondition: exactly one of date and timeUnix must be None and the other not None + @param path: the full path of the file to modify + @param timeUnix: None or the time to set (unix timestamp since 1.1.1970) + @param date: None or the datetime to set (datetime.datetime instance) + @return: True: success False: precondition raised + ''' + dateModified = None + rc = True + if date != None: + dateModified = time.mktime(date.timetuple()) + elif timeUnix == None: + rc = False + else: + dateModified = timeUnix + if dateModified != None: + try: + os.utime(path, (dateModified, dateModified)) + except Exception as exc: + msg = str(exc) + raise exc + return rc + +def tail(filename, maxLines = 1, withLineNumbers = False): + lines = [] + if maxLines < 1: + maxLines = 1 + with open(filename, "r") as fp: + lineNo = 0 + for line in fp: + lineNo += 1 + if len(lines) >= maxLines: + del lines[0] + lines.append(line) + if withLineNumbers: + lineNo -= len(lines) - 1 + for ix in range(len(lines)): + lines[ix] = '{}: {}'.format(lineNo, lines[ix]) + lineNo += 1 + return lines + +def tempFile(node, subDir = None, logger = None): + '''Returns the name of a file laying in the temporary directory. + @param node: the filename without path + @param subdir: None or a subdirectory in the temp directory (may be created) + @param logger: error logger + ''' + path = tempfile.gettempdir() + os.sep + if subDir != None: + path += subDir + os.makedirs(path, 0o777, True) + path += os.sep + path += node + return path + +def unpack(archive, target, logger = None, clear = False): + '''Copies the content of an archive (tar, zip...) into a given directory. + @param archive: name of the archive, the extension defines the type: '.tgz': tar '.zip': zip + @param target: the directory which will be filled by the archive content. Will be created if needed + @param logger: None or logger + ''' + if not os.path.exists(target): + os.makedirs(target, 0o777, True) + elif not os.path.isdir(target): + _error('target is not a directory: ' + target) + archive = None + elif clear: + clearDirectory(target, logger) + if archive == None: + pass + elif archive.endswith('.tgz'): + tar = tarfile.open(archive, 'r:gz') + tar.extractall(target) + elif archive.endswith('.zip'): + zip = zipfile.ZipFile(archive, 'r') + zip.extractall(target) + else: + _error('unknown file extend: ' + archive, logger) + +if __name__ == '__main__': + info = directoryInfo('/etc') + print('{}: file(s): {} / {:.3f} MB dir(s): {} ignored (files/dirs): {} / {}'.format('/etc', info._fileCount, info._fileSizes / 1024 / 1024.0, + info._dirCount, info._ignoredFiles, info._ignoredDirs)) + lines = tail('/etc/fstab', 5, True) + print('{}:\n{}'.format('/etc/fstab', ''.join(lines))) diff --git a/base/JavaConfig.py b/base/JavaConfig.py new file mode 100644 index 0000000..72829e1 --- /dev/null +++ b/base/JavaConfig.py @@ -0,0 +1,92 @@ +''' +Created on 18.07.2018 + +@author: hm +''' +import re +import os.path + + +class JavaConfig(object): + ''' + Handles a java style configuration file. + Format: + <variable> = <value> + # comment + ''' + + def __init__(self, filename, logger, ignoreIniHeader = False): + ''' + Constructor. + @param filename: the filename with path + @param logger: the logger + @param ignoreIniHeader: True: '[<section>]' will be ignored + ''' + self._ignoreIniHeader = ignoreIniHeader + self._vars = dict() + self._logger = logger + self.readConfig(filename) + + def readConfig(self, filename): + self._filename = filename + self._vars = dict() + regExpr = re.compile('([\w.]+)\s*=\s*(.*)$') + if not os.path.exists(filename): + self._logger.error('missing ' + filename) + else: + with open(filename, "r") as fp: + lineNo = 0 + for line in fp: + lineNo += 1 + line = line.strip() + if line.startswith('#') or line == '': + continue + matcher = regExpr.match(line) + if matcher != None: + self._vars[matcher.group(1)] = matcher.group(2) + elif self._ignoreIniHeader and line.startswith('['): + continue + else: + self._logger.error('{:s} line {:d}: unexpected syntax [expected: <var>=<value>]: {:s}'.format(filename, lineNo, line)) + + def getString(self, variable, defaultValue=None): + '''Returns the value of a given variable. + @param variable: name of the Variable + @param defaultValue: if variable does not exist this value is returned + @return: None: Variable not found otherwise: the value + ''' + rc = defaultValue if variable not in self._vars else self._vars[variable] + return rc + + def getInt(self, variable, defaultValue=None): + '''Returns the value of a given variable. + @param variable: name of the Variable + @param defaultValue: if variable does not exist this value is returned + @return: None: Variable not found or not an integer + otherwise: the int value + ''' + rc = defaultValue + if variable in self._vars: + value = self._vars[variable] + try: + rc = int(value) + except ValueError: + self._logger.error("{}: variable {} is not an integer: {}".format(self._filename, variable, value)) + rc = defaultValue + return rc + + def getKeys(self, regExpr = None): + '''Returns an array of (filtered) keys. + @param regExpr: None or a regular expression to filter keys. regExpr can be an object or a text + example: re.compile(r'^\s*pattern.\d+$', re.I) + @return: the array of sorted keys matching the regExpr + ''' + if type(regExpr) == str: + regExpr = re.compile(regExpr) + keys = self._vars.keys() + rc = [] + for key in keys: + if regExpr == None or regExpr.search(key): + rc.append(key) + rc.sort() + return rc diff --git a/base/LinuxUtils.py b/base/LinuxUtils.py new file mode 100644 index 0000000..5272ea5 --- /dev/null +++ b/base/LinuxUtils.py @@ -0,0 +1,234 @@ +''' +Created on 25.04.2018 + +@author: hm +''' + +import os +import base.StringUtils +import subprocess +import re +import stat + +def diskFree(verboseLevel = 0, logger = None): + '''Returns an info about the mounted filesystems. + @return: a list of info entries: entry: [name, totalBytes, freeBytes] + ''' + if logger != None and verboseLevel > 3: + logger.log('taskFileSystem()...') + + rc = [] + ignoredDevs = ['udev', 'devpts', 'tmpfs', 'securityfs', 'pstore', 'cgroup', 'tracefs', 'mqueue', 'hugetlbfs', 'debugfs'] + with open('/proc/mounts', 'r') as f: + for line in f: + dev, path, fstype = line.split()[0:3] + if logger != None and verboseLevel > 3: + logger.log(line) + if (fstype == 'sysfs' or fstype == 'proc' or dev in ignoredDevs or path.startswith('/proc/') + or path.startswith('/sys/') or path.startswith('/run/') or path.startswith('/dev/loop') + or path.startswith('/snap/') or not os.path.isdir(path)): + continue + if logger != None and verboseLevel > 3: + logger.log(path + '...') + try: + stat = os.statvfs(path) + blocksize = stat.f_bsize + rc.append([path, stat.f_blocks*blocksize, stat.f_bfree*blocksize, stat.f_bavail*blocksize]) + except FileNotFoundError: + # if mounted by autofs: the path can not be found + pass + return rc + +def diskIo(): + '''Returns a list of [diskname, countReads, countWrites, countDiscards] arrays. + Data are accumulated since last boot. + Note: sector size: 512 Byte + @see https://www.kernel.org/doc/Documentation/iostats.txt + @return: array of arrays [id, diskname, countReads, countWrites, countDiscards], e.g. [ ['8-0-sda', 299, 498, 22 ] ] + ''' + rc = [] + with open('/proc/diskstats', 'r') as fp: + for line in fp: + # 1......2.....3....4.............5...........6...........7.........8..............9............10...........11.........12.........13.....14.............15................16.............17...............18 + # mainid subid name readscomplete readsmerged readsectors readmsecs writescomplete writesmerged writesectors writesmsec inprogress iomsec weightediomsec discardscompleted discardsmerged discardssectors discardsmsec + # 8 0 sda 101755 2990 6113900 37622 69827 44895 1535408 41169 0 85216 2732 0 0 0 0 + # 8 1 sda1 82 0 6368 22 0 0 0 0 0 76 0 0 0 0 0 + parts = line.split() + rc.append(['{}-{}'.format(parts[0], parts[1]), parts[2], parts[5], parts[9], parts[16]]) + return rc + +def isExecutable(statInfo, euid, egid): + '''Tests whether the file or directory) is executable + @param statInfo: the result of os.stat() + @param euid: the effective UID of the current process. We can get it with os.geteuid() + @param egid: the the effective GID of the current process. We can get it with os.getegid() + @return: True: the file is executable + ''' + if statInfo.st_uid == euid: + # S_IXUSR S_IXGRP S_IXOTH + mask = (stat.S_IXUSR | stat.S_IXOTH) + elif statInfo.st_gid == egid: + mask = (stat.S_IXGRP | stat.S_IXOTH) + else: + mask = stat.S_IXOTH + return (statInfo.st_mode & mask) != 0 + +def isReadable(statInfo, euid, egid): + '''Tests whether the file or directory) is readable. + @param statInfo: the result of os.stat() + @param euid: the effective UID of the current process. We can get it with os.geteuid() + @param egid: the the effective GID of the current process. We can get it with os.getegid() + @return: True: the file is readable + ''' + if statInfo.st_uid == euid: + # S_IXUSR S_IXGRP S_IXOTH + mask = (stat.S_IRUSR | stat.S_IROTH) + elif statInfo.st_gid == egid: + mask = (stat.S_IRGRP | stat.S_IROTH) + else: + mask = stat.S_IROTH + return (statInfo.st_mode & mask) != 0 + +def isWritable(statInfo, euid, egid): + '''Tests whether the file or directory) is writable. + @param statInfo: the result of os.stat() + @param euid: the effective UID of the current process. We can get it with os.geteuid() + @param egid: the the effective GID of the current process. We can get it with os.getegid() + @return: True: the file is writable + ''' + if statInfo.st_uid == euid: + mask = (stat.S_IWUSR | stat.S_IWOTH) + elif statInfo.st_gid == egid: + mask = (stat.S_IWGRP | stat.S_IWOTH) + else: + mask = stat.S_IWOTH + return (statInfo.st_mode & mask) != 0 + +def stress(patternDisks, patternInterface): + '''Returns the load data of a server. + Note: the byte data (ioReadBytes ... netWriteBytes) are summarized since boot time. + @param patternDisk: a regular expression of the disk devices used for the result (sum is built), e.g. 'sd[ab]' + @param patternInterface: a regular expression of the network interfaces used for the result (sum is built), e.g. 'eth0|wlan0' + @return: [ioReadBytes, ioWriteBytes, netReadBytes, netWriteBytes, load1Minute, memoryAvailable, swapAvailable] + ''' + readIO = 0 + writeIO = 0 + rexprDisks = base.StringUtils.regExprCompile(patternDisks, 'disk pattern') + with open('/proc/diskstats', 'r') as fp: + for line in fp: + # 1......2.....3....4.............5...........6...........7.........8..............9............10...........11.........12.........13.....14.............15................16.............17...............18 + # mainid subid name readscomplete readsmerged readsectors readmsecs writescomplete writesmerged writesectors writesmsec inprogress iomsec weightediomsec discardscompleted discardsmerged discardssectors discardsmsec + # 8 0 sda 101755 2990 6113900 37622 69827 44895 1535408 41169 0 85216 2732 0 0 0 0 + # 8 1 sda1 82 0 6368 22 0 0 0 0 0 76 0 0 0 0 0 + parts = line.split() + if rexprDisks.match(parts[2]) != None: + readIO += int(parts[5]) + writeIO += int(parts[9]) + readIO *= 512 + writeIO *= 512 + readNet = 0 + writeNet = 0 + rexprNet = base.StringUtils.regExprCompile(patternInterface, 'interface pattern') + with open('/proc/net/dev', 'r') as fp: + for line in fp: + # 1......2........3......4....5....6....7.....8..........9.........10.......11.....12....13...14...15....16......17 + # Inter-| Receive | Transmit + # face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed + # lo: 33308 376 0 0 0 0 0 0 33308 376 0 0 0 0 0 0 + parts = line.split() + # remove ':' from the first field: + if rexprNet.match(parts[0][0:-1]) != None: + readNet += int(parts[1]) + writeNet += int(parts[9]) + with open('/proc/loadavg', 'rb') as fp: + loadMin1 = float(fp.read().decode().split()[0]) + #@return: [TOTAL_RAM, AVAILABLE_RAM, TOTAL_SWAP, FREE_SWAP, BUFFERS] + with open('/proc/meminfo', 'r') as fp: + lines = fp.read().split('\n') + freeRam = _getNumber(lines[2]) + freeSwap = _getNumber(lines[15]) + return [readIO, writeIO, readNet, writeNet, loadMin1, freeRam, freeSwap] + +def users(): + '''Returns the users currently logged in. + @return: None: parser error. otherwise: tuple of entries (USERNAME, IP, LOGINSTART, LOGINDURATION, CPUTIME) + ''' + with subprocess.Popen('/usr/bin/w', stdout=subprocess.PIPE) as proc: + data = proc.stdout.read().decode() + lines = data.split('\n')[2:] + rc = [] + for line in lines: + if line == '': + break + #hm pts/0 88.67.239.209 21:17 1:32 m 6:37 0.04 s w + #hm pts/0 88.67.239.209 21:17 60s 0.00s 0.00s w + parts = line.split() + if len(parts) < 4: + rc = None + break + rc.append((parts[0], parts[2], parts[3], parts[4], parts[5] if parts[5].find(':') > 0 else parts[6])) + return rc + +def load(): + '''Returns average loads. + @return: [LOAD_1_MINUTE, LOAD_5_MINUTE, LOAD_10_MINUTE, RUNNING_PROCESSES, PROCESSES] + ''' + with open('/proc/loadavg', 'rb') as fp: + data = fp.read().decode() + matcher = re.match(r'(\S+)\s+(\S+)\s+(\S+)\s+(\d+)/(\d+)', data) + if matcher == None: + rc = None + else: + rc = [float(matcher.group(1)), float(matcher.group(2)), float(matcher.group(3)), int(matcher.group(4)), int(matcher.group(5))] + return rc + +def _getNumber(line): + parts = line.split() + return int(parts[1]) + +def memoryInfo(): + '''Returns the memory usage. + @return: [TOTAL_RAM, AVAILABLE_RAM, TOTAL_SWAP, FREE_SWAP, BUFFERS] + ''' + with open('/proc/meminfo', 'rb') as fp: + lines = fp.read().decode().split('\n') + rc = [_getNumber(lines[0]), _getNumber(lines[2]), _getNumber(lines[14]), _getNumber(lines[15]), _getNumber(lines[3])] + return rc + +def mdadmInfo(filename = '/proc/mdstat'): + '''Returns the info about the software raid systems. + @return: a list of array [name, type, members, blocks, status>, + e.g. [['md0', 'raid1', 'dm-12[0] dm-13[1]', 1234, 'OK'], ['md1', 'raid0', 'sda1[0] sdb1[1]', 1234, 'broken']] + status: 'OK', 'recovery', 'broken' + ''' + rc = [] + if os.path.exists(filename): + with open(filename, 'r') as fp: + # md2 : active raid1 sdc1[0] sdd1[1] + # md1 : active raid1 hda14[0] sda11[2](F) + rexpr1 = re.compile(r'^(\w+) : active (raid\d+) (.*)') + # 1953378368 blocks super 1.2 [2/2] [UU] + rexpr2 = re.compile('^\s+(\d+) blocks.*\[([_U]+)\]') + members = None + for line in fp: + matcher = rexpr1.match(line) + if matcher: + name = matcher.group(1) + aType = matcher.group(2) + members = matcher.group(3) + continue + matcher = rexpr2.match(line) + if matcher: + blocks = matcher.group(1) + status = matcher.group(2) + status2 = 'broken' if status.find('_') >= 0 or members != None and members.find('(F)') > 0 else 'OK' + rc.append([name, aType, members, int(blocks), status2]) + continue + if line.find('recovery') > 0: + rc[len[rc - 1][4]] = 'recovery' + return rc + +if __name__ == '__main__': + infos = diskFree() + for info in infos: + print(base.StringUtils.join(' ', info)) diff --git a/base/Logger.py b/base/Logger.py new file mode 100644 index 0000000..a40f51a --- /dev/null +++ b/base/Logger.py @@ -0,0 +1,49 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +import datetime +import os.path +import base.BaseLogger + +class Logger(base.BaseLogger.BaseLogger): + def __init__(self, logfile, verboseLevel): + '''Constructor. + @param logfile: the file for logging + @param verboseLevel: > 0: logging to stdout too + ''' + base.BaseLogger.BaseLogger.__init__(self, verboseLevel) + self._logfile = logfile + # Test accessability: + try: + with open(self._logfile, 'a'): + pass + except OSError as exc: + msg = '+++ cannot open logfile {}: {}'.format(self._logfile, str(exc)) + print(msg) + self.error(msg) + + def log(self, message, minLevel = 0): + '''Logs a message. + @param message: the message to log + @param minLevel: the logging is done only if _verboseLevel >= minLevel + @return: true: OK false: error on log file writing + ''' + rc = False + try: + if not self._inUse and self._mirrorLogger != None: + self._mirrorLogger.log(message) + now = datetime.datetime.now() + message = now.strftime('%Y.%m.%d %H:%M:%S ') + message + if self._verboseLevel >= minLevel: + print(message) + with open(self._logfile, 'a') as fp: + rc = True + fp.write(message + '\n') + except: + pass + return rc + +if __name__ == '__main__': + pass diff --git a/base/MemoryLogger.py b/base/MemoryLogger.py new file mode 100644 index 0000000..82b2854 --- /dev/null +++ b/base/MemoryLogger.py @@ -0,0 +1,64 @@ +''' +Created on 08.05.2018 + +@author: hm +''' +import base.BaseLogger +import re + +class MemoryLogger(base.BaseLogger.BaseLogger): + '''Implements a logger storing the logging messages in an internal array. + ''' + def __init__(self, verboseLevel = 0): + '''Constructor. + @param verboseLevel: > 0: the messages will be printed (to stdout) + ''' + base.BaseLogger.BaseLogger.__init__(self, verboseLevel) + self._lines = [] + + def contains(self, string): + '''Tests whether the log contains a given string. + @param string: string to search + @return: True: the log contains the string + ''' + rc = False + for line in self._lines: + if line.find(string) >= 0: + rc = True + break + return rc + + def getMessages(self): + '''Returns the internal messages as array. + @return: array of messages + ''' + return self._lines + + def log(self, message, minLevel = 0): + '''Logs a message. + @param message: the message to log + @param minLevel: the logging is done only if _verboseLevel >= minLevel + @return: True: OK + ''' + if self._verboseLevel >= minLevel: + print(message) + self._lines.append(message) + return True + + def matches(self, pattern, flags = 0): + '''Tests whether the log contains a given regular expression. + @param pattern: reg expression to search, e.g. r'\d+' + @param flags: flags of the method re.compile(), e.g. re.I (for ignore case) + @return: True: the log contains the string + ''' + rc = False + regExpr = base.StringUtils.regExprCompile(pattern, 'memory logger pattern', None, flags == 0) + if regExpr != None: + for line in self._lines: + if regExpr.search(line): + rc = True + break + return rc + +if __name__ == '__main__': + pass \ No newline at end of file diff --git a/base/ProcessHelper.py b/base/ProcessHelper.py new file mode 100644 index 0000000..b1775a3 --- /dev/null +++ b/base/ProcessHelper.py @@ -0,0 +1,224 @@ +#! /usr/bin/python3 +''' +processhelper: starting external scripts/programs + +@author: hm +''' +import subprocess +import tempfile +import os +import time +import sys + +sys.path.insert(0, '/usr/share/pyrshell') + +import base.StringUtils + + +class ProcessHelper: + + def __init__(self, verboseLevel, logger): + '''Constructor: + @param logger: display output + ''' + self._logger = logger + self._verboseLevel = verboseLevel + self._output = None + self._rawOutput = None + self._error = None + + def execute(self, argv, logOutput, mode='!shell', timeout=None, currentDirectory=None): + '''Executes an external program. + @param argv: a list of arguments, starting with the program name + @param logOutput: True: the result of stdout is written to stdout via logger. Note: the raw output is available as self._output[] + @param timeout: None or the timeout of the external program + @return: None (logOutput==False) or array of strings + ''' + curDir = self.pushd(currentDirectory) + if argv == None: + self._logger.error('execute(): missing argv (is None)') + elif curDir != '': + self._logger.log('executing: ' + ' '.join(argv), 3) + shell = mode == 'shell' + proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell) + (out, err) = proc.communicate(None, timeout) + self._output = [] + self._error = [] + if logOutput: + for line in out.decode().split('\n'): + line2 = line.rstrip() + if len(line) > 1: + self._output.append(line2) + self._logger.log(line2, 1) + for line in err.decode().split('\n'): + msg = line.rstrip() + if msg != '': + self._error.append(msg) + self._logger.error(msg) + self.popd(curDir) + return None if not logOutput else self._output + + def executeCommunicate(self, process, inputString, logOutput, timeout): + '''Handles the output of subprocess calls. + @param process: the process to inspect + @param inputString: None or an input string + @param logOutput: True: output should be returned + @param timeout: the longest time a process should use + ''' + if inputString == None: + (out, err) = process.communicate(timeout=timeout) + else: + (out, err) = process.communicate(inputString.encode(), timeout) + self._rawOutput = out + if logOutput: + for line in out.decode().split('\n'): + if len(line) > 0: + self._output.append(line) + self._logger.log(line) + for line in err.decode().split('\n'): + if line != '': + self._error.append(line) + self._logger.error(line) + + def executeInput(self, argv, logOutput, inputString=None, mode='!shell', timeout=None): + '''Executes an external program with input from stdin. + @param argv: a list of arguments, starting with the program name + @param logOutput: True: the result of stdout is written to stdout via logger. Note: the raw output is available as self._output[] + @param inputString: None or the input for the program as string + @param timeout: None or the timeout of the external program + ''' + self._output = [] + self._error = [] + if inputString == None: + inputString = '' + self._logger.log('executing: ' + ' '.join(argv), 3) + if mode == 'not used and shell': + fn = tempfile.gettempdir() + '/dbtool.' + str(time.time()) + base.StringUtils.toFile(fn, inputString) + command = argv[0] + " '" + "' '".join(argv[1:]) + "' < " + fn + subprocess.run([command], check=True, shell=True) + os.unlink(fn) + else: + try: + proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=mode == 'shell') + self.executeCommunicate(proc, inputString, logOutput, timeout) + except OSError as exc: + msg = str(exc) + self._logger.error(msg) + self._error = msg.split('\n') + except Exception as exc2: + msg = str(exc2) + self._logger.error(msg) + self._error = msg.split('\n') + + def executeInputOutput(self, argv, inputString=None, logOutput=False, mode='!shell', timeout=None): + '''Executes an external program with input from stdin and returns the output. + @param argv: a list of arguments, starting with the program name + @param inputString: None or the input for the program as string + @param timeout: None or the timeout of the external program + @return: a list of lines (program output to stdout) + ''' + self.executeInput(argv, logOutput, inputString, mode, timeout) + rc = self._output + if (rc == None or len(rc) == 0) and self._rawOutput != None and len(self._rawOutput) > 0: + try: + rc = self._rawOutput.decode('utf-8').split('\n') + except UnicodeDecodeError as exc: + self._logger.error('executeInputOutput(): {}\n[{}]\n"{}"'.format(str(exc), ','.join(argv), '' if inputString == None else inputString[0:80])) + rc = base.StringUtils.minimizeArrayUtfError(self._rawOutput.split(b'\n'), self._logger if self._verboseLevel >= 2 else None) + return rc + + def executeInChain(self, argv1, inputString, argv2, mode='shell', timeout=None): + '''Executes 2 programs with input from stdin as chain and returns the output. + @param argv1: a list of arguments for the first program, starting with the program name + @param inputString: None or the input for the first program as string + @param argv2: a list of arguments for the second program, starting with the program name + @param timeout: None or the timeout of the external program + @return: a list of lines (program output to stdout) + ''' + self._output = [] + self._error = [] + self._logger.log('executing: ' + ' '.join(argv1) + '|' + ' '.join(argv2), 3) + rc = [] + if mode == 'shell': + fnOut = tempfile.gettempdir() + '/dbtool.out.' + str(time.time()) + if inputString == None: + inputPart = '' + else: + fnIn = tempfile.gettempdir() + '/dbtool.in.' + str(time.time()) + inputPart = "< '" + fnIn + "' " + base.StringUtils.toFile(fnIn, inputString) + command = argv1[0] + " '" + "' '".join(argv1[1:]) + "' " + inputPart + "| " + argv2[0] + " '" + "' '".join(argv2[1:]) + "' > " + fnOut + try: + subprocess.run([command], check=True, shell=True) + data = base.StringUtils.fromFile(fnOut) + rc = self._output = data.split('\n') + except Exception as exc: + self._logger.error(str(exc)) + if inputString != None: + os.unlink(fnIn) + os.unlink(fnOut) + else: + try: + p1 = subprocess.Popen(argv1, stdout=subprocess.PIPE) + p2 = subprocess.Popen(argv2, stdin=p1.stdout, stdout=subprocess.PIPE) + # Allow p1 to receive a SIGPIPE if p2 exits. + p1.stdout.close() + self.executeCommunicate(p2, None, True, timeout) + rc = self._output + except Exception as exc: + self._logger.error(str(exc)) + return rc + + def executeScript(self, script, node=None, logOutput=True, args=None, timeout=None): + '''Executes an external program with input from stdin. + @param script: content of the script + @param node: script name without path (optional) + @param logOutput: True: the result of stdout is written to stdout via logger. Note: the raw output is available as self._output[] + @param args: None or an array of additional arguments, e.g. ['-v', '--dump'] + @param timeout: None or the timeout of the external program + @return: None (logOutput==False) or array of strings + ''' + self._logger.log('executing {}...'.format('script' if node == None else node), 3) + if node == None: + node = 'processtool.script' + fn = tempfile.gettempdir() + os.sep + node + str(time.time()) + base.StringUtils.toFile(fn, script) + os.chmod(fn, 0o777) + argv = [fn] + if args != None: + argv += args + rc = self.execute(argv, logOutput, 'shell', timeout) + os.unlink(fn) + return rc + + def popd(self, directory): + '''Changes the current direcory (if needed and possible). + @param directory: None or the new current directory + @return None: directory = None + '': changing directory failed + otherwise: the current directory (before changing) + ''' + if directory != None and directory != '': + os.chdir(directory) + if os.path.realpath(os.curdir) != os.path.realpath(directory): + self._logger.error('cannot change to directory ' + directory) + + def pushd(self, directory): + '''Changes the current direcory (if needed and possible). + @param directory: None or the new current directory + @return None: directory = None + '': changing directory failed + otherwise: the current directory (before changing) + ''' + if directory == None: + rc = None + else: + rc = os.curdir + os.chdir(directory) + if os.path.realpath(os.curdir) != os.path.realpath(directory): + os.chdir(rc) + self._logger.error('cannot change to directory ' + directory) + rc = '' + return rc + diff --git a/base/PythonToTypeScript.py b/base/PythonToTypeScript.py new file mode 100644 index 0000000..613cda7 --- /dev/null +++ b/base/PythonToTypeScript.py @@ -0,0 +1,652 @@ +''' +Created on 09.08.2019 + +@author: hm +''' +import re +import math + +import base.StringUtils + +class PySymbol: + '''Implements the info of a class, class variable, parameter, local variable... + ''' + def __init__(self, name): + self._name = name + self._dataType = None + self._isConst = True + +class Symbols: + '''Implements a symbol table: dictionaries for classes, class variables ... + ''' + def __init__(self, parent): + '''Constructor. + + ''' + self._classes = dict() + self._classVariables = dict() + self._localVariables = dict() + self._parent = parent + self._regClassVariableAssignment = re.compile(r'self\.(\w+)\s*(/?[+-/*]?)=(?!=)\s*(\S+)') + self._regLocalVariableAssignment = re.compile(r'(\w+)\s*(/?[+-/*]?)=(?!=)\s*(\S+)') + self._regReturn = re.compile(r'return\s+(\S+)') + self._regSymbol = re.compile(r'[_a-zA-Z][\w+]*') + self._regNumber = re.compile(r'-?\d+') + self._regBoolean = re.compile(r'\b(True|False)\b') + self._regObject = re.compile(r'[({\[]|(dict|list|[A-Z]\w*)\s*\(') + self._regObjectMethodCall = re.compile('^self\.(\w+)\.\w') + + def declareClassVariables(self): + '''Puts the declaration of the class variables into the output. + ''' + for name in self._classVarNames: + (aType, isConst) = self.getType('class', name, 'string') + self._parent.put(1, 'private ' + name + ': ' + aType + ';') + + def findType(self, symbolType, line, defaultType = None): + '''Tries to detect the type of a variable. + @param symbolType: "local" or "class" + @param line: the line to inspect + @param defaultType: the return value if no type is detected + @return: tuple (name, type): + name: None or the symbol name, type: defaultType or the found data type + ''' + aType = defaultType + name = None + if line.find('rc = 0') >= 0: + name = None + regExpr = self._regLocalVariableAssignment if symbolType == 'local' else self._regClassVariableAssignment + matcher = regExpr.match(line) + if matcher != None: + name = matcher.group(1) + value = matcher.group(3) + rc2 = self.typeOfValue(value) + if rc2 != None: + aType = rc2 + return (name, aType) + + def getType(self, symbolType, name, defaultType = None): + '''Returns the data type of a variable. + @param symbolType: "local" or "class" + @param name: the variable name + @return: tuple (type, isConst): type: None or the data type of the variable isConst: True: the symbol is a constant + ''' + aType = defaultType + isConst = False + variables = self._localVariables if symbolType == 'local' else self._classVariables + if name in variables.keys(): + aType = variables[name]._dataType + if aType == None: + aType = defaultType + isConst = variables[name]._isConst + return (aType, isConst) + + def isFirstAssignment(self, line): + '''Tests whether the line contains an assignment to a not known variable. + @param line: line to inspect + @return: True: the line contains the an assignment to a not known variable + ''' + rc = False + matcher = self._regLocalVariableAssignment.match(line) + if matcher != None: + # group(2) contains the first char of '+=' ... + rc = matcher.group(2) == '' + return rc + + def isVariable(self, symbolType, name): + '''Tests whether a name is a known variable. + @param symbolType: 'local' or 'class' + @param name: the variable to test + ''' + dictionary = self._localVariables if symbolType == 'local' else self._classVariables + return name in dictionary.keys() + + def parseClass(self, ixLine): + '''Finds the properties of the class variables. + @param ixLine: the start index of the method + ''' + converter = self._parent + converter._lockPut = True + lastLine = converter.find('class ', ixLine) + ixFirst = ixLine + 1 + self._classVarNames = [] + while ixLine < lastLine: + (ixLine, line, level) = converter.getLine(ixLine) + (name, aType) = self.findType('class', line) + if name == None: + matcher = self._regObjectMethodCall.match(line) + if matcher != None: + name, aType = matcher.group(1), 'object' + if name != None and name not in self._classVarNames: + self.setType(aType, name, self._classVariables) + self._classVarNames.append(name) + ixLine += 1 + converter._lockPut = False + + def parseMethod(self, ixLine): + '''Finds the properties of the symbols of a method. + @param ixLine: the start index of the method + @return: ixEnd: the last line of the method + ''' + converter = self._parent + converter._lockPut = True + lastLine = converter.find('def ', ixLine) + while ixLine <= lastLine: + (ixLine, line, level) = converter.getLine(ixLine) + matcher = self._regReturn.match(line) + if matcher == None: + (name, aType) = self.findType('local', line) + else: + converter._hasReturn = True + (name, aType) = ('$return', self.typeOfValue(matcher.group(1))) + self.setType(aType, name, self._localVariables) + ixLine += 1 + converter._lockPut = False + return lastLine + + def setType(self, aType, name, variables): + '''Sets the type if defined. + @param aType: None: the method does nothing otherwise: the variable type will be set if it is undefined + @param name: the variable name + @param variables: the dictionary of the variables + ''' + if name != None: + if name not in variables.keys(): + variables[name] = PySymbol(name) + variables[name]._dataType = aType + else: + if variables[name]._dataType == None: + variables[name]._dataType = aType + elif aType != None and variables[name]._dataType != aType: + variables[name]._dataType = 'any' + # second assignment: is not a constant + variables[name]._isConst = False + + def typeOfValue(self, value): + '''Detects the type of a variable given by a assignment. + @param matcher: a re.MatchExpression instance + @return: the detected variable type (attempt) + ''' + aType = None + if self._regBoolean.match(value) != None: + aType = 'boolean' + elif self._regNumber.match(value) != None: + aType = 'number' + elif value.startswith('"') or value.startswith("'"): + aType = 'string' + elif self._regObject.match(value): + aType = 'object' + else: + matcher = self._regSymbol.match(value) + if matcher != None: + name = matcher.group(0) + kind = "class" if name.startswith('self.') else "local" + (aType, isConst) = self.getType(kind, name) + return aType + +class PythonToTypeScript: + def __init__(self, pythonSource, typeScript): + '''Constructor. + @param pythonSource: IN: a TextTool instance which contains the python source + @param typeScript: OUT: the converted type script + ''' + self._hasReturn = False + self._lockPut = False + self._symbols = Symbols(self) + self._python = pythonSource + self._logger = pythonSource._logger + self._verboseLevel = 3 + self._typeScript = typeScript + self._regReservedWords = re.compile(r'\b(True|False|None|self|__init__|print)\b') + self._reservedWords = { 'True' : 'true', 'False' : 'false', 'None' : 'null', 'self' : 'this', + '__init__' : 'constructor', 'print' : 'console.log' } + self._indentItem = '\t' + self._indentWidth = 1 + self._currentMethod = None + self._baseClass = None + self._regEndOfModule = re.compile(r'if\s+__name__\s*==\s*\S__main__\S\s*:\s*$') + regEndOfModule = re.compile(r'if\s+__name__\s*==\s*.__main__.\s*:') + + def appendSemicolon(self): + '''Appends a semicolon in the last output line if meaningful + ''' + if len(self._typeScript._lines) > 0: + lines = self._typeScript._lines + # search the last non comment line: + ix = len(lines) - 1 + while ix > 0 and lines[ix].lstrip().startswith('//'): + ix -= 1 + line = lines[ix] + if line.find('{') < 0 and not line.endswith(';') and not line.endswith('*/'): + self._typeScript._lines[ix] +=';' + + def kindOf(self, line): + '''Detects the object type of the line. + @param line: the stripped line to inspect. + @return: "class", "method" or "statement" + ''' + if type(line) != str: + line = str(line) + if line.startswith('class '): + rc = 'class' + elif line.startswith('def '): + rc = 'method' + else: + rc = 'statement' + return rc + + def convert(self): + '''Converts python code to typescript code + ''' + self._indentItem = self.findIdentItem() + ixLine = 0 + line = None + while not self.endOfContent(ixLine): + if line == None: + (ixLine, line, level) = self.getLine(ixLine) + ix = self.docComment(ixLine) + if ix > ixLine: + (ixLine, line, level) = self.getLine(ix) + self._logger.log('{}: {}'.format(ixLine, line), 3) + (ixLine, line, kind) = self.convertItem(ixLine, line) + base.StringUtils.toFile('/tmp/out.ts', '\n'.join(self._typeScript._lines)) + + def convertClass(self, ixLine, firstLine): + '''Converts a class. + @param ixLine: the index of the first line of the class + @param line: the stripped line to inspect + @return: the tuple (ixLine, line, kind) with the index and content of the next line below the class + ''' + self._currentClass = firstLine.split()[1] + baseClass = '' + self._baseClass = None + ix = firstLine.find('(') + if ix > 0: + self._baseClass = firstLine[ix + 1:firstLine.find(')')].split('.')[-1] + baseClass = ' extends ' + self._baseClass + ixEnd = self._currentClass.find('(') + if ixEnd > 0: + self._currentClass = self._currentClass[0:ixEnd].strip() + else: + # skip ':' + self._currentClass = self._currentClass[0:-1].strip() + kind = 'class' + while not self.endOfContent(ixLine) and kind == 'class': + level = self.getIndentLevel(ixLine) + ixLine += 1 + ixLine = self.docComment(ixLine) + self._symbols.parseClass(ixLine) + self.put(level, 'export class ' + self._currentClass + baseClass + ' {') + self._symbols.declareClassVariables() + self._positionClass = len(self._typeScript._lines) + kind = 'statement' + lastIx = -1 + while not self.endOfContent(ixLine) and kind != 'class': + (ixLine, line, kind) = self.convertMethod(ixLine, level) + if ixLine == lastIx: + break + lastIx = ixLine + self.unputEmpty() + self.put(level, '}') + self.put(0, '') + return (ixLine, line, kind) + + def convertItem(self, ixLine, line, ixEnd = None): + '''Converts a code item (class, method or statement). + @param ixLine: the index of the first line of the class + @param line: the stripped line to inspect + @param ixEnd: None or the last line to process + @return: the tuple (ixLine, line, kind) with the index and content of the next line below the processed line + ''' + ixStart = ixLine + level = self.getIndentLevel(ixLine) + while line.startswith('import'): + parts = line.split()[1].split('.') + if len(parts) == 2 and parts[0] != 'os': + package = parts[0] + module = parts[1] + self.put(level, 'import { $M } from "../$P/$M";'.replace('$M', module).replace('$P', package)) + ixLine += 1 + (ixLine, line, level) = self.getLine(ixLine) + kind = self.kindOf(line) + if kind == 'class': + (ixLine, line, kind) = self.convertClass(ixLine, line) + elif kind == 'method': + (ixLine, line, kind) = self.convertMethod(ixLine, level) + elif kind == 'statement': + (ixLine, line, kind) = self.convertStatement(ixLine, level, ixEnd) + elif line.startswith("'''") or line.startswith('"""'): + ixLine = self.docComment(ixLine) + (ixLine, line, level) = self.getLine(ixLine) + else: + self._python._logger.error('unrecognized input in line {}: {}'.format(ixLine + 1, line) ) + ixLine += 1 + (ixLine, line, level) = self.getLine(ixLine) + kind = self.kindOf(line) + return (ixLine if ixLine > ixStart else ixLine + 1, line, kind) + + def convertMethod(self, ixLine, line): + '''Converts a class. + @param ixLine: the index of the first line of the class + @param level: the level of indention + @return: the tuple (ixLine, line) with the index and content of the next line below the method + ''' + self._hasReturn = False + (ixLine, line) = self.skipEmptyLines(ixLine) + level = self.getIndentLevel(ixLine) + kind = self.kindOf(line) + if kind == 'method': + ixLine = self.docComment(ixLine + 1) + ixEnd = self._symbols.parseMethod(ixLine) + out = line.replace('def ', '') + self._currentMethod = out.split()[0].split('(')[0] + params = line[line.find('(')+1:line.find(')')].split(',') + out = self._currentMethod + '(' + for param in params: + if param == 'self': + continue + param = param.strip() + if not out.endswith('('): + out += ', ' + (name, aType) = self._symbols.findType("local", param) + if aType == None: + aType = 'string' + if name == None: + out += param + ': string' + else: + out += name + ': ' + aType + param[len(name):] + out += ')' + if self._hasReturn: + (aType, isConst) = self._symbols.getType('local', '$return', 'string') + out += ': ' + aType + out += ' {' + self.put(level, out[0:-1] + ' {') + kind = 'statement' + (ixLine, line, level) = self.getLine(ixLine) + while not self.endOfContent(ixLine) and kind == 'statement' and (ixEnd == None or ixLine <= ixEnd): + (ixLine, line, kind) = self.convertItem(ixLine, line, ixEnd) + self.appendSemicolon() + self.put(level - 1, '}') + self.put(0, '') + return (ixLine, line, kind) + + def convertStatement(self, ixLine, line, ixEnd = None): + '''Converts a statement. + @param ixLine: the index of the first line of the class + @param level: the level of indention + @param ixEnd: None or the last line to process + @return: the tuple (ixLine, line) with the index and content of the next line below the method + ''' + (ixLine, line) = self.skipComments(ixLine) + openBlocks = [] + while not self.endOfContent(ixLine) and (ixEnd == None or ixLine <= ixEnd): + (ixLine, line) = self.skipEmptyLines(ixLine) + (ixLine, line) = self.skipComments(ixLine) + if self._currentMethod == '__init__': + if self._baseClass != None: + ix = line.find(self._baseClass) + ix += 1 + if ix > 0: + args = line.split('(self,')[1].strip()[0:-1] + self.put(self.getIndentLevel(ixLine), 'super({});'.format(args)) + (ixLine, line, currentLevel) = self.getLine(ixLine + 1) + (name, aType) = self._symbols.findType('class', line, 'string') + self._logger.log('{}: {}'.format(ixLine, line), 3) + isFirstAssignment = self._symbols.isFirstAssignment(line) + (name, aType) = self._symbols.findType('local', line, 'string') + (aType, isConst) = self._symbols.getType('local', name) + if name == None: + line = None + elif isFirstAssignment: + line = ('let ' if not isConst else 'const ') + line + kind = self.kindOf(line) + if kind != 'statement': + break + else: + if line == None: + (ixLine, line) = self.skipComments(ixLine) + kind = self.kindOf(line) + if kind != 'statement': + break + currentLevel = self.getIndentLevel(ixLine) + #ignoreIf = toPush != None and toPush[1] == '// else' + ignoreIf = line.startswith('else:') or line.startswith('elif ') + while len(openBlocks) > 0 and openBlocks[-1][0] >= currentLevel : + (level2, body) = openBlocks.pop() + if not (ignoreIf and level2 == currentLevel and body.startswith('// if')): + self.put(level2, '} ' + body) + toPush = self.putStatement(currentLevel, line) + self.appendSemicolon() + if toPush != None: + openBlocks.append(toPush) + # if level2 < currentLevel and not (line.startswith('else:') or line.startswith('else ')): + (ixLine, line, currentLevel) = self.getLine(ixLine + 1) + if len(openBlocks) > 0: + (level2, body) = openBlocks.pop() + self.put(level2, '}' + body) + return (ixLine, line, self.kindOf(line)) + + def docComment(self, ixLine): + '''Puts a document comment from Python to JScript + @param ixLine: the index in _python where the comment starts. + @pre: the first non blank symbol is """ or 3 ticks (') + @return: the index of the next line below the doc comment + ''' + self._hasReturn = False + line = self._python._lines[ixLine].strip() + marker = line[0:3] + if marker == '"""' or marker == "'''": + level = self.getIndentLevel(ixLine) + commentIndent = 0 if level < 1 else level - 1 + self.put(commentIndent, '/**') + self.put(commentIndent, ' * ' + line[3:]) + ixLine += 1 + while not self.endOfContent(ixLine): + line = self._python._lines[ixLine].strip() + ixLine += 1 + if line.endswith(marker): + self.put(commentIndent, ' * ' + line[0:-3]) + break + else: + self.put(commentIndent, ' * ' + line) + self.unputEmpty() + self.put(commentIndent, ' */') + return ixLine + + def endOfContent(self, ixLine): + '''Returns whether the source is completely processed. + @return True: the content is completely processed + ''' + if ixLine == None or type(ixLine) != int: + rc = True + else: + rc = ixLine >= len(self._python._lines) + return rc + + def find(self, token, ixLine): + '''Searches the given token starting with a given index. + @param token: string to detect + @param ixLine: the start index + @return: the line index above the line the token or (if not found) last line of the source + ''' + rc = None + while rc == None and not self.endOfContent(ixLine): + (ixLine, line, level) = self.getLine(ixLine) + if line.startswith(token) or self._regEndOfModule.match(line): + rc = ixLine - 1 + ixLine += 1 + if rc == None: + rc = len(self._python._lines) + return rc + + def findIdentItem(self): + '''Inspects some lines of the input to detect the width of one indention step. + @return: the string representing one level of indent. + ''' + regIndent = re.compile('^(\s+)(\w+)') + ixLine = 0 + indents =dict() + lengths = [0, 0, 0, 0, 0, 0, 0, 0, 0] + countRelevantLines = 0 + linesWithBlanks = 0 + self._indentWidth = 1 + countIndenters = 0 + while ixLine < len(self._python._lines): + line = self._python._lines[ixLine] + if line.startswith('def '): + countIndenters += 1 + matcher = regIndent.match(line) + ixLine += 1 + if matcher != None and matcher.group(1) != '': + if matcher.group(2) in ['def']: + countIndenters += 1 + if countIndenters or matcher.group(2) in ['if', 'else', 'for', 'while', 'return']: + countRelevantLines += 1 + indent = matcher.group(1) + level = len(indent) + if level > 0 and level < 9: + lengths[level] += 1 + if indent[0] == ' ': + linesWithBlanks += 1 + if level in indents: + indents[level] += 1 + else: + indents[level] = 1 + if countRelevantLines > 50: + break + factors = [0, 0, 0, 0, 0, 0, 0, 0, 0] + for ix in indents.keys(): + for factor in range(2, len(factors)): + if ix % factor == 0: + factors[factor] += 1 + for factor in range(1, len(factors)): + # a factor can exist but no matching length: factor=2 and indent=' '*4 + # lengths[factor] can be incorrect, we assume at least 50% correct: + if self._indentWidth < factors[factor] and lengths[factor] >= countIndenters: + self._indentWidth = factor + item = ' ' if linesWithBlanks >= countRelevantLines - linesWithBlanks else '\t' + rc = item * self._indentWidth if self._indentWidth > 1 else '\t' + self._logger.log('indention width: {} item: {}'.format(self._indentWidth, '\\t' if item == '\t' else '" "')) + return rc + + def getLine(self, ixLine): + '''Gets the info of the line at a given index. + @param ixLine: the index of the line + @return: the tuple (ixLine, line, level). Because of comments the ixLine may be incremented + ''' + changed = True + ix = ixLine + while changed: + (ix, line) = self.skipComments(ix) + (ix, line) = self.skipEmptyLines(ix) + changed = ix != ixLine + ixLine = ix + level = self.getIndentLevel(ixLine) + return (ixLine, line, level) + + def getIndentLevel(self, ixLine): + '''Returns the whitespaces of the python source with index ixLine. + @param ixLine: the index of the source line + @return: the indention level + ''' + rc = 0 + if not self.endOfContent(ixLine): + ix = 0 + line = self._python._lines[ixLine] + while ix < len(line) and line[ix].isspace(): + ix += 1 + if ix > 0: + if line[0] != '\t': + rc = ix // self._indentWidth + return rc + + def put(self, level, line, position = None): + '''Puts a given line into the typescript lines. + @param level: the indention level + @param line: the line to put + ''' + def subst(matcher): + rc = self._reservedWords[matcher.group(1)] + return '?' if rc == None else rc + if not self._lockPut: + if line.startswith('}'): + line += '' + if type(level) != int: + level += 1 + if level > 0: + line = self._indentItem * level + line + if position == None: + self._typeScript._lines.append(self._regReservedWords.sub(subst, line)) + else: + self._typeScript._lines.insert(position, line) + + def putStatement(self, level, line): + '''Completes a statement, e.g. inserts '(' for if, while..., and put it to the output + @param level: the indention level + @param line: line to inspect + @return: None or tuple (level, comment) + ''' + out = line + toPush = None + if out.endswith(':'): + out = out[0:-1] + ') {' + if out.startswith('if '): + toPush = (level, '// if') + out = out.replace('if ', 'if (') + elif out.startswith('else) {'): + toPush = (level, '// else') + out = '} else {' + elif out.startswith('elif '): + toPush = (level, '// elif') + out = out.replace('elif ', 'else if (') + elif out.startswith('while '): + toPush = (level, '// while') + out = out.replace('while ', 'while (') + elif out.startswith('for '): + toPush = (level, '// for') + name = line.split()[1] + if self._symbols.isVariable('local', name): + out = out.replace('for ', 'for (') + else: + out = out.replace('for ', 'for (let ') + if out != '': + self.put(level, out) + return toPush + + def unputEmpty(self): + '''if the last line of the output is empty it will be removed. + ''' + if len(self._typeScript._lines) > 0 and (self._typeScript._lines[-1].strip() == '' or self._typeScript._lines[-1].strip() == '*'): + self._typeScript._lines.pop() + + def skipEmptyLines(self, ixLine): + '''Skippes empty lines and put them into _typeScript + @param ixLine: the index in _python to start + @return: a tuple(ixLine, line): the index and the stripped content of the next not empty line + ''' + line = '' + while not self.endOfContent(ixLine): + line = self._python._lines[ixLine].strip() + if line == '': + ixLine += 1 + else: + break + return (ixLine, line) + + def skipComments(self, ixLine): + '''Skippes empty lines and put them into _typeScript + @param ixLine: the index in _python where the comment starts. + @return: a tuple(ixLine, line): the index and the stripped content of the line below the comment + ''' + line = '' + while not self.endOfContent(ixLine): + line = self._python._lines[ixLine].strip() + if line.startswith('#'): + self.put(self.getIndentLevel(ixLine), '//' + line[1:]) + ixLine += 1 + else: + break + return (ixLine, line) + +if __name__ == '__main__': + pass \ No newline at end of file diff --git a/base/StringUtils.py b/base/StringUtils.py new file mode 100644 index 0000000..db8a60d --- /dev/null +++ b/base/StringUtils.py @@ -0,0 +1,436 @@ +''' +Created on 19.04.2018 + +@author: hm +''' +import re +import os +import datetime + +# .................................1.....1....2.....2....3.....3 +stringUtilRexprDate = re.compile(r'^(\d{4})[.-](\d\d?)[.-](\d\d?)') +# ...................................1.....1.2....2 a..3.....3a +stringUtilRexprTime = re.compile(r'^(\d\d?):(\d\d?)(?::(\d\d?))?$') +stringUtilRexprInt = re.compile(r'^0[xX]([0-9a-fA-F]+)|0([0-7]+)|(\d+)$') + +def arrayContains(lines, regExpr): + '''Tests whether at least one line of the array lines contains a given regular expression. + @param lines: array of text lines + @param regExpr: a string or a regexpr object + @return: True: at least one item of lines contains the regular expression regExpr + ''' + if type(regExpr) == str: + regExpr = re.compile(regExpr) + found = False + for line in lines: + if regExpr.search(line) != None: + found = True + break + return found + +def asInt(value, defaultValue = None): + '''Tests whether a value is an integer. If not the defaultValue is returned. Othewise the integer is returned. + @param value: string value to test + @return: defaultValue: the value is not an integer Otherwise: the value as integer + ''' + matcher = stringUtilRexprInt.match(value) + if matcher == None: + rc = defaultValue + else: + if value.startswith('0x') or value.startswith('0X'): + rc = int(matcher.group(1), 16) + elif value.startswith('0'): + rc = int(value, 8) + else: + rc = int(value) + return rc + +def escChars(text): + '''Return the text with escaped meta characters like \n, \t, \\. + @param text: text to convert + @return: the text with escaped chars. + ''' + text = text.replace('\\', '\\\\') + text = text.replace('\t', '\\t') + text = text.replace('\n', '\\n') + return text + +def firstMatch(aList, regExpr, start=0): + '''Return the matching object of the first matching line of a given line list. + @param aList: an array of lines to inspect + @param regExpr: a compiled regular expression, e.g. re.compile(r'^\w+ =\s(.*)$') + @param start: the first line index to start searching + @return: None: nothing found + otherwise: the matching object of the hit + ''' + matcher = None + while start < len(aList): + matcher = regExpr.search(aList[start]) + if matcher != None: + break + start += 1 + return matcher + +def formatSize(size): + '''Formats the filesize with minimal length. + @param size: size in bytes + @return: a string with a number and a unit, e.g. '103 kByte' + ''' + if size < 1000: + rc = str(size) + ' Byte' + else: + if size < 1000000: + unit = 'KB' + size /= 1000.0 + elif size < 1000000000: + unit = 'MB' + size /= 1000000.0 + elif size < 1000000000000: + unit = 'GB' + size /= 1000000000.0 + else: + unit = 'TB' + size /= 1000000000000.0 + rc = '{:.3f} {:s}'.format(size, unit) + return rc + +def fromFile(filename, sep = None): + '''Reads the content of a file. + @param filename: the name of the file to read + @param sep: None or the split separator + @param content: the content of the file. If sep == None: a string. Otherwise an array + ''' + rc = '' + if os.path.exists(filename): + with open(filename, 'r') as fp: + rc = fp.read() + if sep != None: + rc = rc.split(sep) + return rc + +def grepInFile(filename, regExpr, limit = None, group = None): + '''Returns all lines of a given file matching a given regular expression. + @param filename: the name of the file to inspect + @param regExpr: a compiled regular expression, e.g. re.compile(r'^\w+ =') + @param limit: the maximal count of returned lines + @param group: None or: the content of the group (defined by the group-th parenthesis) will be returned + @return: a list of found lines or groups (see group), may be empty + ''' + rc = [] + if type(regExpr) == str: + regExpr = re.compile(regExpr) + if os.path.exists(filename): + with open(filename, 'r') as fp: + for line in fp: + line = line.strip() + matcher = regExpr.search(line) + if matcher != None: + if group != None: + rc.append(matcher.group(group)) + else: + rc.append(line) + if limit != None: + limit -= 1 + if limit <= 0: + break + return rc + +def hasContent(filename, beginOfComment = '#'): + '''Tests whether a file has a content without empty lines or comment lines. + @param beginOfComment this string starts a comment line + @return: True: there are lines which are not empty and not comments. + ''' + rc = False + if os.path.exists(filename): + with open(filename, 'r') as fp: + for line in fp: + line = line.strip() + if line != '' and not line.startswith(beginOfComment): + rc = True + break + return rc + +def join(separator, args): + '''Joins all entries of a list into a string. + @param separator: the separator between the list items + @param args: list to join. Items may be not strings + @return: a string with all items of args separated by separator + ''' + rc = '' + if args != None: + for item in args: + if rc != '': + rc += separator + rc += str(item) + return rc + +def limitItemLength(array, maxLength, elipsis = '...'): + '''Copies the input array and limits each item to the given maximum. + @param array: source array + @param maxLength: the maximal length of each item of the result + @param suffix: the suffix for limited items, e.g. '...' + @return: the copy of the array with limited items + ''' + rc = [] + lenElipsis = len(elipsis) + for item in array: + if len(item) > maxLength: + if maxLength >= lenElipsis: + item = item[0:maxLength-lenElipsis] + elipsis + else: + item = item[0:maxLength] + rc.append(item) + return rc + +def minimizeArrayUtfError(lines, logger = None): + '''Converts a string array of bytes into an array of UTF-8 strings. + It minimizes the part which can not be converted. + @param lines: a list of byte lines + @param logger: None or the error logger + @param logError: True: conversion errors will be logged + ''' + rc = [] + for line in lines: + try: + rc.append(line.decode('utf-8')) + except UnicodeDecodeError: + rc.append(minimizeStringUtfError(line, logger)) + return rc + +def minimizeStringUtfError(line, logger = None): + '''Converts a string of bytes into an UTF-8 string. + It minimizes the part which can not be converted. + @param lines: a list of byte lines + @param logger: None or the error logger + ''' + rc = '' + def convert(part): + try: + rc = part.decode('utf-8') + except UnicodeDecodeError as exc: + if logger != None: + logger.error('cannot decode: ' + part.decode('ascii', 'ignore')[0:80]) + rc = None + if len(line) < 10: + part = convert(line) + if part != None: + rc += part + else: + try: + rc = line.decode('latin-1') + except: + rc = line.decode('ascii') + else: + half = int(len(line) / 2) + part = convert(line[0:half]) + if part != None: + rc += part + else: + rc += minimizeStringUtfError(line[0:half], logger) + part = convert(line[half:]) + if part != None: + rc += part + else: + rc += minimizeStringUtfError(line[half:], logger) + return rc + +def regExprCompile(pattern, location, logger = None, isCaseSensitive = False): + '''Compiles a regular expression. + @param pattern: a regular expression. + @param logger: for error logging + @param isCaseSensitive: true: the case is relevant + @return: None: error occurred Otherwise: the re.RegExpr instance + ''' + rc = None + try: + rc = re.compile(pattern, 0 if isCaseSensitive else re.I) + except Exception as exc: + msg = 'error in regular expression in {}: {}'.format(location, str(exc)) + if logger == None: + print('+++ ' + msg) + else: + logger.error(msg) + return rc + +def toFile(filename, content, separator=''): + '''Writes a string into a file. + @param filename: the name of the file to write + @param content: the string to write + ''' + if type(content) == list: + content = separator.join(content) + mode = 'wb' if type(content) == bytes else 'w' + with open(filename, mode) as fp: + fp.write(content) + +def toFloat(value): + '''Converts a string into a float. + Possible data types: int, date, datetime, float. + Value of date/datetime: seconds since 1.1.1970 + Value of time: seconds since midnight + @param value: the string to convert + @return [float, dataType] or [error_message, dataType] + ''' + if type(value) == float: + rc = value + else: + if type(value) != str: + value = str(value) + matcher = stringUtilRexprDate.match(value) + if matcher != None: + length = len(matcher.group(0)) + value = value[length+1:] + rc = datetime.datetime(int(matcher.group(1)), int(matcher.group(2)), int(matcher.group(3))).timestamp() + matcher = stringUtilRexprTime.match(value) + if matcher != None: + hours, mins = int(matcher.group(1)), int(matcher.group(2)) + secs = (hours * 60 + mins)*60 + rc += secs + if matcher.group(3): + rc += int(matcher.group(3)) + else: + matcher = stringUtilRexprTime.match(value) + if matcher != None: + hours, mins = int(matcher.group(1)), int(matcher.group(2)) + rc = (hours * 60 + mins)*60 + if matcher.group(3): + rc += int(matcher.group(3)) + else: + matcher = stringUtilRexprInt.match(value) + if matcher != None: + if matcher.group(3): + rc = float(matcher.group(3)) + elif matcher.group(1): + rc = float(int(value[2:], 16)) + elif matcher.group(2): + rc = float(int(value, 8)) + else: + try: + rc = float(value) + except ValueError: + rc = 'float (or int or date(time)) expected, found: ' + value + return rc + +def toFloatAndType(value): + '''Converts a string into a float. + Possible data types: int, date, datetime, float. + Value of date/datetime: seconds since 1.1.1970 + Value of time: seconds since midnight + @param value: the string to convert + @return [float, dataType] or [error_message, dataType] + ''' + dataType = 'undef' + if type(value) == float: + dataType = 'float' + rc = value + else: + matcher = stringUtilRexprDate.match(value) + if matcher != None: + dataType = 'date' + length = len(matcher.group(0)) + value = value[length+1:] + rc = datetime.datetime(int(matcher.group(1)), int(matcher.group(2)), int(matcher.group(3))).timestamp() + matcher = stringUtilRexprTime.match(value) + if matcher != None: + dataType += 'time' + hours, mins = int(matcher.group(1)), int(matcher.group(2)) + secs = (hours * 60 + mins)*60 + rc += secs + if matcher.group(3): + rc += int(matcher.group(3)) + else: + matcher = stringUtilRexprTime.match(value) + if matcher != None: + hours, mins = int(matcher.group(1)), int(matcher.group(2)) + dataType = 'time' + rc = (hours * 60 + mins)*60 + if matcher.group(3): + rc += int(matcher.group(3)) + else: + matcher = stringUtilRexprInt.match(value) + if matcher != None: + dataType = 'int' + if matcher.group(3): + rc = float(matcher.group(3)) + elif matcher.group(1): + rc = float(int(value[2:], 16)) + elif matcher.group(2): + rc = float(int(value, 8)) + else: + try: + rc = float(value) + dataType = 'float' + except ValueError: + rc = 'float (or int or date(time)) expected, found: ' + value + return [rc, dataType] + +def toString(value, dataType, floatPrecision = None): + '''Converts a numeric value into a string. + @param value: a numeric value + @param dataType: 'date', 'datetime', 'time', 'float', 'int' + @param floatPrecision: None or if the type is a float, the number of digits behind the point + @return: the value as string + ''' + if dataType == 'date': + date = datetime.datetime.fromtimestamp(value) + rc = date.strftime('%Y.%m.%d') + elif dataType == 'datetime': + if type(value) == str and value.find(':') >= 0: + rc = value + else: + date = datetime.datetime.fromtimestamp(value) + rc = date.strftime('%Y.%m.%d %H:%M') + elif dataType == 'time': + if type(value) == 'str' and value.find(':') >= 0: + rc = value + else: + rc = '{:2d}:{:2d}'.format(value / 3600, value % 3600 / 60) + elif floatPrecision != None: + if type(value) == str: + value = float(value) + aFormat = '{' + ':.{}f'.format(floatPrecision) + '}' + rc = aFormat.format(value) + else: + rc = '{}'.format(value) + return rc + +def tailOfWord(words, wordPrefix): + '''Returns the part of a word behind the word prefix. + Example: words: "-e! -m" wordPrefix: "-e" result: "!" + @param words: a string with words separated by space or tab + @param wordPrefix: the word starting with this prefix will be searched + @return: None: word prefix not found + the word suffix + ''' + rc = None + if words.startswith(wordPrefix): + ixStart = 0 + else: + ixStart = words.find(wordPrefix) + if ixStart > 0 and not words[ixStart-1].isspace(): + ixStart = words.find(' ' + wordPrefix) + if ixStart < 0: + ixStart = words.find('\t' + wordPrefix) + if ixStart >= 0: + ixStart += len(wordPrefix) + ixEnd = words.find(' ', ixStart) + ixEnd2 = words.find('\t', ixStart) + if ixEnd < 0 or ixEnd2 > 0 and ixEnd2 < ixEnd: + ixEnd = ixEnd2 + if ixEnd < 0: + ixEnd = len(words) + rc = words[ixStart:ixEnd] + return rc + +def unescChars(text): + '''Returns the text without escaped meta characters like \n, \t, \\. + @param text: text to convert + @return: the text with unescaped chars + ''' + text = text.replace('\\n', '\n') + text = text.replace('\\t', '\t') + text = text.replace('\\\\', '\\') + return text + +if __name__ == '__main__': + pass \ No newline at end of file diff --git a/base/TextProcessor.py b/base/TextProcessor.py new file mode 100644 index 0000000..12cca84 --- /dev/null +++ b/base/TextProcessor.py @@ -0,0 +1,1776 @@ +''' +Created on 08.06.2018 + +@author: hm +''' +import re +import os.path +import sys +import base.MemoryLogger + +IGNORE=re.RegexFlag.IGNORECASE +MARKER_MANUAL = '°manual°' +MARKER_EXAMPLE = '°example°' + +class Pos: + def __init__(self, line, col): + '''Constructor. + @param line: the line index + @param col: the column index + ''' + self._line = line + self._col = col + + def adaptDeleted(self, start, end): + '''Adapts the position if a part of a given line is deleted. + @param start: start position + @param end: end position + ''' + if self._line > start._line or self._line == start._line and self._col > start._col: + if self._line > end._line: + self._line -= end._line - start._line + elif self._line == end._line: + self._line = start._line + if self._col > end._col: + self._col = start._col + else: + self._col = start._col + self._col - end._col + else: + self._col = start._col + self._line = start._line + + def adaptInserted(self, start, lines): + '''Adapts the position if a part of a given line is deleted. + @param start: start position + @param lines: number of inserted lines + ''' + if self._line > start._line: + self._line += lines + + def clone(self, source): + '''Makes a deep copy of a given source. + @param source: the source to copy + ''' + self._line = source._line + self._col = source._col + + def less(self, position): + '''Tests whether a given position is less (above) the instance. + @param position: the position to compare. Type: Pos + @return: position < self + ''' + return position._line < self._line or ( + position._line == self._line and position._col < self._col) + +class Method: + def __init__(self, name, argNames, argTypes, defaultValues, resultType): + ''' + @param name: the method's name + @param argNames: a list of argument names e.g. ['length', 'start'] + @param argTypes: a list of argument types, e.g. ['int', 'text'] + @param defaultValues: a list of argument default values, e.g. [3, 'nice'] + @param resultType: the data type of the method's result, e.g. 'int' + @param parent: an instance of Variable + ''' + self._name = name + self._argTypes = argTypes + self._resultType = resultType + self._argNames = argNames + self._defaultValues = defaultValues + + def checkArgs(self, args): + '''Checks and calculates the argument vector of the method. + @param args: the argument vector: each item contains int, string or Variable + @return: None: ok + otherwise: the error message + ''' + rc = None + if len(args) > len(self._argNames): + rc = 'too many arguments: {:d} / {:d}'.format(len(args), len(self._argNames)) + else: + for ix in range(len(args)): + arg = args[ix] + expectedType = self._argTypes[ix] + if isinstance(arg, Variable): + if expectedType.startswith('var.'): + expectedType2 = expectedType[4:] + if arg._type != expectedType2: + rc = 'not an expected variable type {:s} for argument {:d} ({:s}): {:s}'.format(expectedType2, ix+1, self._argNames[ix], arg._name) + elif arg._type != expectedType: + rc = 'not an expected type {:s} for argument {:d} ({:s}): {:s}'.format(expectedType, ix+1, self._argNames[ix], arg._name) + elif expectedType == 'int' or expectedType == 'text': + args[ix] = arg._value + elif expectedType == 'int': + if type(arg) != int: + try: + args[ix] = int(arg) + except ValueError: + rc = 'not an int value for argument {:d} ({:s}): {:s}'.format(ix+1, self._argNames[ix], arg) + if rc: + break + for ix in range(len(args), len(self._defaultValues)): + args.append(self._defaultValues[ix]) + return rc + +class Variable: + '''Base class of variables. + ''' + def __init__(self, name, varType): + '''Constructor. + @param name: the variable's name + @param varType: the variable's type:. 'text', 'dict' or 'list' + ''' + self._name = name + self._type = varType + def getMethod(self, name): + '''Gets the Method instance given by name. + @param name: the methods name + @return None: not found + otherwise: the Method instance + ''' + return None + +class DictVariable(Variable): + '''Implements a variable containing a dictionary: a list of (string, string) tuples. + ''' + def __init__(self, name): + '''Constructor. + @param name: the variable's name + ''' + Variable.__init__(self, name, 'dict') + self._dict = dict() + self._addMethod('keys', ['listVar'], ['dict'], None, 'list') + + def call(self, name, argv, textProcessor): + '''Calls a variable specific method. + @param name: name of the method to call + @param argv: argument vector + @param textProcessor: delivers the environment + @return: the return value of the method + ''' + rc = None + if name == 'keys': + rc = argv[0].join(self._list) + return rc + + def getMethod(self, name): + '''Gets the Method instance given by name. + @param name: the methods name + @return None: not found + otherwise: the Method instance + ''' + return None if name not in DictVariable._methods else DictVariable._methods[name] + + _methods = dict() + @staticmethod + def _addMethod(name, argNames, argTypes, defaultValues, resultType): + DictVariable._methods[name] = Method(name, argNames, argTypes, defaultValues, resultType) +DictVariable._addMethod('keys', ['listVar'], ['dict'], [None], None) + +class IntegerVariable(Variable): + '''Implements a variable containing an integer. + ''' + def __init__(self, name): + '''Constructor. + @param name: the variable's name + ''' + Variable.__init__(self, name, 'int') + self._value = 0 + +class ListVariable(Variable): + '''Implements a variable containing a list of strings. + ''' + def __init__(self, name): + '''Constructor. + @param name: the variable's name + ''' + Variable.__init__(self, name, 'list') + self._list = [] + + def call(self, name, argv, textProcessor): + '''Calls a variable specific method. + @param name: name of the method to call + @param argv: argument vector + @param textProcessor: delivers the environment + @return: the return value of the method + ''' + rc = None + if name == 'size': + rc = len(self._list) + elif name == 'join': + rc = argv[0].join(self._list) + return rc + + def getMethod(self, name): + '''Gets the Method instance given by name. + @param name: the methods name + @return None: not found + otherwise: the Method instance + ''' + return None if name not in ListVariable._methods else ListVariable._methods[name] + + _methods = dict() + @staticmethod + def _addMethod(name, argNames, argTypes, defaultValues, resultType): + ListVariable._methods[name] = Method(name, argNames, argTypes, defaultValues, resultType) +ListVariable._addMethod('size', [], [], [], 'int') +ListVariable._addMethod('join', ['glue'], ['text'], ['\n'], None) + +class TextVariable(Variable): + '''Implements a variable containing a simple string. + ''' + def __init__(self, name, value = ''): + '''Constructor. + @param name: the variable's name + @param value: the initial value of the variable + ''' + Variable.__init__(self, name, 'text') + self._value = value + + def call(self, name, argv, textProcessor): + '''Calls a variable specific method. + @param name: name of the method to call + @param argv: argument vector + @param textProcessor: delivers the environment + @return: the return value of the method + ''' + rc = None + if name == 'length': + rc = len(self._value) + elif name == 'split': + separator = argv[1] + if len(separator) > 3 and separator.startswith('/') and separator.endswith('/'): + argv[0]._list = re.split(separator[1:-1], self._value, argv[2]) + else: + argv[0]._list = self._value.split(separator, argv[2]) + return rc + + def getMethod(self, name): + '''Gets the Method instance given by name. + @param name: the methods name + @return None: not found + otherwise: the Method instance + ''' + return None if name not in TextVariable._methods else TextVariable._methods[name] + + _methods = dict() + @staticmethod + def _addMethod(name, argNames, argTypes, defaultValues, resultType): + TextVariable._methods[name] = Method(name, argNames, argTypes, defaultValues, resultType) +TextVariable._addMethod('length', [], [], [], 'int') +TextVariable._addMethod('split', ['listVar', 'separator', 'maxSplit'], ['list', 'text', 'int'], [None, r'\s+', 0x7fffffff], None) + +class TextProcessor: + '''Engine to manipulate text data. + <current position>: + The processor has exactly one current position. It can be changed by the statements + 'goto' and 'find'. The current position can be notified by '#'. + <marks>: + The processor knows 10 "marks", notified by '#0' .. '#9'. + A mark can be set by the "mark" command. It copies the current position into the + specified mark. + <pattern>: + A pattern is notified by a delimiter (a non blank and non alfanumeric character), + a regular expression and the same delimiter, e.g. /\s(\w+)/ or !([a-f0-9])?! + <_range>: + The "_range" command defines a _range with a start and an end position. + If a _range exists all statements are executed only inside the _range. + Example: the hit of a replacement command is outside the _range: + the replacement will not be done. + If a "goto" or "find" command defines a current position outside the _range + the command sequence will be stopped. + <statements>: + !<var>[<index>] = { <int> | string } + d(elete) <length>|<mark> + else + f(ind) <pattern> + f(ind)b(ackwards) + fi + g(oto) <line> [<col>] | <mark> + i(nsert) <string> + if + l(oad) + m(ark) <mark> + n(umericassign) + p(rint) + r(ange) [ <markStart> [ <markEnd> ]] + read + s(ubstitute) /<pattern>/<replacement>/flags + v(ariable) + w(rite) + ''' + def __init__(self, filename=None, content=None, logger = None): + '''Constructor. + @param filename: None or the file with the lines to process + @param content: string: content to process, will be splitted into a list of lines + array: list of lines + ''' + self._logger = logger if logger else base.MemoryLogger.MemoryLogger() + self._name = None + self._errors = 0 + self._lines = None + self._lineNo = None + self._changed = False + self._ignoreMacros = False + self._scriptName = None + self._reFlags = 0 + self._currentPos = Pos(0, 0) + self._startRange = Pos(0, 0) + self._vars = dict() + self._reExpr = dict() + self._endRange = Pos(0x7fffffff, 0x7fffffff) + # Note: index 10: beginOfFile index 11: endOfFile + self._marks = [Pos(-1, -1), Pos(-1, -1), Pos(-1, -1), Pos(-1, -1), Pos(-1, -1), + Pos(-1, -1), Pos(-1, -1), Pos(-1, -1), Pos(-1, -1), Pos(-1, -1), + Pos(0, 0), Pos(0, 0)] + if content != None: + if type(content) == list: + self._lines = content + elif type(content) == str: + self._lines = content.split('\n') + else: + self._logger.error('unexpected content type: ' + str(type(content))) + elif filename != None: + self._readFile(filename) + self._clearProgram() + + def _adaptDeleted(self, start, end): + '''Adapts the internal positions changed by a deletion. + @param start: start position + @param end: end position + ''' + self._startRange.adaptDeleted(start, end) + self._endRange.adaptDeleted(start, end) + for ix in range(len(self._marks)): + self._marks[ix].adaptDeleted(start, end) + + def _adaptInserted(self, start, lines): + '''Adapts the internal positions changed by an insertion. + @param start: start position + @param lines: a list of inserted lines + ''' + aCount = len(lines) + self._startRange.adaptInserted(start, aCount) + self._endRange.adaptInserted(start, aCount) + for ix in range(len(self._marks)): + self._marks[ix].adaptInserted(start, aCount) + + def _assign(self, statements, doExecute=True): + '''Handles the "assign" command. + '$' <var>[ '[' <int>|<key> ']' = { <int> | <delim><string><delim> } <opts> + @param statements: a string starting with a "assignment" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "assigment" command + ''' + if statements == MARKER_MANUAL: + return '''<var> = { <int> | <delim><string><delim> | <var_expression>} <opts> +<var>: + '!'<name> or '!<name>'[' {<integer> | <string>}']' +<var_expression> + { '!'<name> | '!<name>'.'<method>'('<args>')' } +<args>: + { '' | <arg> { ',' <arg> }* } +<arg>: + { <number> | <delim><string><delim> | '!'<variable> } +<opt>: + -p<char> or --var-prefix=<char> + Variables in <string> will be expanded + -e<char> or --meta-esc=<char> + String meta characters will be expanded: if <char> == '\': \t (tabulator) \n \r \\x<hexdigit><hexdigit> +''' + elif statements == MARKER_EXAMPLE: + return '''!number = 33 +!title = 'The Bible' +!color["red"] = /f00/ +!names[3] = $name +!argv=!arguments.split(";") +!text="%{prefix}title: %color[/red/]-%items[%ix]" -p% +''' + elif not doExecute: + #.........................1...1..2...2..3......3 + matcher = self._match(r'\$(\w+)\.(\w+)\(([^()]*)\)[\s;]*', + 'call', statements, False) + if matcher: + argv = self._extractArgv(matcher.group(3)) + # count==3 + #..................var...............method............argv. + self._args.append([matcher.group(1), matcher.group(2), argv]) + else: + # ..................,.....1...1a....2......2..a........3....4..45...5.....6...6..7...7..8.....8...3....9 + matcher = self._match(r'\$(\w+)(?:\[([^\]]*)\])?\s*=\s*(\d+|(\W)(.*?)\4|\$(\w+)\.(\w+)\(([^()]*)\))\s*((?:-\w\S*|--[\w-]+(=\S+)?)\s*)*[\s;]*', + 'assignment', statements) + if matcher: + if matcher.group(6) != None: + argv = self._extractArgv(matcher.group(8)) + # count==6 + #...........................name...............var............method.....argv..options + self._args.append([matcher.group(1), matcher.group(6), matcher.group(7), argv, matcher.group(9), '!call']) + else: + # count==5 + # ................name...............key...............number............string...........options + self._args.append([matcher.group(1), matcher.group(2), matcher.group(3), matcher.group(5), matcher.group(9)]) + else: + matcher = None + count = len(self._args[self._ixStatement]) + if count == 6: + # assignment with call + key = None + (name, var, method, argv, options, dummy) = self._args[self._ixStatement] + value = self._callMethod(var, method, argv) + if type(value) == int: + number = value + string = None + else: + string = value + number = None + if count == 5: + (name, key, number, string, options) = self._args[self._ixStatement] + + if count == 3: + (var, method, argv) = self._args[self._ixStatement] + self._callMethod(var, method, argv) + else: + if name not in self._vars: + self._error('unknown variable: ' + name) + if string != None: + number = None + varInfo = self._vars[name] + (key, string) = self._checkOptsAndExpand(options, [key, string]) + if varInfo._type == 'int': + if key != None: + self._error('index not meaningful for int variable {:s}'.format(name)) + elif number != None: + varInfo._value = int(number) + else: + try: + varInfo._value = int(string) if string else 0 + except ValueError: + self._error("assignment of non integer to variable {:s}: '{:s}'".format( + name, string)) + elif varInfo._type == 'text': + if key != None: + self._error('index not meaningful for test variable {:s}'.format(name)) + else: + varInfo._value = string if number == None else number + elif varInfo._type == 'dict': + if key == None: + self._error('missing index for dict variable ' + name) + else: + varInfo._dict[key] = string if string != None else number + elif varInfo._type == 'list': + if key == None: + self._error('missing index for list variable ' + name) + elif key == '': + varInfo._list.append(string if string != None else number) + else: + try: + index = int(key) + except ValueError: + self._error('index {:s} is not a number for list variable {:s}'.format( + key, name)) + index = None + if index != None: + if index < 0 or index >= len(varInfo._list): + self._error('index {:d} out of _range for list variable {:s}: {:d}'.format( + index, name, len(varInfo._list) - 1)) + else: + varInfo._list[index] = string if string != None else number + else: + pass + return None if matcher == None else matcher.end() + + def _cacheRegExpr(self, pattern, name): + '''Returns a compiled regular expression using a cache. + @param pattern: the regular expression pattern + @param name: the name of the command + ''' + if name in self._reExpr: + reExpr = self._reExpr[name] + else: + reExpr = base.StringUtils.regExprCompile(pattern, 'cache pattern', self._logger) + self._reExpr[name] = reExpr + return reExpr + + def _callMethod(self, name, method, argv): + '''Calculates the value of a method belonging to a given variable. + @param name: variable name + @param method: name of the method + @param argv: the argument vector + ''' + rc = None + if name not in self._vars: + self._error('unknown variable ' + name) + else: + varInfo = self._vars[name] + methodInfo = varInfo.getMethod(method) + if methodInfo == None: + self._error('unknown method {:s} for variable {:s} with type {:s}'.format(method, name, varInfo._type)) + else: + error = methodInfo.checkArgs(argv) + if error != None: + self._error(error) + else: + rc = varInfo.call(method, argv, self) + return rc + + def _checkOptions(self, options, shortOpts, longOpts): + '''Tests whether the options are valid. + @param options: the options to inspect, e.g. '-m2 -a. --max-depth=4' + @param shortOpts: a string with all short options (one char) e.g. 'afv' + @param longOpts: a blank separated list of long options, e.g. 'max-depth min-depth' + ''' + if options != None: + opts = options.split() + longs = longOpts.split(' ') + for opt in opts: + if opt.startswith('--'): + ix = opt.find('=') + name = opt[2:] if ix < 0 else opt[2:ix] + if name not in longs: + self._error('unknown long option: ' + name + ' allowed: ' + longOpts) + elif opt.startswith('-'): + if opt[1] not in shortOpts: + self._error('unknown short option: ' + opt[1] + ' allowed: ' + shortOpts) + + def _checkOptsAndExpand(self, options, toChange, additionalShortOpts = '', additionalLongOpts = ''): + '''Check the options and expand if ordered. + @param options: the list of the current options, e.g. "-v+ --max-depth=3" + @param toChange: string or a list of strings to change. If option of expanding is given + this string or these strings will be expanded. + @param additionalShortOpts: '' or a string with all short options but 'e' and 'p' + @param additionalLongOpts: '' or a string with all long options but 'meta-esc' and 'var-prefix' + @return: the expanded version of toChange (string or list of strings) + ''' + self._checkOptions(options, 'ep' + additionalShortOpts, 'var-prefix meta-esc' + additionalLongOpts) + escMeta = self._parseOption(options, 'e', 'meta-esc') + if escMeta != None: + if len(escMeta) != 1 or escMeta.isalnum(): + self._error('invalid meta-esc (length=1, non alfanum): ' + escMeta) + else: + if type(toChange) == str: + toChange = self._expandMeta(toChange, escMeta) + else: + rc = [] + for value in toChange: + rc.append(self._expandMeta(value, escMeta)) + toChange = rc + varPrefix = self._parseOption(options, 'p', 'var-prefix') + if toChange != None and varPrefix != None: + if len(varPrefix) != 1 or varPrefix.isalnum(): + self._error('invalid var-prefix (length=1, non alfanum): ' + varPrefix) + else: + if type(toChange) == str: + toChange = self._expandVar(toChange, varPrefix) + else: + rc = [] + for value in toChange: + rc.append(self._expandVar(value, varPrefix)) + toChange = rc + return toChange + + def _clearProgram(self): + '''Resets the internal structs representing the program. + ''' + # list of full statements, e.g. ["g7 2;i "a", "p #8"] + self._statements = [] + # list of statement names, e.g. ["g", "i", "p"] + self._commands = [] + # index in args/statements/commands + self._ixStatement = -1 + # list of parts of the parsed statements, e.g. [[7, None, None, None, None],[None, "a"], ["8"]] + self._args = [] + # for compound statements: + # _infoOfBlock[<indexOfIf>] = ['if', <indexOfElse>, <indexOfFi>] + # _infoOfBlock[<indexOfElse>] = ['else', <indexOfFi>] + self._infoOfBlock = dict() + # parsing: _openBlocks[topOfStack>] = ['if', <indexOfIf>] + self._openBlocks = [] + + def _delete(self, statements, doExecute=True): + '''Handles the "delete" command. + d(elete) <length> [c(hars)|l(ines)|<mark> + @param statements: a string starting with a "delete" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "delete" command + ''' + if not doExecute: + # ..................,....a.......a....b..1...1...2.c.......c...d.......d.2...3..3b + matcher = self._match(r'd(?:elete)?\s*(?:(\d+)\s*(c(?:hars?)?|l(?:ines?)?)?|#(\d))[\s;]*', + 'delete', statements) + self._args.append([matcher.group(1), matcher.group(2), matcher.group(3)]) + else: + matcher = None + (length, unit, markNo) = self._args[self._ixStatement] + if length != None: + length = int(length) + if unit != None and unit.startswith('l'): + # delete lines + ixEnd = len(self._lines) + if self._currentPos._line + length < ixEnd: + ixEnd = self._currentPos._line + length + self._deleteBetween(Pos(self._currentPos._line, 0), Pos(ixEnd, 0)) + else: + self._deleteBetween(self._currentPos, Pos(self._currentPos._line, self._currentPos._col + length)) + else: + mark = self._marks[int(markNo)] + if mark._line < 0: + self._error('undefined mark: ' + statements[0:20]) + elif self._currentPos.less(mark): + # mark is above the current position: + self._deleteBetween(mark, self._currentPos) + self._currentPos = mark + else: + self._deleteBetween(self._currentPos, mark) + return None if matcher == None else matcher.end() + + def _deleteBetween(self, start, end): + '''Deletes the text between two given positions. + @param start: the start position + @param end: the end position + ''' + if start._line == end._line: + line = self._lines[start._line] + length = len(line) + if end._col > length: + self._lines[start._line] = line[0:start._col] + else: + self._lines[start._line] = line[0:start._col] + line[end._col:] + else: + line = self._lines[start._line][0:start._col] + line2 = self._lines[end._line] + if end._col < len(line2): + line += self._lines[end._line][end._col:] + self._lines[start._line] = line + for ix in range(min(len(self._lines) - 1, end._line), start._line - 1, -1): + del self._lines[ix] + self._adaptDeleted(start, end) + + def _else(self, doExecute): + '''Handles the "else" command. + @param doExecute: False: parsing step True: interpreting step + ''' + if not doExecute: + ixElse = len(self._args) + self._args.append(None) + blocks = len(self._openBlocks) - 1 + if blocks < 0: + self._error('else without if') + elif self._openBlocks[blocks][0] != 'if': + self._error('else without if: end of {:s} expected'.format(self._openBlocks[blocks][0])) + else: + ixIf = self._openBlocks[blocks][1] + if self._infoOfBlock[ixIf][1] != None: + self._error('2nd else found. First is statement ' + str(self._infoOfBlock[ixIf][1])) + else: + self._infoOfBlock[ixElse] = ['else', None] + self._infoOfBlock[ixIf][1] = ixElse + else: + ixFi = self._infoOfBlock[self._ixStatement][1] + self._ixStatement = ixFi + + def _error(self, msg): + '''Handles an error. + @param msg: the error message + ''' + self._errors += 1 + no = self._ixStatement + 1 + if no <= 0: + no = len(self._statements) + 1 + self._logger.error(msg + ' [statement {:d}]'.format(no)) + + def _executeScript(self, script): + '''Executes a script. + @script: a list of lines representing the script + ''' + self._lineNo = 0 + for line in script: + self._lineNo += 1 + if not line.strip().startswith('#'): + self.parse(line) + self.interpret() + + def _expandMeta(self, string, escChar = '\\', varChar = '&'): + '''Expands variables and meta characters. + @param string: the string containing meta characters and variables, e.g. "\\t\\x2e" + @param escChar: the introduction of a meta character, e.g. '\\' + @return string with each meta char and variable replaced by its value + ''' + # ..........................1.....................1 + pattern = '\\' + escChar + '([tnr]|x[0-9a-fA-F]{2})' + reExpr = self._cacheRegExpr(pattern, 'expandmeta' + escChar) + rc = '' + lastPos = 0 + for matcher in reExpr.finditer(string): + found = matcher.group(1) + tag = found[0] + if tag == 't': + expanded = '\t' + elif tag == 'n': + expanded = '\n' + elif tag == 'r': + expanded = '\r' + else: + expanded = chr(int(found[1:3], 16)) + rc += string [lastPos:matcher.start()] + expanded + lastPos = matcher.end() + rc += string[lastPos:] + return rc + + def _expandVar(self, string, varChar = '&'): + '''Expands variables. + @param string: the string containing meta characters and variables, e.g. "\t&{x}y&dict[key] &list[3]" + @param varChar: the introduction of a variable, e.g. '&' + @return string with each variable replaced by its value + ''' + # replace scalar variables: may be in index expression of a compound variables, e.g. &x[&ix] + pattern = '\\' + varChar + '(\w+|\{\w+\})' + reExpr = self._cacheRegExpr(pattern, 'expandvar' + varChar) + rc = '' + lastPos = 0 + for matcher in reExpr.finditer(string): + found = matcher.group(1) + endPos = matcher.end() + if endPos < len(string) and string[endPos] == '[': + continue + name = found[1:-1] if found.startswith('{') else found + if name not in self._vars: + self._error('unknown variable {:s} while string expanding'.format(name)) + break + varInfo = self._vars[name] + if varInfo._type not in ['int', 'text']: + self._error('wrong type {:s} of variable {:s} while string expanding'.format(varInfo._type, name)) + break + rc += string [lastPos:matcher.start()] + str(varInfo._value) + lastPos = matcher.end() + rc += string[lastPos:] + #replace dictionaries and lists: + # ..........................1...........1..2......2 + pattern = '\\' + varChar + '(\w+|\{\w+\})\[([^\]]+)\]' + reExpr = self._cacheRegExpr(pattern, 'expandcompound' + varChar) + string = rc + rc = '' + lastPos = 0 + for matcher in reExpr.finditer(string): + found = matcher.group(1) + name = found[1:-1] if found.startswith('{') else found + if name not in self._vars: + self._error('unknown variable {:s} while string expanding'.format(name)) + break + varInfo = self._vars[name] + index = matcher.group(2) + if varInfo._type == 'list': + intVal = int(index) + if intVal == 0 and index != '0': + self._error('index "{:s}" of list variable {:s} is not an integer'.format(index, name)) + break + if intVal < 0 or intVal > len(varInfo._list): + self._error('wrong index {:d} of list variable {:s}: 0-{:d}'.format(intVal, name, -1+len(varInfo._list))) + break + expanded = varInfo._list[intVal] + elif varInfo._type == 'dict': + if index not in varInfo._dict: + self._error('wrong index {:s} for dict variable {:s}'.format(index, name)) + break + expanded = varInfo._dict[index] + else: + self._error('variable {:s} is not a list or a dictionary'.format(name)) + break + rc += string [lastPos:matcher.start()] + expanded + lastPos = matcher.end() + rc += string[lastPos:] + return rc + + def _extractArgv(self, args): + '''Puts an argument list into a list ("argument vector"). + @param args: the arguments as string: a comma separated list of terms, e.g. "3, 'wow', buffer" + @return: the argument vector, e.g. [3, 'wow', <Variable-instance>] + ''' + argv = [] + # .........................a..1..12...2...3...3.4...........4a + matcher = self._match(r'\s*(?:(\W)(.*?)\1|(\d+)|([a-zA-Z]\w*))?\s*', 'arg', args) + while args != '' and matcher != None: + if matcher.group(2) != None: + argv.append(matcher.group(2)) + elif matcher.group(3) != None: + argv.append(int(matcher.group(3))) + elif matcher.group(4) != None: + name = matcher.group(4) + if name not in self._vars: + self._error('unknown variable ' + name) + break + argv.append(self._vars[name]) + args = args[matcher.end():] + if args.startswith(','): + args = args[1:] + matcher = self._reExpr['arg'].match(args) + if args != '' and self._errors == 0: + self._error('invalid syntax in argument list: ' + args) + return argv + + def _fi(self, doExecute): + '''Handles the "fi" command. + @param doExecute: False: parsing step True: interpreting step + ''' + if not doExecute: + ixFi = len(self._args) + self._args.append(None) + blocks = len(self._openBlocks) - 1 + if blocks < 0: + self._error('fi without if') + elif self._openBlocks[blocks][0] != 'if': + self._error('unexpected fi: expected block end of ' + self._openBlocks[blocks][0]) + else: + ixIf = self._openBlocks[blocks][1] + # self._infoOfBlock[ixFi] = ['fi', ixIf] + self._infoOfBlock[ixIf][2] = ixFi + ixElse = self._infoOfBlock[ixIf][1] + if ixElse != None: + self._infoOfBlock[ixElse][1] = ixFi + self._openBlocks.pop() + + def _find(self, statements, doExecute=True): + '''Handles the "find" command. + f(ind) [b(ackwards)] <pattern><flags> + @param statements: a string starting with a "find" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "find" command + ''' + if not doExecute: + # .......................a.....a.....1b..........b....1.2..23...3.....4....4.5 + matcher = self._match(r'f(?:ind)?\s*(b(?:ackwards)?\s*)?(\W)(.*?)\2\s*([ic])?(\s*(?:(?:-\w\S*|--[\w-]+(?:=\S+)?)\s*)*)[\s;]*', + 'find', statements) + self._args.append([matcher.group(1) != None, matcher.group(3), matcher.group(4), matcher.group(5)]) + else: + matcher = None + (backwards, pattern, flags, options) = self._args[self._ixStatement] + pattern = self._checkOptsAndExpand(options, pattern) + if backwards: + aTuple = self.rindexOf(pattern, self._currentPos._line, self._currentPos._col, flags) + else: + aTuple = self.indexOf(pattern, self._currentPos._line, self._currentPos._col, flags) + if aTuple == None: + self._error('pattern not found: ' + pattern) + else: + self._currentPos._line = aTuple[0] + self._currentPos._col = aTuple[1] + return None if matcher == None else matcher.end() + + def _firstPos(self, firstLine, firstCol): + '''Calculates the first position respecting the _range + @param firstLine: the line number to start + @param firstCol: the column to start + @return: a tuple (line, col) + ''' + ix = max(firstLine, 0, self._startRange._line) + col = 0 + if ix == firstLine and firstCol > col: + col = firstCol + if self._startRange._line == ix and self._startRange._col > firstCol: + firstCol = self._startRange._col + return (ix, col) + + def _for(self, statements, doExecute=True): + '''Handles the "for" command. + A for statement has a variable, a value storage and a block. + Each loop through the block is introduced by setting the variable to the next value from the storage. + If the storage is a list variable the values are the list values. + If the storage is a dict variable the values are the keys of the dict. + If the storage is a string the values are the word in the string (separated by blanks). + + @param statements: a string starting with a "if" command + @param doExecute: False: no action, syntax check and command splitting + @return: tuple (condition, the length of the "if" command) + ''' + if not doExecute: + # ..........................1...1....a..2...2.3..34...4..a5 + matcher = self._match(r'for (\w+) in (?:(\w+)|(\W)(.*?)\3|range)(\s*(?:(?:-\w\S*|--[\w-]+(?:=\S+)?)\s*)*)', + 'for', statements) + ixIf = len(self._args) + self._openBlocks.append(['if', ixIf]) + self._infoOfBlock[ixIf] = ['if', None, None] + self._args.append([matcher.group(2), matcher.group(3), matcher.group(5), matcher.group(6)]) + else: + matcher = None + (pattern, flags, expression, options) = self._args[self._ixStatement] + (pattern, expression) = self._checkOptsAndExpand(options, [pattern, expression]) + + def _if(self, statements, doExecute=True): + '''Handles the "if" command. + The if statement has a condition and one or two blocks (then block, else block) + Depending on the condition the first or the second block will be executed. + @param statements: a string starting with a "if" command + @param doExecute: False: no action, syntax check and command splitting + @return: tuple (condition, the length of the "if" command) + ''' + if not doExecute: + # ...........................a.........1..12...2.....3....3..4..45...5..a6 + matcher = self._match(r'if\s+(?:find\s+(\W)(.*?)\1\s*([ic])?|(\W)(.*?)\4)(\s*(?:(?:-\w\S*|--[\w-]+(?:=\S+)?)\s*)*)then', + 'if', statements) + ixIf = len(self._args) + self._openBlocks.append(['if', ixIf]) + self._infoOfBlock[ixIf] = ['if', None, None] + self._args.append([matcher.group(2), matcher.group(3), matcher.group(5), matcher.group(6)]) + else: + matcher = None + (pattern, flags, expression, options) = self._args[self._ixStatement] + (pattern, expression) = self._checkOptsAndExpand(options, [pattern, expression]) + if pattern != None: + aTuple = self.indexOf(pattern, self._currentPos._line, self._currentPos._col, flags) + condition = aTuple != None + if condition: + self._currentPos._line = aTuple[0] + self._currentPos._col = aTuple[1] + elif expression != None: + condition = expression != '' and expression != '0' + else: + self._error('I am confused: if: no pattern and no expression') + condition = False + if not condition: + ixElse = self._infoOfBlock[self._ixStatement][1] + self._ixStatement = ixElse + return None if matcher == None else matcher.end() + + def _insert(self, statements, doExecute=True): + '''Handles the "insert" command. + Inserts a given string at the current position. + i(nsert) <delim>string<delim> [<options>] [ + @param statements: a string starting with a "delete" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "goto" command + ''' + if not doExecute: + # .......................a.......a....1..12...2..3...b..c.................d......d.c...b.3 + matcher = self._match(r'i(?:nsert)?\s*(\W)(.*?)\1(\s*(?:(?:-\w\S*|--[\w-]+(?:=\S+)?)\s*)*)[\s;]*', + 'insert', statements) + self._args.append([matcher.group(2), matcher.group(3)]) + else: + matcher = None + (toInsert, options) = self._args[self._ixStatement] + toInsert = self._checkOptsAndExpand(options, toInsert) + if toInsert != None: + lines = toInsert.split('\n') + ix = self._currentPos._line + col = self._currentPos._col + if len(lines) == 1: + self._lines[ix] = self._lines[ix][0:col] + toInsert + self._lines[ix][col:] + elif ix >= len(self._lines): + for line in lines: + self._lines.append(line) + else: + self._lines.insert(ix + 1, lines[len(lines) - 1] + self._lines[ix][col:]) + self._lines[ix] = self._lines[ix][0:col] + lines[0] + for ix2 in range(1, len(lines) - 1): + self._lines.insert(ix + ix2, lines[ix2]) + self._adaptInserted(self._currentPos, lines) + + return None if matcher == None else matcher.end() + + def _goto(self, statements, doExecute=True): + '''Handles the "goto" command. + Changes the current position by line/column (absolute and relative) or by a given mark + g(oto) [+|-]<line> [[+|-]<col>] | <mark> + @param statements: a string starting with a "delete" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "goto" command + ''' + if not doExecute: + # .......................a.....a....b..1.....12...2c.....3.....34...4c...5.....-.5b + matcher = self._match(r'g(?:oto)?\s*(?:([+-]?)(\d+)(?:\s*([-+]?)(\d+))?|#([<>0-9]))?[\s;]*', + 'goto', statements) + self._args.append([matcher.group(1), matcher.group(2), matcher.group(3), matcher.group(4), matcher.group(5)]) + else: + matcher = None + (signLine, line, signCol, col, markNo) = self._args[self._ixStatement] + if line != None: + line = int(line) + if signLine == '': + self._currentPos._line = line + elif signLine == '+': + self._currentPos._line += line + else: + self._currentPos._line -= line + if col == None: + self._currentPos._col = 0 + else: + col = int(col) + if signCol == '': + self._currentPos._col = col + elif signCol == '+': + self._currentPos._col += col + else: + self._currentPos._col -= col + elif markNo != None: + ixMark = self._indexOfMark(markNo) + if self._marks[ixMark] < 0: + self._error('undefined mark in "goto" command: #' + str(ixMark)) + else: + self._currentPos.clone(self._marks[ixMark]) + else: + self._errors('I am confused') + return None if matcher == None else matcher.end() + + def _indexOfMark(self, name): + '''Returns the index number of the mark given by name. + @param name: '<', '>' or a decimal digit + @return: 0..11 + ''' + if name == '<': + rc = 10 + elif name == '>': + rc = 11 + self._marks[rc]._line = len(self._lines) + else: + rc = int(name) + return rc + + def _load(self, statements, doExecute=True): + '''Handles the "load" command. + Transfers content from the file content (current position) to a given variable. + l(oad) <variable> { #[0-9]|<number> [c(ars)|l(ines)] } + @param statements: a string starting with a "load" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "load" command + ''' + if not doExecute: + # .......................1...1.......2...2...3.4.......4.5...5...6.a.......a...b.....b...6.3 + matcher = self._match(r'l(oad)?\s+\$?(\w+)\s*(#([<>0-9])|(\d+)\s*(l(?:ines?)?|c(?:hars?)?)?)?[\s;]*', 'load', statements) + if matcher: + self._args.append([matcher.group(2), matcher.group(4), matcher.group(5), matcher.group(6)]) + else: + matcher = None + loadAll = False + (name, markNo, number, unit) = self._args[self._ixStatement] + if markNo != None: + ixMark = self._indexOfMark(markNo) + mark = self._marks[ixMark] + if mark.less(self._currentPos): + content = self._loadRange(self._currentPos, mark) + else: + content = self._loadRange(mark, self._currentPos) + elif number != None: + number = int(number) + if unit == None or unit.startswith('c'): + content = self._loadRange(self._currentPos, + Pos(self._currentPos._line, self._currentPos._col + number)) + else: + content = self._loadRange(self._currentPos, + Pos(self._currentPos._line + number, 0)) + else: + loadAll = True + if name not in self._vars: + self._error('unknown variable {:s} in load'.format(name)) + else: + varInfo = self._vars[name] + if loadAll and varInfo._type != 'list': + bof = self._indexOfMark('<') + eof = self._indexOfMark('>') + content = content = self._loadRange(self._marks[bof], self._marks[eof]) + if varInfo._type == 'int': + try: + varInfo._value = int(content) + except ValueError: + self._error('wrong value {:s} for the int variable {:s}'.format(content[0:20], name)) + elif varInfo._type == 'text': + varInfo._value = content + elif varInfo._type == 'list': + if loadAll: + varInfo._list = self._lines[:] + else: + varInfo._list = content.split('\n') + else: + self._error('cannot load into the {:s} variable {:s}'.format(varInfo._type, name)) + return None if matcher == None else matcher.end() + + def _loadRange(self, start, end): + '''Returns a _range of the lines as string. + @param start: the start position + @param end: the end position (excluded). end > start + ''' + countLines = len(self._lines) + if start._line == end._line: + rc = self._lines[start._line][start._col:min(len(self._lines[start._line]), end._col)] + else: + rc = self._lines[start._line][start._col:] + for ix in range(start._line + 1, min(end._line, countLines)): + rc += '\n' + self._lines[ix] + if end._line < countLines: + endCol = min(end._col, len(self._lines[end._line])) + rc += '\n' + self._lines[end._line][0:endCol] + return rc + + def _mark(self, statements, doExecute=True): + '''Handles the "mark" command. + Sets a given mark to the current position. + m(ark) #[0-9] + @param statements: a string starting with a "mark" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "mark" command + ''' + if not doExecute: + # .......................a.....a.....1.....1 + matcher = self._match(r'm(?:ark)?\s*#([0-9]);]*', 'mark', statements) + self._args.append(int(matcher.group(1))) + else: + matcher = None + no = self._args[self._ixStatement] + self._marks[no].clone(self._currentPos) + return None if matcher == None else matcher.end() + + def _match(self, pattern, name, statements, cryOnError=True): + '''Returns a re.Matcher instance of a pattern for a given command. + A cache of compiled reg. expressions will be maintained. + @param pattern: the regular expression pattern + @param name: the name of the command + @param statements: a text starting a statements with the given command + ''' + reExpr = self._cacheRegExpr(pattern, name) + matcher = reExpr.match(statements) + if matcher == None and cryOnError: + self._error('syntax error in "{:s}" command. Syntax: {:s}\n{:s}'.format( + name, pattern, statements[0:40], ) ) + return matcher + + def _numericAssignment(self, statements, doExecute=True): + '''Handles the "assign" command. + '$' <var> {-=|+=|*=|/=|%=|:=} { <term> {-|+|*|/|%} <term> } + @param statements: a string starting with a "numeric assignment" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "numeric assignment" command + ''' + if not doExecute: + # ..................,.....1...1...2.......2....a..3.....3...4...4a...b..5.......5...c..6.....6...7...7cb + matcher = self._match(r'\$(\w+)\s*([-+*/%:])=\s*(?:(-?\d+)|\$(\w+))\s*(?:([-+*/%])\s*(?:(-?\d+)|\$(\w+)))?[\s;]*', + 'numassign', statements) + if matcher: + self._args.append([matcher.group(1), matcher.group(2), matcher.group(3), matcher.group(4), + matcher.group(5), matcher.group(6), matcher.group(7)]) + else: + matcher = None + (name, opAssignment, number1, var1, op, number2, var2) = self._args[self._ixStatement] + if name not in self._vars: + self._error('unknown variable: ' + name) + else: + varInfo = self._vars[name] + term1 = self._term(number1, var1) + if term1 != None and op != None: + term2 = self._term(number2, var2) + if term2 != None: + if op == '+': + term1 += term2 + elif op == '-': + term1 -= term2 + elif op == '*': + term1 *= term2 + elif op == '/': + if term2 == 0: + self._error('division by 0') + term1 = None + else: + term1 = int(term1 / term2) + elif op == '%': + if term2 == 0: + self._error('0 at % operator') + term1 = None + else: + term1 %= term2 + if term1 != None: + if opAssignment == ':': + varInfo._value = term1 + elif varInfo._value != None: + if opAssignment == '+': + varInfo._value += term1 + elif opAssignment == '-': + varInfo._value -= term1 + elif opAssignment == '*': + varInfo._value *= term1 + elif opAssignment == '/': + if term1 == 0: + self._error('division by 0') + varInfo._value = None + else: + varInfo._value = int(varInfo._value / term1) + elif opAssignment == '%': + varInfo._value %= term1 + return None if matcher == None else matcher.end() + + def _oneStatement(self, cmd, statements, doExecute): + '''Executes or parses one statements. + @param cmd: the statement name + @param statements: the statements. The first of them will be executed / parsed + @param doExecute: True: the statement will be interpreted. Otherwise it will be parsed to split + @param opt: optional parameter + @return: the length of the first statement + ''' + length = None + if cmd == 'assign': + length = self._assign(statements, doExecute) + elif cmd == 'numeric': + length = self._numericAssignment(statements, doExecute) + elif cmd == 'd' or cmd == 'delete': + length = self._delete(statements, doExecute) + elif cmd == 'else': + length = 4 + self._else(doExecute) + elif cmd == 'f' or cmd == 'find' or cmd == 'fb' or cmd == 'findbackwards' or cmd == 'fbackwards': + length = self._find(statements, doExecute) + elif cmd == 'fi': + length = 2 + self._fi(doExecute) + elif cmd == 'g' or cmd == 'goto': + length = self._goto(statements, doExecute) + elif cmd == 'i' or cmd == 'insert': + length = self._insert(statements, doExecute) + elif cmd == 'if': + length = self._if(statements, doExecute) + elif cmd == 'l' or cmd == 'load': + length = self._load(statements, doExecute) + elif cmd == 'm' or cmd == 'mark': + length = self._mark(statements, doExecute) + elif cmd == 'p' or cmd == 'print': + length = self._print(statements, doExecute) + elif cmd == 'r' or cmd == '_range': + length = self._range(statements, doExecute) + elif cmd == 'read': + length = self._read(statements, doExecute) + elif cmd == 's' or cmd == 'substitute': + length = self._substitute(statements, doExecute) + elif cmd == 'v' or cmd == 'var' or cmd == 'variable': + length = self._variable(statements, doExecute) + elif cmd == 'w' or cmd == 'write': + length = self._write(statements, doExecute) + else: + self._logger.error('unknown command: ' + cmd) + return length + + def _parseOption(self, options, shortOption, longOption): + '''Gets the value of a given option from all options. + @param options: a string with all found options + @param shortOption: a char as short option name + @param longOption: a string as long option name + @return: None: option not found + otherwise: the value of the option + ''' + rc = None + if options != None and options != '': + rc = base.StringUtils.tailOfWord(options, '--' + longOption + '=') + if rc == None: + rc = base.StringUtils.tailOfWord(options, '--' + longOption) + if rc == None: + rc = base.StringUtils.tailOfWord(options, '-' + shortOption) + return rc + + def _print(self, statements, doExecute=True): + '''Handles the "print" command. + Transfers content from the file content (current position) to a given variable. + p(rint) { #[0-9]|<number> [c(ars)|l(ines)]|<delim><string><delim> [<opts>] } + @param statements: a string starting with a "print" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "print" command + ''' + if not doExecute: + # .......................a......a....b...1.....1.2...2...3.c.......c...d.......d.3..4..45...5..b6...e..f.................g......g.f...e.6 + matcher = self._match(r'p(?:rint)?\s+(?:#([<>0-9])|(\d+)\s*(l(?:ines?)?|c(?:hars?)?)?|(\W)(.*?)\4)(\s*(?:(?:-\w[^;\s]*|--[\w-]+(?:=\S+)?)\s*)*)[\s;]*', + 'print', statements) + self._args.append([matcher.group(1), matcher.group(2), matcher.group(3), matcher.group(5), matcher.group(6)]) + else: + matcher = None + (markNo, number, unit, content, options) = self._args[self._ixStatement] + if content != None: + content = self._checkOptsAndExpand(options, content, 'l', 'to-log') + else: + self._checkOptions(options, 'l', 'to-log') + toLog = self._parseOption(options, 'l', 'to-log') + if markNo != None: + ixMark = self._indexOfMark(markNo) + mark = self._marks[ixMark] + if mark.less(self._currentPos): + content = self._loadRange(self._currentPos, mark) + else: + content = self._loadRange(mark, self._currentPos) + elif number != None: + number = int(number) + if unit == None or unit.startswith('c'): + content = self._loadRange(self._currentPos, + Pos(self._currentPos._line, self._currentPos._col + number)) + else: + content = self._loadRange(self._currentPos, + Pos(self._currentPos._line + number, 0)) + if content.endswith('\n'): + content = content[0:-1] + if toLog == None: + print(content) + else: + self._logger.log(content) + return None if matcher == None else matcher.end() + + def _lastPos(self, lastLine, lastCol): + '''Calculates the last position respecting the _range + @param lastLine: the line number to start + @param lastCol: the column to start + @return: a tuple (line, col) + ''' + ix = min(lastLine, len(self._lines) - 1, self._endRange._line) + col = len(self._lines[ix]) + if ix == lastLine and lastCol < col: + col = lastCol + if self._endRange._line == ix and self._endRange._col < lastCol: + lastCol = self._endRange._col + return (ix, col) + + def _range(self, statements, doExecute=True): + '''Handles the "_range" command. + Sets a line _range for operating. + r(ange) [ <markStart> [ <markEnd> ]] + @param statements: a string starting with a "_range" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "_range" command + ''' + if not doExecute: + # ....................a......a....b..1...1...c...2..2.cb + matcher = re.match('^r(?:ange)?\s*(?:(#\d)\s*(?:#(\d)?))?[;\s]*', statements) + self._args.append([int(matcher.group(1)), matcher.group(2)]) + else: + matcher = None + (start, end) = self._args[self._ixStatement] + if end == None: + self._startRange = Pos(0, 0) + self._endRange = Pos(len(self._lines), 0) + else: + self._startRange.clone(self._marks[start]) + if end == None: + self._endRange = Pos(len(self._lines), 0) + else: + self._endRange.clone(self._marks[int(end)]) + return None if matcher == None else matcher.end() + + def _read(self, statements, doExecute=True): + '''Handles the "read" command. + Reads the file given by name + r(ead) <delim><string><delim> [<opts>] + @param statements: a string starting with a "write" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "write" command + ''' + if not doExecute: + # .............................1..12...2..3...c..d.................e......e.d...c.3 + matcher = self._match(r'read\s*(\W)(.*?)\1(\s*(?:(?:-\w[^\s;]*|--[\w-]+(?:=\S+)?)\s*)*)[\s;]*', 'read', statements) + self._args.append([matcher.group(2), matcher.group(3)]) + else: + matcher = None + (name, options) = self._args[self._ixStatement] + if name == None: + name = self._name + else: + name = self._checkOptsAndExpand(options, name) + self._readFile(name) + return None if matcher == None else matcher.end() + + def _readFile(self, name): + '''Reads a given file into the internal lines. + @param name: the filename + ''' + self._lines = [] + if not os.path.exists(name): + self._error('file does not exists: ' + name) + else: + with open(name, "r") as fp: + self._lines = fp.read().split('\n') + + def _readStdIn(self): + '''Reads the content of stdin and returns the array of lines. + @return: the list of lines + ''' + rc = [] + for line in sys.stdin: + if line.endswith('\n'): + line = line[0:-1] + if line.endswith('\r'): + line = line[0:-1] + rc.append(line) + return rc + + def _substitute(self, statements, doExecute=True): + '''Handles the "find" command. + s(ubstitute) <delim>pattern<delim>replacement<delim> [c|i] <opts> + @param statements: a string starting with a "substitute" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "substitute" command + ''' + if not doExecute: + # .......................a...........a....1..12...2..3...3.....4....4.5...b..c...............d....d.c...b.5 + matcher = self._match(r's(?:ubstitute)?\s*(\W)(.*?)\1(.*?)\1\s*([ic])?(\s*(?:(-\w\S*|--[\w-]+(=\S+)?)\s*)*)[\s;]*', + 'substitute', statements) + self._args.append([matcher.group(2), matcher.group(3), matcher.group(4), matcher.group(5)]) + else: + matcher = None + (pattern, replacement, flags, options) = self._args[self._ixStatement] + (pattern, replacement) = self._checkOptsAndExpand(options, [pattern, replacement], + 'l', ' limit') + limit = self._parseOption(options, 'l', 'limit') + if limit == None: + limit = 0x7fffffff + else: + try: + limit = int(limit) + except ValueError: + self._error('+++ option limit is invalid: ' + limit) + limit = 1 + self.replace(pattern, replacement, limit, self._currentPos._line, self._currentPos._col, flags) + return None if matcher == None else matcher.end() + + def _term(self, number, name): + '''Returns the numeric value of a term: a number or a numeric variable. + @param number: None or a numeric constant + @param name: None or a variable name. Must contain a numeric value + @return: None: error found + otherwise: the numeric value of the term + ''' + rc = None + if number != None: + rc = int(number) + elif name != None: + if name not in self._vars: + self._error('unknown variable: ' + name) + else: + varInfo = self._vars[name] + if varInfo._type == 'int': + rc = varInfo._value + elif varInfo._type == 'text': + try: + rc = int(varInfo._value) + except ValueError: + self._error('text variable {:s} contains no numeric value: {:s}'.format(name, varInfo._value)) + else: + self._error('numeric term expected but variable {:s} has type {:s}'.format(name, varInfo._type)) + else: + self._error('I am confused: no num, no term') + return rc + + def _variable(self, statements, doExecute=True): + '''Handles the "variable" command. + c(reate)\s+(var(iable)?\s+$(\w+)|dict(ionary)?\s+%(\w+)|array\s+@(\w+)) + @param statements: a string starting with a "delete" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "delete" command + ''' + if not doExecute: + #........................a....b.......b.a..1.c.....c...d.....d...e.....e...f....f.1...2...23g........g.3 + matcher = self._match(r'v(?:ar(?:iable)?)? (t(?:ext)?|l(?:ist)?|d(?:ict)?|i(?:nt)?)\s+(\w+)((?:\s+\w+)+)?[\s;]*', + 'var', statements) + names = [matcher.group(2)] + rest = matcher.group(3) + if rest != None: + for name in rest.split(): + names.append(name) + aType = matcher.group(1) + self._args.append(aType) + for name in names: + if name in self._vars: + msg = '' if aType == self._vars[name]._type else ' with a different type: ' + self._vars[name]._type + self._error('variable {:s} defined twice'.format(name) + msg) + continue + if aType.startswith('t'): + self._vars[name] = TextVariable(name) + elif aType.startswith('d'): + self._vars[name] = DictVariable(name) + elif aType.startswith('l'): + self._vars[name] = ListVariable(name) + elif aType.startswith('i'): + self._vars[name] = IntegerVariable(name) + else: + self._error('unknown variable type ' + aType) + break + else: + matcher = None + return None if matcher == None else matcher.end() + + def _write(self, statements, doExecute=True): + '''Handles the "write" command. + Write the lines into a file + w(rite) [<delim><filename><delim>] [<opts>] } + @param statements: a string starting with a "write" command + @param doExecute: False: no action, syntax check and command splitting + @return: the length of the "write" command + ''' + if not doExecute: + # .......................a......a....b..1..12...2..b.3...c..d.................e......e.d...c.3 + matcher = self._match(r'w(?:rite)?\s*(?:(\W)(.*?)\1)?(\s*(?:(?:-\w\S*|--[\w-]+(?:=\S+)?)\s*)*)[\s;]*', 'write', statements) + self._args.append([matcher.group(2), matcher.group(3)]) + else: + matcher = None + (name, options) = self._args[self._ixStatement] + if name == None: + name = self._name + else: + name = self._checkOptsAndExpand(options, name) + if name == None or name == '': + self._error('missing file name for writing') + else: + with open(name, "w") as fp: + for line in self._lines: + fp.write(line + '\n') + return None if matcher == None else matcher.end() + + def execute(self, argv): + '''Executes a sequence of statements on a given file. + @param argv: the arguments: [<statements>] or [<statements>, <file>] + @return: None: OK + otherwise: the error message + ''' + rc = None + if len(argv) == 0: + rc = 'missing statements' + else: + statements = argv[0] + if len(argv) > 1: + if argv[1] != '-': + self._readFile(argv[1]) + else: + self._lines = self._readStdIn() + self.parseAndInterpret(statements) + return rc + + def parseAndInterpret(self, statements): + '''Parses and executes a statement list. + @param statements: a sequence of statements + @return: 0: success 1: syntax error + ''' + self._clearProgram() + self.parse(statements) + if self._errors > 0: + rc = 1 + else: + rc = self.interpret() + return rc + + def indexOf(self, pattern, startLine=0, startCol = 0, flags = None): + '''Returns the first index of a pattern given by a reg. expression. + Note: the _range is respected while the search + @param pattern: a regular expression to search + @param startLine: the index of the first line to search: 0..N-1 + @param startCol: the index of the char in the first line for starting the search: 0..M-1 + @param flags: flags for re.compile(): 'i': ignore case 'c': case sensitive otherwise: re.RegexFlag e.g. re.I + @return: None: not found + otherwise: a tuple (ixLine, ixCol, length, matcher) of the expression + ''' + ix = startLine + maxLine = len(self._lines) + if self._endRange._line < maxLine: + maxLine = self._endRange._line + unprocessedLine = None + colPrefixLength = 0 + if flags == None: + flags = self._reFlags + elif flags == 'i': + flags = IGNORE + elif flags == 'c': + flags = 0 + + reExpr = base.StringUtils.regExprCompile(pattern, 'indexOf', self._logger, flags == 0) + ixLine = None + while unprocessedLine != None or ix < maxLine: + if unprocessedLine != None: + toInspect = unprocessedLine + else: + colPrefixLength = 0 + toInspect = self._lines[ix] + if ix == startLine and startCol > 0: + toInspect = toInspect[startCol:] + colPrefixLength = startCol + ix += 1 + matcher = reExpr.search(toInspect) + if matcher == None: + unprocessedLine = None + else: + ixLine = ix - 1 + ixCol = matcher.start() + length = matcher.end() - ixCol + unprocessedLine = toInspect[ixCol + length:] + colPrefixLength += ixCol + length; + break + return None if ixLine == None else (ixLine, colPrefixLength - length, length, matcher) + + def indexOfMultiple(self, patterns, startLine=0, startCol = 0): + '''Returns the first index of a squences of patterns given by reg. expressions. + @param patterns: a list of regular expressions to search + @param startLine: the index of the first line to search: 0..N-1 + @param startCol: the index of the char in the first line for starting the search: 0..M-1 + @return: None: not found + otherwise: a tuple (ixLine, ixCol, length, matcher) of the expression + ''' + rc = None + for pattern in patterns: + rc = self.indexOf(pattern, startLine, startCol) + if rc == None: + break + else: + startLine = rc[0] + startCol = rc[1] + rc[2] + return rc + + def interpret(self): + '''Interprets all statements. + ''' + rc = 0 + self._ixStatement = 0 + doExecute = True + while self._ixStatement < len(self._statements) and self._errors == 0: + self._oneStatement(self._commands[self._ixStatement], self._statements[self._ixStatement], doExecute) + self._ixStatement += 1 + return rc + + def rindexOf(self, pattern, startLine=0x7fffffff, startCol = 0x7ffffff, flags = None): + '''Returns the last index of a pattern given by a reg. expression. + Note: the _range is respected while the search + @param pattern: a regular expression to search + @param startLine: the index of the first line (from the end) to search: 0..N-1 + @param startCol: the index of the char in the first line for starting the search: 0..M-1 + @param flags: flags for re.compile(), e.g. re.I + @return: None: not found + otherwise: a tuple (ixLine, ixCol, length, matcher) of the expression + ''' + if flags == 'i': + flags = IGNORE + elif flags == 'c': + flags = 0 + else: + flags = self._reFlags + reExpr = re.compile(pattern, flags if flags != None else self._reFlags) + def find(line, start, end): + matcherList = [] + for m in reExpr.finditer(line, start, end): + matcherList.append(m) + count = len(matcherList) + if count == 0: + return None + # Bug in finditer(): if a nonempty match is preceeded by an empty it eats the first char + if count > 1 and matcherList[count - 2].group() == '' and matcherList[count - 1].group() != '': + # search again 1 char above the last hit: + return reExpr.search(max(0, matcherList.start() - 1)) + else: + return matcherList[count - 1] + (ix, lastCol) = self._lastPos(startLine, startCol) + firstCol = 0 if ix != self._startRange._line else self._startRange._col + matcher = None if lastCol <= 0 else find(self._lines[ix], firstCol, lastCol) + if matcher == None: + (ixFirst, firstCol) = self._firstPos(0, 0) + # search in full lines: + ix -= 1 + while ix > ixFirst: + matcher = find(self._lines[ix], 0, len(self._lines[ix])) + if matcher != None: + break + ix -= 1 + if matcher == None: + firstCol = 0 if self._startRange._line != ixFirst else self._startRange._col + matcher = find(self._lines[ixFirst], firstCol, len(self._lines[ixFirst])) + rc = None if matcher == None else (ix, matcher.start(), matcher.end() - matcher.start(), matcher) + return rc + + def parse(self, statements): + '''Splits a command sequence into single statements. + Note: the method can be called multiple times to collect some sources. + @param statements: a sequence of statements to parse + ''' + doExecute = False + while statements != None and len(statements) > 0: + # ..........................12......2.........3........3.......1 + matcher = re.match(r'^[\s;]*(([a-z]+)|\$\w+\s*([-+*/%:])=|\$\w+)', statements) + if matcher == None: + self._logger.error('unknown command: ' + statements.strip()[0:40]) + ix = statements.find(';') + if ix > 0: + statements = statements[ix:] + else: + break + elif matcher.group(2) != None: + cmd = matcher.group(2) + elif matcher.group(3) != None: + cmd = 'numeric' + else: + cmd = 'assign' + if matcher.start(1) > 0: + statements = statements[matcher.start(1):] + length = self._oneStatement(cmd, statements, doExecute) + if length == None: + length = statements.find(';') + if self._errors == 0: + self._error('cannot parse: ' + statements[0:length]) + if length < 0: + break; + self._statements.append(statements[0:length]) + self._commands.append(cmd) + statements = statements[length:] + + def replace(self, pattern, replacement, limit = 0x7ffffff, startLine = 0, startCol = 0, flags = None): + '''Replaces a search expression given by patterns with a replacement. + The last reg. expression will be replaced by a given string + @param pattern: a regular expression to search + @param replacement: the reg. expression will be replaced by this string + @param limit: the search/replacement process will be done in such many lines + @param startLine: the index of the line to start search + @param startCol: the index of the column (of the startLine) to start search + @param flags: search flags: + string: 'i': ignore case 'c': case sensitive None: default + int: flags like re.I|re.D + @return: False: not found True: found and replaced + ''' + rc = False + hasMacros = not self._ignoreMacros and re.search(r'\[1-9]', replacement) != None + replacement2 = replacement + while limit > 0: + limit -= 1 + aTuple = self.indexOf(pattern, startLine, startCol, flags) + rc = aTuple != None + if not rc: + break + else: + (startLine, startCol, length, matcher) = aTuple + line = self._lines[startLine] + if hasMacros: + replacement2 = matcher.expand(replacement) + line = line[0:startCol] + replacement2 + line[startCol+length:] + self._lines[startLine] = line + self._changed = True + startCol += len(replacement2) + return rc + + def script(self, argv): + '''Reads a sequence of statements from a file or stdin and executes them on a given file. + @param argv: the arguments: [<script>] or [<script>, <file>] + @return: None: OK + otherwise: the error message + ''' + rc = None + if len(argv) == 0: + rc = 'missing script' + else: + if argv[0] == '-': + script = self._readStdIn() + else: + self._scriptName = argv[0] + if not os.path.exists(argv[0]): + rc = 'script {:s} does not exist'.format(argv[0]) + else: + with open(argv[0], 'r') as fp: + script = fp.read().split() + if rc == None: + if len(argv) > 1: + if argv[1] != '-': + self._readFile(argv[1]) + else: + self._lines = self._readStdIn() + self._executeScript(script) + return rc + + def setIgnoreCase(self, ignoreCase=True): + '''Sets or clears the case sensivity of the following searches. + @param ignoreCase: True: the search will be case sensitive + ''' + if ignoreCase: + self._reFlags |= IGNORE + else: + self._reFlags &= IGNORE + +if __name__ == '__main__': + pass diff --git a/base/ThreadLogger.py b/base/ThreadLogger.py new file mode 100644 index 0000000..ccfb6b9 --- /dev/null +++ b/base/ThreadLogger.py @@ -0,0 +1,38 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +import datetime +import threading +from base.Logger import Logger + +class ThreadLogger(Logger): + def __init__(self, logfile, verbose): + '''Constructor. + @param logfile: the file for logging + @param verbose: true: logging to stdout too + ''' + self._lock = threading.Lock() + Logger.__init__(self, logfile, verbose) + + def log(self, message, minLevel = 0): + '''Logs a message. + @param message: the message to log + @param minLevel: the logging is done only if _verboseLevel >= minLevel + @return: true: OK false: error on log file writing + ''' + rc = False + now = datetime.datetime.now() + message = now.strftime('%Y.%m.%d %H:%M:%S') + '[{}] {:s}'.format(threading.current_thread()._name, message) + with self._lock: + if self._verboseLevel >= minLevel: + print(message) + with open(self._logfile, 'a') as fp: + rc = True + fp.write(message + '\n') + return rc + +if __name__ == '__main__': + logger = ThreadLogger('/tmp/test.log', True) + logger.log('Hi'); diff --git a/base/Zipper.py b/base/Zipper.py new file mode 100644 index 0000000..58564b7 --- /dev/null +++ b/base/Zipper.py @@ -0,0 +1,468 @@ +''' +Created on 05.07.2018 + +@author: hm +''' +import zipfile +import re +import os +import time +import base.MemoryLogger +from _ast import arg +from fnmatch import fnmatch + +class Zipper(object): + ''' + Maintain a zip archive. + ''' + + def __init__(self, name, mode = 'w', logger = None, + noCompression = 'zip|tgz|gzip|gz|bz2|lz|lzma|xz|7z|deb|rpm|cab|rar|jar|war|ear' + + '|jpg|png|gif|tiff?jpeg|gzip|mp3|mpeg|mp4|m4p||flv|f4[abv]|vob|ogv|avi|mov' + + '|odt|ott|ods|odg|otg|odp|odb|doc|docx|xlsx?|xltx|xlw'): + ''' + Constructor. + @param name: the archive filename + @param mode: 'w' create a new archive 'a': append to an achive 'r': open readonly. + @param noCompression: a '|' separated list of file extensions: these files will be stored without expression + ''' + self._name = name + self._createStatistic = ZipCreateStatistic() + self._extractStatistic = ZipExtractStatistic() + self._logger = logger if logger != None else base.MemoryLogger.MemoryLogger(1) + self._mode = mode + self._fullNames = None + self._nodes = [] + self._zip = None + try: + self._zip = zipfile.ZipFile(name, mode, zipfile.ZIP_DEFLATED) + except Exception as exc: + self._logger.error('cannot {:s} archive {:s}: {:s}'.format('read' if mode=='r' else 'write', name, str(exc))) + self._rexprNoCompression = re.compile(r'\.(' + noCompression + ')$', re.RegexFlag.IGNORECASE); + self._dst = time.daylight + + def append(self, name, internalName = None, compression=True): + '''Appends a file to the archive. + @param name: the full name of the file to store + @param internalName: this name will be stored. If None the name will be used (without a preceding '/') + @param compression: False: never compress the file True: compress if file extension "allows" this + ''' + self._createStatistic._files += 1 + if not compression: + self._createStatistic._uncompressedFiles += 1 + if self._mode == 'r': + raise Exception('cannot append to readonly archive') + if not compression: + self._createStatistic._uncompressedFiles += 1 + else: + mapper = self._rexprNoCompression.search(name) + if mapper != None: + compression = False + self._createStatistic._uncompressedByExt += 1 + try: + if not os.path.islink(name): + self._zip.write(name, internalName, zipfile.ZIP_DEFLATED if compression else zipfile.ZIP_STORED) + else: + self._createStatistic._links += 1 + zipInfo = zipfile.ZipInfo(internalName) + zipInfo.create_system = 3 + # long type of hex val of '0xA1ED0000L', + # say, symlink attr magic... + zipInfo.external_attr = 2716663808 + self._zip.writestr(zipInfo, os.readlink(name)) + except Exception as exc: + self._logger.error('cannot read: {:s} [{:s}]'.format(name, str(exc))) + + def appendDir(self, dirName, lengthBase=0, regExprFilesIgnored = None, recursive=True, regExprDirsIgnored=None, + regExprDirsNoCompression = None, compression=True): + '''Appends all file of a directory to the archive. + @param dirName: the full name of the directory to store + @param lengthBase: if > 0: the stored name will be cut at the top by this number of characters + @param regExprFilesIgnored: None or a compiled regular expression of file which are not stored, + e.g. re.compile('~$') + @param recursive: True: all subdirectories will be stored too + @param regExprDirsIgnored: None or a compiled regular expression of directories which are not stored, + e.g. re.compile(r'^\.(metadata|project)$'); + @param regExprDirsNoCompression: None or the compiled regular expression of directories which will not be compressed + e.g. re.compile(r'^\.(git|svn|cvs)$'); + @param compressed: False: none of the files/dirs will be compressed + ''' + self._createStatistic._directories += 1 + if not compression: + self._createStatistic._uncompressedDirs += 1 + files = os.listdir(dirName) + dirs = [] + fullPrefix = dirName + os.sep + internalPrefix = (dirName if lengthBase == 0 else dirName[lengthBase:]) + '/' + self.appendDirectoryEntry(dirName, internalPrefix) + if os.sep != '/': + internalPrefix = internalPrefix.replace(os.sep, '/') + for name in files: + full = fullPrefix + name + if not os.path.isdir(full): + if regExprFilesIgnored == None: + self.append(full, internalPrefix + name, compression) + elif regExprFilesIgnored.search(name): + self._createStatistic._ignoredFiles += 1 + else: + self.append(full, internalPrefix + name, compression) + elif recursive: + if regExprDirsIgnored == None: + dirs.append(name) + elif regExprDirsIgnored.search(name) == None: + dirs.append(name) + else: + self._createStatistic._ignoredDirs += 1 + for name in dirs: + if not compression: + compr = False + else: + if regExprDirsNoCompression == None: + compr = True + else: + if name.find('.git') >= 0: + pass + matcher = regExprDirsNoCompression.search(name) + if matcher == None: + compr = True + else: + compr = False + self._createStatistic._uncompressedRoots += 1 + self.appendDir(fullPrefix + name, lengthBase, regExprFilesIgnored, recursive, regExprDirsIgnored, + regExprDirsNoCompression, compr) + + def appendDirectoryEntry(self, dirName, internalName): + '''Appends a directory into the zip archive. + @param dirName: the full directory name + @param internalName: the name in the archive + ''' + if not internalName.endswith('/'): + internalName += '/' + statInfo = os.lstat(dirName) + mTime = time.localtime(statInfo.st_mtime) + zipInfo = zipfile.ZipInfo(internalName, mTime) + zipInfo.external_attr = statInfo.st_mode << 16 + self._zip.writestr(zipInfo, '') + + def close(self): + '''Frees the resources. + ''' + self._zip.close() + + def create(self, argv): + '''Creates a zip archive. + @param argv: the program arguments (behind the archive>) + @return: None: OK otherwise: the error message + ''' + rc = None + mode = 'f' + regExprFilesIgnored = None + regExprDirsIgnored = None + regExprDirsNoCompression = None + logStatistic = False + for arg in argv: + if arg == '--store-node-only': + mode = 'n' + elif arg == '--statistic': + logStatistic = True + elif arg == '--shortest-path': + mode = 's' + elif arg.startswith('--ignored-files='): + regExprFilesIgnored = base.StringUtils.regExprCompile(arg[16:], 'ignored files pattern', self._logger) + elif arg.startswith('--ignored-dirs='): + regExprDirsIgnored = base.StringUtils.regExprCompile(arg[15:], 'ignored dirs pattern', self._logger) + elif arg.startswith('--already-compressed-dirs='): + expr = arg[26:] + regExprDirsNoCompression = base.StringUtils.regExprCompile(expr, 'already compressed dirs pattern', self._logger) + elif arg.startswith('-'): + rc = 'unknown option: ' + arg + else: + if arg.endswith(os.sep): + arg = arg[:-1] + if mode == 'f': + start = 0 + elif mode == 's': + start = len(arg) + else: + start = arg.rfind(os.sep) + 1 + if not os.path.exists(arg): + self._logger.error('file does not exist: ' + arg) + elif os.path.isdir(arg): + compr = True + if regExprDirsNoCompression != None: + node = os.path.basename(arg) + compr = regExprDirsNoCompression.search(node) == None + if not compr: + self._createStatistic._uncompressedRoots += 1 + self.appendDir(arg, start, regExprFilesIgnored, True, regExprDirsIgnored, regExprDirsNoCompression, compr) + else: + if mode == 's': + start = start = arg.rfind(os.sep) + 1 + elif mode == 'n': + start = arg.rfind(os.sep, 0, start - 1) + self.append(arg, arg[start:]) + if rc: + if len(argv) == 0: + rc = 'missing input file(s)' + if logStatistic: + self._createStatistic.log(self._logger) + return rc + + def extract(self, argv): + '''Creates a zip archive. + @param argv: the program arguments (behind the archive>) + @return: None: OK otherwise: the error message + ''' + rc = None + patternMode = 's' + targetMode = 'n' + writeMode = 'o' + destination = '.' + os.sep + for arg in argv: + if arg.startswith('-d') or arg.startswith('--destination='): + destination = arg[2:] if arg.startswith('-d') else arg[14:] + if not os.path.isdir(destination): + self._logger.error('not a directory: ' + destination) + break + destination += os.sep + elif arg == '-u' or arg == '--update': + writeMode = 'u' + elif arg == '-o' or arg == '--overwrite': + writeMode = 'o' + elif arg == '-t' or arg == '--not-overwrite': + writeMode = 'n' + elif arg == '-f' or arg == '--freshen': + writeMode = 'f' + elif re.match(r'(?:-r|--reg-expr(?:ession)?)$', arg): + patternMode = 'r' + elif re.match(r'(?:-w|--wildcards?)$', arg): + patternMode = 'w' + elif re.match(r'(?:-s|--substring)$', arg): + patternMode = 's' + elif re.match(r'(?:-p|--path)$', arg): + targetMode = 'p' + elif re.match(r'(?:-f|--full(?:-name)?)$', arg): + targetMode = 'p' + elif re.match(r'(?:-p|--path)$', arg): + targetMode = 'p' + elif re.match(r'(?:-n|--node)$', arg): + targetMode = 'n' + elif arg.startswith('-'): + self._logger.error('unknown option: ' + arg) + else: + if self._fullNames == None: + self._fullNames = self._zip.namelist() + if patternMode == 'r': + pattern = base.StringUtils.regExprCompile(arg, 'pattern', self._logger) + for name in self._fullNames: + if targetMode == 'p': + item = os.path.dirname(name) + elif targetMode == 'n': + item = os.path.basename(name) + else: + item = name + if patternMode == 'f': + found = item == arg + elif patternMode == 'r': + found = pattern.search(item) + elif patternMode == 's': + found = item.find(arg) >= 0 + else: + found = fnmatch(item, arg) + if found: + self._extractStatistic._matches += 1 + doWrite = False + full = destination + name + if writeMode == 'o': + doWrite = True + elif writeMode == 'u': + try: + mtime = os.lstat(full).st_mtime + info = self._zip.getinfo(name) + ziptime = time.mktime((info.date_time[0], info.date_time[1], info.date_time[2], + info.date_time[3], info.date_time[4], info.date_time[5], 0, 0, self._dst)) + doWrite = ziptime < mtime + if doWrite: + self._extractStatistic._updates += 1 + except OSError: + doWrite = True + self._extractStatistic._creations += 1 + elif writeMode == 'f': + doWrite = os.path.exists(full) + if doWrite: + self._extractStatistic._refreshs += 1 + elif writeMode == 't': + doWrite = not os.path.exists(full) + else: + self._logger.error('I am confused: writeMode=' + writeMode) + if doWrite: + try: + self._zip.extract(name, destination) + except Exception as exc: + self._logger.error('cannot extract {:s}: {:s}'.format(full, str(exc)) ) + return rc + + def info(self, argv): + '''Displays info about a zip archive. + @param argv: the program arguments (behind the archive>) + @return: None: OK otherwise: the error message + ''' + rc = None + nameOnly = False + patternMode = 's' + targetMode = 'n' + for arg in argv: + if arg == '-1' or arg == '--name-only': + nameOnly = True + elif re.match(r'(?:-r|--reg-expr(?:ession)?)$', arg): + patternMode = 'r' + elif re.match(r'(?:-w|--wildcards?)$', arg): + patternMode = 'w' + elif re.match(r'(?:-s|--substring)$', arg): + patternMode = 's' + elif re.match(r'(?:-p|--path)$', arg): + targetMode = 'p' + elif re.match(r'(?:-f|--full(?:-name)?)$', arg): + targetMode = 'p' + elif re.match(r'(?:-p|--path)$', arg): + targetMode = 'p' + elif re.match(r'(?:-n|--node)$', arg): + targetMode = 'n' + elif arg.startswith('-'): + self._logger.error('unknown option: ' + arg) + else: + if self._fullNames == None: + self._fullNames = self._zip.namelist() + if patternMode == 'r': + pattern = base.StringUtils.regExprCompile(arg, 'r pattern', self._logger) + for name in self._fullNames: + if targetMode == 'p': + item = os.path.dirname(name) + elif targetMode == 'n': + item = os.path.basename(name) + else: + item = name + if patternMode == 'f': + found = item == arg + elif patternMode == 'r': + found = pattern.search(item) + elif patternMode == 's': + found = item.find(arg) >= 0 + else: + found = fnmatch(item, arg) + if found: + if nameOnly: + self._logger.log(name) + else: + info = self._zip.getinfo(name) + (year, month, day, hour, minute, second) = info.date_time + percent = 0 if info.file_size == 0 else int(info.compress_size * 100 / info.file_size) + self._logger.log('{:12d} {:3}% {:d}.{:02d}.{:02d}-{:02d}:{:02d}:{:02d} {:s}'.format( + info.file_size, percent, year, month, day, hour, minute, second, name)) + return rc + + def infosOf(self, node=None, path=None): + '''Returns a list of ZipInfo instances given by path and node name or patterns. + @param node: None or complete node name. + If a directory is given and filename is None: all files of the directory will be returned + string: the complete node name + compiled regular expression: all matching files will be returned, + e.g. re.compile('[.]txt$', re.I) + @param path: None of the complete path of the files to return + string: the complete path, ending with '/' + compiled regular expression: files of all matching directory will be returned, + e.g. re.compile('^(etc|home)/$') + @return: a list of ZipInfo instances. May be empty + ''' + rc = [] + allInfos = node == None and path == None + if not allInfos and self._fullNames == None: + self._fullNames = self._zip.namelist() + countNodes = len(self._nodes) + if allInfos: + rc = self._zip.infolist() + elif type(node) == str and type(path) ==str: + full = path + node + for current in self._fullNames: + if full == current: + info = self._zip.getinfo(full) + rc.append(info) + break + elif node == None and type(path) == str: + pathLength = len(path) + for current in self._fullNames: + if current.startswith(path) and pathLength == current.rfind('/') + 1: + info = self._zip.getinfo(current) + rc.append(info) + else: + for ix in range(len(self._fullNames)): + full = self._fullNames[ix] + pathCurrent = None + if ix < countNodes: + nodeCurrent = self._nodes[ix] + else: + ix2 = full.rfind('/') + if ix2 < 0: + nodeCurrent = full + pathCurrent = '' + else: + nodeCurrent = full[ix2+1:] + pathCurrent = full[0:ix2+1] + self._nodes.append(nodeCurrent) + if node == None: + found = True + elif type(node) == str: + found = node == nodeCurrent + else: + found = node.search(nodeCurrent) + if found: + if path == None: + found = True + elif type(path) == str: + if pathCurrent != None: + found = pathCurrent == path + else: + found = full.startswith(path) and len(path) + len(node) == len(full) + else: + if pathCurrent != None: + found = path.search(pathCurrent) + else: + if path.search(full) == None: + found = False + else: + pathCurrent = full[0:len(full)- len(nodeCurrent)] + found = path.search(pathCurrent) + if found: + info = self._zip.getinfo(full) + rc.append(info) + return rc + +class ZipExtractStatistic: + '''Holds the data of zip archive extraction. + ''' + def __init__(self): + self._updates = 0 + self._overwrites = 0 + self._refreshs = 0 + self._creations = 0 + self._matches = 0 + +class ZipCreateStatistic: + '''Holds the data of a zip archive creation. + ''' + def __init__(self): + self._links = 0 + self._directories = 0 + self._files = 0 + self._ignoredDirs = 0 + self._ignoredFiles = 0 + self._uncompressedFiles = 0 + self._uncompressedDirs = 0 + self._uncompressedRoots = 0 + self._uncompressedByExt = 0 + + def log(self, logger): + logger.log('file(s): {:d} dir(s): {:d} links: {:d}'.format(self._files, self._directories, self._links)) + logger.log('ignored file(s): {:d} ignored dir(s): {:d}'.format(self._ignoredFiles, self._ignoredDirs)) + logger.log('uncompressed file(s)/by extension: {:d}/{:d} uncompressed dir(s)/roots: {:d}/{:d}'.format( + self._uncompressedFiles, self._uncompressedByExt, self._uncompressedDirs, self._uncompressedRoots)) \ No newline at end of file diff --git a/base/__init__.py b/base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/doc/mysql.txt b/doc/mysql.txt new file mode 100644 index 0000000..2a78c99 --- /dev/null +++ b/doc/mysql.txt @@ -0,0 +1 @@ +apt-get install python3-mysqldb libmysqlclient-dev diff --git a/examples/safe/php.ini b/examples/safe/php.ini new file mode 100644 index 0000000..adf6ee3 --- /dev/null +++ b/examples/safe/php.ini @@ -0,0 +1,562 @@ +[PHP] + +;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;; + + + + + + +; ! boolean NOT + + + + + +;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;; + + + + +;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;; + + + + + +; log_errors + + + + + + + + + + +;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;; + + + +;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;; + +engine = On + +short_open_tag = Off + +precision = 14 + +; reasons. +; Off = Disabled +output_buffering = 4096 + +; is doing. + + + +zlib.output_compression = Off + + + +implicit_flush = Off + +unserialize_callback_func = + +; precision. +serialize_precision = -1 + +;open_basedir = + +disable_functions = pcntl_alarm,pcntl_fork,pcntl_waitpid,pcntl_wait,pcntl_wifexited,pcntl_wifstopped,pcntl_wifsignaled,pcntl_wifcontinued,pcntl_wexitstatus,pcntl_wtermsig,pcntl_wstopsig,pcntl_signal,pcntl_signal_get_handler,pcntl_signal_dispatch,pcntl_get_last_error,pcntl_strerror,pcntl_sigprocmask,pcntl_sigwaitinfo,pcntl_sigtimedwait,pcntl_exec,pcntl_getpriority,pcntl_setpriority,pcntl_async_signals, + +disable_classes = + + + + + +zend.enable_gc = On + + + +;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;; + +expose_php = Off + +;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;; + +max_execution_time=600 + +max_input_time=600 + + + +memory_limit=2048M + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Error handling and logging ; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; +; of PHP +; +error_reporting = E_ALL & ~E_DEPRECATED & ~E_STRICT + +display_errors = Off + +display_startup_errors = Off + +log_errors = On + +log_errors_max_len = 1024 + +ignore_repeated_errors = Off + +ignore_repeated_source = Off + +report_memleaks = On + + + + + +html_errors = On + + + + + +;error_log = php_errors.log + + + + + +;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;; + + + +variables_order = "GPCS" + +request_order = "GP" + +register_argc_argv = Off + +auto_globals_jit = On + + +post_max_size=512M + +auto_prepend_file = + +auto_append_file = + +; +default_mimetype = "text/html" + +default_charset = "UTF-8" + + + + +;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;; + +; +; + +doc_root = + +user_dir = + + + +enable_dl = Off + + +;cgi.nph = 1 + + + + + + +;cgi.rfc2616_headers = 0 + +;cgi.check_shebang_line=1 + +;;;;;;;;;;;;;;;; +; File Uploads ; +;;;;;;;;;;;;;;;; + +file_uploads = On + +; specified). + +upload_max_filesize=512M + +max_file_uploads=100 + +;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;; + +allow_url_fopen = On + +allow_url_include = Off + +;from="john@doe.com" + + +default_socket_timeout = 60 + + +;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;; + +; +; +; +; +; +; +; +; +; + + + +;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;; + +[CLI Server] +cli_server.color = On + +[Date] + + + + + +[filter] + + +[iconv] + + + +[imap] + +[intl] + +[sqlite3] + + +[Pcre] + + + +[Pdo] + + +[Pdo_mysql] +pdo_mysql.default_socket= + +[Phar] + + + +[mail function] +SMTP = localhost +smtp_port = 25 + + + + +mail.add_x_header = Off + +;mail.log = + +[ODBC] + + + + +odbc.allow_persistent = On + +odbc.check_persistent = On + +odbc.max_persistent = -1 + +odbc.max_links = -1 + +odbc.defaultlrl = 4096 + +odbc.defaultbinmode = 1 + +[Interbase] +ibase.allow_persistent = 1 + +ibase.max_persistent = -1 + +ibase.max_links = -1 + + + + + +ibase.timestampformat = "%Y-%m-%d %H:%M:%S" + +ibase.dateformat = "%Y-%m-%d" + +ibase.timeformat = "%H:%M:%S" + +[MySQLi] + +mysqli.max_persistent = -1 + + +mysqli.allow_persistent = On + +mysqli.max_links = -1 + +mysqli.default_port = 3306 + +mysqli.default_socket = + +mysqli.default_host = + +mysqli.default_user = + +mysqli.default_pw = + +mysqli.reconnect = Off + +[mysqlnd] +mysqlnd.collect_statistics = On + +mysqlnd.collect_memory_statistics = Off + +; file. + + + + + + + +[OCI8] + + + + + + + + + + +[PostgreSQL] +pgsql.allow_persistent = On + +pgsql.auto_reset_persistent = Off + +pgsql.max_persistent = -1 + +pgsql.max_links = -1 + +pgsql.ignore_notice = 0 + +pgsql.log_notice = 0 + +[bcmath] +bcmath.scale = 0 + +[browscap] + +[Session] +session.save_handler = files + +; +; +; +; +; +; +; + +session.use_strict_mode = 0 + +session.use_cookies = 1 + + +session.use_only_cookies = 1 + +session.name = PHPSESSID + +session.auto_start = 0 + +session.cookie_lifetime = 0 + +session.cookie_path = / + +session.cookie_domain = + +session.cookie_httponly = + +session.cookie_samesite = + +session.serialize_handler = php + +session.gc_probability = 0 + +session.gc_divisor = 1000 + +session.gc_maxlifetime = 1440 + + +session.referer_check = + +session.cache_limiter = nocache + +session.cache_expire = 180 + +session.use_trans_sid = 0 + +session.sid_length = 26 + +session.trans_sid_tags = "a=href,area=href,frame=src,form=" + + +session.sid_bits_per_character = 5 + + + + + + + + +[Assertion] +zend.assertions = -1 + + + + + + + +[COM] + + + + + +;com.code_page= + +[mbstring] + + + + + + + + + + + +[gd] + +[exif] + + + + + + +[Tidy] + +tidy.clean_output = Off + +[soap] +soap.wsdl_cache_enabled=1 + +soap.wsdl_cache_dir="/tmp" + +soap.wsdl_cache_ttl=86400 + +soap.wsdl_cache_limit = 5 + +[sysvshm] + +[ldap] +ldap.max_links = -1 + +[dba] + +[opcache] +;opcache.enable=1 + +;opcache.enable_cli=0 + + + + + + + + + + + +; passes + + + +; are cached. + + + +;opcache.error_log= + + + + + +; errors. +;opcache.mmap_base= + +;opcache.file_cache= + + + +;opcache.file_cache_fallback=1 + + + + + +[curl] + +[openssl] +;openssl.cafile= + + +; Local Variables: +; End: + + +xdebug.remote_port=9000 +xdebug.remote_enable=On +xdebug.remote_handler=dbgp +xdebug.remote_host=127.0.0.1 +xdebug.remote_autostart=1 diff --git a/net/EMail.py b/net/EMail.py new file mode 100644 index 0000000..9a89d10 --- /dev/null +++ b/net/EMail.py @@ -0,0 +1,130 @@ +''' +Created on 11.05.2018 + +@author: hm +''' +import smtplib +import re +import imghdr +import email.message + +import base.MemoryLogger + +class EMail: + + def __init__(self, subject, text=None, html=None): + '''Constructor. + @param subject: the subject of the mail + @param text: the ascii text of the mail + @param html: the html text of the mail + ''' + self._subject = subject + self._text = text + self._html = html + self._multiPart = email.message.EmailMessage() + if subject != None: + self._multiPart['Subject'] = subject + if text != None: + self._multiPart.set_content(text) + if html != None: + self._multiPart.add_alternative(html, subtype='html') + + def addImage(self, filename): + '''Adds a image file as attachement. + @param filename: this file will be appended, should be MIME type "image" + ''' + with open(filename, 'rb') as fp: + img_data = fp.read() + self._multiPart.add_attachment(img_data, maintype='image', subtype=imghdr.what(None, img_data)) + + def sendTo(self, sender, recipient, cc=None, bcc=None, logger = None): + ''' + Sends the email to the given recipients. + @param sender: None or an email address of the sender + @param recipient: the recipient or a list of recipients + @param cc: None or a carbon copy recipient or a list of CC + @param bcc: None or blind carbon copy recipient or a list of BCC + @param logger: None or logger + @return: dictionary: empty: success otherwise: e.g. { "three@three.org" : ( 550 ,"User unknown" ) } + ''' + def normEmail(email): + if type(email) == list: + rc = ', '.join(email) + elif email.find(' ') > 0: + rc = re.sub(r' +', ', ', email.strip()) + else: + rc = email + return rc + rc = [] + try: + server = smtplib.SMTP(self._host, self._port) + server.ehlo() + if self._withTls: + server.starttls() + server.ehlo() + server.login(self._username, self._code) + self._multiPart['From'] = sender + self._multiPart['To'] = normEmail(recipient) + if cc != None: + self._multiPart['Cc'] = normEmail(cc) + if bcc != None: + self._multiPart['BCC'] = normEmail(bcc) + rc = server.send_message(self._multiPart) + except Exception as exc: + if logger != None: + logger.error('sending email failed: {}\n{}\n{}'.format(str(exc), self._subject, self._text)) + return rc + + def setSmtpLogin(self, host, port, username, code, withTls = True): + '''Sets the login data for the SMTP server. + @param host: the SMTP server, e.g. 'smtp.gmx.net' + @param port: the port of the SMTP service + @param username: the user known to the SMTP service + @param code: the password of the SMTP service + @param withTls: True: TSL encryption will be used + ''' + self._host = host + self._port = port + self._username = username + self._code = code + self._withTls = withTls + +def sendStatusEmail(recipient, subject, body, sender, host, port, user, code, withTls, logger): + '''Sends an email. + @param receipient: the email address of the recipient or a blank separated list of recipients + @param subject: a short info + @param body: HTML or plain text + @param sender: the email address of the sender + @param host: the SMTP host + @param port: the SMTP port + @param user: the SMTP user + @param code: the SMTP password + @param withTls: True: TLS (encryption) is used + @param logger: for error logging + ''' + email = EMail(subject, body) + parts = recipient.split(' ') + email.setSmtpLogin(host, port, user, code, withTls) + cc = None if len(parts) < 2 else parts[1:] + rc = email.sendTo(sender, parts[0], cc, logger) + if len(rc) == 0: + logger.log('email sent to ' + recipient) + else: + logger.error('sending email "{}" failed: {}'.format(subject, str(rc))) + +def main(): + logger = base.MemoryLogger.MemoryLogger(1) + email = EMail('Testmail von Python', 'Hi!\nHat funktoniert', ''' +<html> +<body> +<h1>Hi</h1> +<p>Hat funktioniert!</p +</body> +</html> + ''') + # email.setSmtpLogin('smtp.1und1.de', 587, 'software@infeos.de', '4WebApp2018!') + email.setSmtpLogin('smtp.gmx.net', 587, 'hm.neutral@gmx.de', '1G.e.h.t.H.e.i.m', True) + email.sendTo('hamatoma@gmx.de', 'hamatoma@gmx.de', None, None, logger) + +if __name__ == '__main__': + main() diff --git a/net/FileTcpTaskHandler.py b/net/FileTcpTaskHandler.py new file mode 100644 index 0000000..29c0fc4 --- /dev/null +++ b/net/FileTcpTaskHandler.py @@ -0,0 +1,242 @@ +''' +Created on 21.04.2018 + +@author: hm +''' + +import os.path +import shutil +import time +import subprocess +import re +import net.TcpTaskHandler + +class FileTcpTaskHandler(net.TcpTaskHandler.TcpTaskHandler): + '''Handler for file operations. + For security reasons only files in predefined directories can be processed. + Most of the tasks return 'OK' if success or an error message + ''' + def __init__(self, name, baseDirectories, logger, blocksize = 8*1024*1024): + '''Constructor. + @param name: if more than once instances exists the client use this name to address the handler + @param baseDirectories: a list of base directories only files in these directories tree can be processed + @param logger: a logger, type ThreadLogger + @param blockSize: put and get will transfer blocks with this size + ''' + self._name = name + self._directories = baseDirectories + self._blocksize = blocksize + net.TcpTaskHandler.TcpTaskHandler.__init__(self, logger) + + def argumentError(self, message, serverThread, exception = None): + '''Handles an argument error, e.g. wrong number of arguments. + @param message: the error message + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + @param exception: None or the raised exception + ''' + info = '' + if exception != None and len(exception.args) > 1: + info = ' ' + exception.args[1] + serverThread.send('+++ "file": ' + message + info) + + def copy(self, args, serverThread): + '''Copies a file. + @param args: the command arguments: source and target + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + if len(args) < 2: + self.argumentError('copy: missing target', serverThread) + elif not self.validPath(args[0]) or not self.validPath(args[1]): + self.argumentError('copy: invalid path', serverThread) + else: + try: + shutil.copyfile(args[0], args[1], follow_symlinks=False) + serverThread.send('OK') + except OSError as exc: + self.argumentError('copy failed', serverThread, exc) + + def delete(self, args, serverThread): + '''Deletes a file. + @param args: the command arguments: source + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + if not self.validPath(args[0]): + self.argumentError('delete: invalid path', serverThread) + else: + try: + os.unlink(args[0]) + serverThread.send('OK') + except OSError as exc: + self.argumentError('delete failed', serverThread, exc) + + def diskFree(self, args, serverThread): + '''Gets the information of disk spaces. + @param args: the command arguments: not used + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + try: + with subprocess.Popen('/bin/df', stdout=subprocess.PIPE) as proc: + data = proc.stdout.read().decode() + lines = data.split('\n') + # /dev/mapper/ocz1t-home 32896880 15790076 15412700 51% /home + regExpr = re.compile(r'(\S+)\s+(\d+)\s+(\d+)\s+\d+\s+\d+%\s+(\S+)') + answer = 'OK\n' + for line in lines: + matcher = regExpr.match(line) + if matcher: + dev, total, used, mount = matcher.group(1), matcher.group(2), matcher.group(3), matcher.group(4) + if dev != 'tmpfs' and not mount.startswith('/run/'): + answer += '\t'.join([mount, total, used]) + '\n' + serverThread.send(answer) + except OSError as exc: + self.argumentError('delete failed', serverThread, exc) + + def fulfill(self, args, serverThread): + '''Fulfills the tasks 'file': (move, copy, delete, mkdir, rmdir, put, get). + @param args: the command arguments. args[0] contains the command + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + @return: True: request is handled False: request is unknown, next chain member should be asked + ''' + rc = True + if args[0] == 'file': + if len(args) < 3: + serverThread.send('+++ "file": missing arguments') + elif args[2] == 'df': + self.diskFree(args[3:], serverThread) + elif len(args) < 4: + serverThread.send('+++ "file": missing arguments') + elif args[1] != self._name: + rc = False + elif args[2] == 'move': + self.move(args[3:], serverThread) + elif args[2] == 'copy': + self.copy(args[3:], serverThread) + elif args[2] == 'delete': + self.delete(args[3:], serverThread) + elif args[2] == 'mkdir': + self.mkDir(args[3:], serverThread) + elif args[2] == 'get': + self.get(args[3:], serverThread) + elif args[2] == 'put': + self.put(args[3:], serverThread) + else: + rc = self.argumentError('unknown subcommand: ' + args[2], serverThread) + else: + rc = False + return rc + + def get(self, args, serverThread): + '''Copies a local file to the client. + Format of the answer: + 'OK' SIZE OFFSET LENGTH MTIME MODE OWNER GROUP '\n' + All numbers are hexadecimal. MTIME: microseconds + @param args: the command arguments: source and offset + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + if len(args) < 2: + self.argumentError('get: offset', serverThread) + elif not self.validPath(args[0]): + self.argumentError('get: invalid path', serverThread) + else: + try: + offset = int(args[1]) + info = os.stat(args[0]) + length = self._blocksize if info.st_size - offset > self._blocksize else info.st_size - offset + head = 'OK {:x} {:x} {:x} {:x} {:x} {:x} {:x}\n'.format(info.st_size, offset, length, int(info.st_mtime_ns / 1000), info.st_mode, info.st_uid, info.st_gid) + with open(args[0], 'rb') as fp: + fp.seek(offset) + data = head.encode() + fp.read(self._blocksize) + serverThread.sendBinary(data) + except Exception as exc: + self.argumentError('get: file not found', serverThread, exc) + pass + def mkDir(self, args, serverThread): + '''Creates a subdirectory. + @param args: the command arguments: source [mtime mode uid gid] + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + name = args[0] + if not self.validPath(name): + self.argumentError('mkdir: invalid path', serverThread) + else: + try: + mode = 0o777 if len(args) < 3 else int(args[2], 16) + os.makedirs(name, mode, True) + if len(args) >= 5: + atime = int(time.time() * 1000000000) + mtime = int(args[1], 16) * 1000 + os.utime(name, ns=(atime, mtime)) + os.chmod(name, mode) + os.setuid(name, int(args[3], 16)) + os.setuid(name, int(args[4], 16)) + serverThread.send('OK') + except OSError as exc: + self.argumentError('mkdir failed', serverThread, exc) + + def move(self, args, serverThread): + '''Moves a file. + @param args: the command arguments: source and target + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + if len(args) < 2: + self.argumentError('move: missing target', serverThread) + elif not self.validPath(args[0]) or not self.validPath(args[1]): + self.argumentError('move: invalid path', serverThread) + else: + try: + os.replace(args[0], args[1]) + serverThread.send('OK') + except OSError as exc: + self.argumentError('move failed', serverThread) + + def put(self, args, serverThread): + '''Receives a file from the client. + @param args: the command arguments: source SIZE OFFSET LENGTH MTIME MODE OWNER GROUP + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + name = args[0] + if len(args) < 8: + self.argumentError('put: missing group', serverThread) + elif not self.validPath(args[0]): + self.argumentError('put: invalid path', serverThread) + else: + try: + offset = args[2] + mode = 'wb' if offset == 0 else 'ab' + fp = open(name, mode) + if fp == None: + self.argumentError('put cannot open for writing: ' + name, serverThread) + else: + fp.write(serverThread._firstBlock) + while True: + data = serverThread.readBinary() + if not data: + break + else: + fp.write(data) + atime = int(time.time() * 1000000000) + mtime = int(args[4], 16) * 1000 + fileNo = fp.fileno() + os.fchmod(fileNo, int(args[5], 16)) + os.fchown(fileNo, int(args[6], 16), int(args[7], 16)) + fp.close() + os.utime(name, ns=(atime, mtime)) + serverThread.send('OK') + except Exception as exc: + self.argumentError('put failed', serverThread, exc) + + def validPath(self, path): + '''Tests whether the path is inside a directory tree specified in _directories[] + @param path: path to test + @return: True: path is valid False: invalid path + ''' + rc = False + path = os.path.normpath(path) + for aDir in self._directories: + if path.startswith(aDir): + rc = True + break + return rc + +if __name__ == '__main__': + pass \ No newline at end of file diff --git a/net/FtpEngine.py b/net/FtpEngine.py new file mode 100644 index 0000000..0920e12 --- /dev/null +++ b/net/FtpEngine.py @@ -0,0 +1,295 @@ +#! /usr/bin/python3 +''' +Created on 07.04.2018 + +@author: hm +''' +from ftplib import FTP +import re +import datetime +import stat +import os +import base.StringUtils + +class FtpEngine: + def __init__(self, host, port, user, passw, logger, verboseLevel = 1): + '''Constructor. + @param host: the server name, e.g. 'ftp.hamatoma.de' + @param port: the server port + @param user: the username for login + @param passw: the password for login + @param logger: the logger + @param verboseLevel: details of logging: 1: summary 2: directory 3: file + ''' + self.sep = '/' + self._logger = logger + self._ftp = FTP() + self._ftp.connect(host, port, 30) + self._ftp.login(user, passw) + self._printDepth = 0 + features = self._ftp.sendcmd('FEAT') + if features.find('MLST') < 0: + self._logger.error ('+++ ftp command MLSD not supported') + self._maxDepth = 2048 + self._utc = datetime.timezone.utc + self._verboseLevel = verboseLevel + def close(self): + '''Closes the FTP connection and frees the resouces. + ''' + self._ftp.close() + + def compare(self, ftpDir, localDir, depth): + '''Compares a local directory tree with a remote FTP directory tree. + @param ftpDir: the remote directory + @param localDir: the local directory + @param depth: the depth of the directory + @return: a tuple (total, fileCount, dirCount) + ''' + if depth == 0 and self._verboseLevel == 1 or self._verboseLevel >= 2: + self._logger.log('= {:s}:'.format(ftpDir)) + fileCount = 0 + dirCount = 1 + total = 0 + dirs = [] + sepLocal = '' if localDir == '/' else '/' + sepFtp = '' if ftpDir == '/' else '/' + localFiles = os.listdir(localDir) + for name, facts in self._ftp.mlsd(ftpDir, ['type', 'size', 'modify']): + if name == '.' or name == '..': + continue + try: + ix = localFiles.index(name) + except: + ix = -1 + if ix < 0: + self._logger.error('+++ missing local file: ' + localDir + sepLocal + name) + else : + del(localFiles[ix]) + aType = facts['type'] + if aType == 'dir': + if depth < self._maxDepth: + if name != '.' and name != '..': + dirs.append(name) + self.testFile(localDir + sepLocal + name, facts) + elif aType == 'file': + total += int(facts['size']) + fileCount += 1 + self.testFile(localDir + sepLocal + name, facts) + elif aType == 'OS.unix=symlink': + self.testFile(localDir + sepLocal + name, facts) + for name in localFiles: + self._logger.error('+++ missing ftp file: ' + localDir + sepLocal + name) + for node in dirs: + (diffTotal, diffFiles, diffDirs) = self.compare(ftpDir + sepFtp + node, localDir + sepLocal + node, depth + 1) + total += diffTotal + fileCount += diffFiles + dirCount += diffDirs + if depth < self._printDepth: + self._logger.log('{:12d} [{:6d} {:5d}] {:s}'.format(total, fileCount, dirCount, ftpDir)) + return (total, fileCount, dirCount) + + def diskUsage(self, directory, depth): + '''Calculate the space used by a given FTP directory. + This method uses the MLSD extension of the FTP server. + @param directory: name of the directory to inspect + @param depth: the depth of the directory relative to the start point + @return: a tuple (total, fileCount, dirCount) + ''' + if depth == 0 and self._verboseLevel == 1 or self._verboseLevel >= 2: + self._logger.log('= {:s}:'.format(directory)) + fileCount = 0 + dirCount = 1 + total = 0 + dirs = [] + infos = self._ftp.mlsd(directory, ['type', 'size']) + for name, facts in infos: + aType = facts['type'] + if aType == 'dir': + if depth < self._maxDepth: + if name != '.' and name != '..': + dirs.append(name) + pass + elif aType == 'file' or aType == 'OS.unix=symlink': + total += int(facts['size']) + fileCount += 1 + # print('dirs: ' + ' '.join(dirs)) + separator = '' if directory == '/' else '/' + for node in dirs: + (diffTotal, diffFiles, diffDirs) = self.diskUsage(directory + separator + node, depth + 1) + total += diffTotal + fileCount += diffFiles + dirCount += diffDirs + if depth < self._printDepth: + print('{:d} [{:d} {:d}] {:s}'.format(total, fileCount, dirCount, directory)) + return (total, fileCount, dirCount) + + def diskUsage2(self, directory, depth): + '''Calculate the space used by a given FTP directory. + This method uses the LIST command which is always available. + @param directory: name of the directory to inspect + @param depth: the depth of the directory relative to the start point + @return: a tuple (total, fileCount, dirCount) + ''' + if depth == 0 and self._verboseLevel == 1 or self._verboseLevel >= 2: + self._logger.log('= {:s}:'.format(directory)) + self._dirs += 1 + lines = [] + self._ftp.retrlines('LIST ' + directory, lines.append) + sizes = 0 + dirs = [] + for line in lines: + matcher = re.match('(\S+)\s+\d+\s+\d+\s+\d+\s+(\d+)\s+\S+\s+\S+\s+\S+\s+(.*)', line) + if matcher: + flags = matcher.group(1) + size = matcher.group(2) + name = matcher.group(3) + if flags[0] == 'd': + if depth < self._maxDepth: + if name != '.' and name != '..': + dirs.append(name) + elif flags[0] == '-': + sizes += int(size) + self._files += 1 + # print('dirs: ' + ' '.join(dirs)) + if directory == '/': + for node in dirs: + sizes += self.diskUsage2(directory + node, depth + 1) + else: + for node in dirs: + sizes += self.diskUsage2(directory + '/' + node, depth + 1) + return sizes + + def listFile(self, name, facts): + '''Lists one file. + @param name: the full filename + @param facts: the file's properties + ''' + dt = facts['modify'] + aType = facts['type'] + if aType == 'file': + size = int(facts['size']) + size = base.StringUtils.formatSize(size) + self._logger.log('{:10s} {}.{}.{}/{}:{}:{} {:s}'.format(size, dt[0:4], dt[4:6], dt[6:8], dt[8:10], dt[10:12], dt[12:14], name)) + elif aType == 'dir': + self._logger.log('{:10s} {}.{}.{}/{}:{}:{} {:s}'.format('<dir>', dt[0:4], dt[4:6], dt[6:8], dt[8:10], dt[10:12], dt[12:14], name)) + elif aType == 'OS.unix=symlink': + self._logger.log('{:10s} {}.{}.{}/{}:{}:{} {:s}'.format('<link>', dt[0:4], dt[4:6], dt[6:8], dt[8:10], dt[10:12], dt[12:14], name)) + else: + self._logger.log('{:10s} {}.{}.{}/{}:{}:{} {:s}'.format('<' + aType + '>', dt[0:4], dt[4:6], dt[6:8], dt[8:10], dt[10:12], dt[12:14], name)) + + def listTree(self, ftpDir, depth): + '''Compares a local directory tree with a remote FTP directory tree. + @param ftpDir: the remote directory + @param depth: the depth of the directory + @return: a tuple (total, fileCount, dirCount) + ''' + if depth == 0 and self._verboseLevel == 1 or self._verboseLevel >= 2: + self._logger.log('= {:s}:'.format(ftpDir)) + fileCount = 0 + dirCount = 1 + total = 0 + dirs = [] + sepFtp = '' if ftpDir == '/' else '/' + for name, facts in self._ftp.mlsd(ftpDir, ['type', 'size', 'modify']): + if name == '.' or name == '..': + continue + aType = facts['type'] + if aType == 'dir': + if depth < self._maxDepth: + if name != '.' and name != '..': + dirs.append(name) + self.listFile(ftpDir + sepFtp + name, facts) + elif aType == 'file': + total += int(facts['size']) + fileCount += 1 + self.listFile(ftpDir + sepFtp + name, facts) + elif aType == 'OS.unix=symlink': + self.listFile(ftpDir + sepFtp + name, facts) + for node in dirs: + (diffTotal, diffFiles, diffDirs) = self.listTree(ftpDir + sepFtp + node, depth + 1) + total += diffTotal + fileCount += diffFiles + dirCount += diffDirs + return (total, fileCount, dirCount) + + def removeTree(self, directory): + '''Deletes a directory with all its files and subdirectories. + @param directory: the directory to remove + @return a tuple (bytes, dirCount, fileCount) + ''' + self._logger.log('= {:s}:'.format(directory), 2) + fileCount = 0 + dirCount = 1 + total = 0 + dirs = [] + try: + infos = self._ftp.mlsd(directory, ['type', 'size']) + for name, facts in infos: + aType = facts['type'] + if aType == 'dir': + if name != '.' and name != '..': + dirs.append(name) + pass + elif aType == 'file' or aType == 'OS.unix=symlink': + full = directory + self.sep + name + try: + self._ftp.delete(full) + self._logger.log(full, 3) + except Exception as exc: + self._logger.error('cannot delete {:s}: {:s}'.format(full, str(exc))) + total += int(facts['size']) + fileCount += 1 + # print('dirs: ' + ' '.join(dirs)) + separator = '' if directory == self.sep else self.sep + for node in dirs: + full = directory + separator + node + (diffTotal, diffFiles, diffDirs) = self.removeTree(full) + total += diffTotal + fileCount += diffFiles + dirCount += diffDirs + except Exception as exc: + self._logger.error('removeTree() failed: ' + str(exc)) + if directory != '/': + try: + self._ftp.rmd(directory) + except Exception as exc: + self._logger.error('cannot delete directory {:s}: {:s}'.format(directory, str(exc))) + return (total, fileCount, dirCount) + + def testFile(self, localFile, facts): + '''Tests a local file against facts given from the FTP server + @param localFile: the name of the local file + @param facts: the data of the FTP file + ''' + self._logger.log(localFile, 3) + try: + statInfo = os.lstat(localFile) + if stat.S_ISDIR(statInfo.st_mode): + localType = 'dir' + elif stat.S_ISLNK(statInfo.st_mode): + localType = 'OS.unix=symlink' + elif stat.S_ISREG(statInfo.st_mode): + localType = 'file' + else: + localType = 'unknown'; + if localType != facts['type']: + self._logger.error('+++ different type: {:s} / {:s}: {:s}'.format(localType, facts['type'], localFile)) + else: + if localType != 'dir': + size = int(facts['size']) + if statInfo.st_size != size: + print('+++ different sizes: {:d} / {:d} '.format(statInfo.st_size, size) + localFile) + localDate = datetime.datetime.fromtimestamp(statInfo.st_mtime, self._utc) + localModified = localDate.strftime('%Y%m%d%H%M%S') + ftpModified = facts['modify'][0:14] + if localModified != ftpModified: + self._logger.error('+++ different modified date: {:s} / {:s} '.format(localModified, ftpModified)) + except OSError: + self._logger.error('+++ missing local: ' + localFile) + + def serverInfo(self): + '''Logs some FTP server info. + ''' + self._logger.log(self._ftp.getwelcome()) + self._logger.log('Features:') + self._logger.log(self._ftp.sendcmd('FEAT')) diff --git a/net/HttpClient.py b/net/HttpClient.py new file mode 100644 index 0000000..aed879f --- /dev/null +++ b/net/HttpClient.py @@ -0,0 +1,154 @@ +''' +Created on 17.05.2018 + +@author: hm +''' + +import http.client +import urllib.parse +import base.Logger +import ssl +import urllib3 + +class HttpClient: + '''Implements a HTTP or HTTPS client. + ''' + def __init__(self, verboseLevel, logger): + '''Constructor. + @param logger: the logger, type Logger + ''' + self._logger = logger + self._verboseLevel = verboseLevel + self._data = None + self._response = None + self._pool = urllib3.PoolManager() + + def close(self): + '''Frees the resources. + ''' + pass + + def _handleSingleRequest(self, url, method, timeout=10): + '''Handles a single HTTP request (not following relocations). + @param url: the URL of the website + @param method: the request method, e.g. 'HEAD' + ''' + self._data = None + self._response = None + try: + self._response = self._pool.request(method, url, timeout=timeout, retries=False) + if method == 'GET': + self._data = self._response.read() + if self._data == b'' and self._response.data != None: + self._data = self._response.data + except Exception as exc: + self._logger.error('error on processing [{}] {}: {}'.format(method, url, str(exc))) + # connection.close() + if self._verboseLevel > 1: + if self._response != None: + self._logger.debug('url: {} status: {} reason: {}'.format(url, self._response.status, self._response.reason)) + else: + self._logger.debug('url: {}: no response'.format(url)) + + def _handleSingleRequest2(self, url, method, timeout=10): + '''Handles a single HTTP request (not following relocations). + @param url: the URL of the website + @param method: the request method, e.g. 'HEAD' + ''' + parts = urllib.parse.urlparse(url) + netloc = parts.netloc + # reusing connection ("pipelining") is not supported by http.client + if url.startswith('https'): + context = ssl._create_unverified_context() + + connection = http.client.HTTPSConnection(netloc, timeout=timeout, context=context) + elif parts.scheme == 'http': + connection = http.client.HTTPConnection(netloc, timeout=timeout) + path = '/' if parts.path == '' else parts.path + self._data = None + if parts.query != '': + ix = url.find(path) + if ix > 0: + path = url[ix:] + self._response = None + try: + connection.request(method, path) + self._response = connection.getresponse() + if method == 'GET': + self._data = self._response.read() + except Exception as exc: + self._logger.error('error on processing [{}] {}: {}'.format(method, url, str(exc))) + connection.close() + if self._response != None: + self._logger.log('url: {} status: {} reason: {}'.format(url, self._response.status, self._response.reason), 2) + else: + self._logger.log('url: {}: no response'.format(url), 2) + + def getContent(self, url, relocationCount = 0): + '''Returns the header of a website. + @param url: the URL of the website + @param relocationCount: number of relocations to follow + @return: '' or the html content + ''' + self.handleRequest(url, 'GET', relocationCount) + data = self._data + return data + + def getHead(self, url, relocationCount = 5): + '''Returns the header of a website. + @param url: the URL of the website + @param relocationCount: number of relocations to follow + @return: '' or the content-type + ''' + self.handleRequest(url, 'HEAD', relocationCount) + rc = self.getHeaderField('content-type', '') + return rc + + def getHeaderField(self, field, defaultValue = None): + '''Gets the value of a header field. + @param field: name of the header field, e.g. 'content-length' + @return defaultValue: the field is unknown + otherwise: the value of the field + ''' + rc = defaultValue + for key in self._response.headers.keys(): + if key.lower() == field.lower(): + rc = self._response.headers[key] + break + return rc + + def getRealUrl(self, url): + '''Returns the first not relocated URL of a given URL. + @param url: URL to inspect + @return: <url>: url is not relocated otherwise: the first not relocated URL of a chain + ''' + self._handleSingleRequest(url, 'HEAD') + while self._response != None and self._response.status > 300 and self._response.status < 400: + url = self.getHeaderField('location', '') + self._handleSingleRequest(url, 'HEAD') + return url + + def handleRequest(self, url, method, relocationCount): + '''Handles a HTTP request. + @param url: the URL of the website + @param method: the request method, e.g. 'HEAD' + @param relocationCount: number of relocations to follow + @return: url of the end of the relocation chain + ''' + self._handleSingleRequest(url, method) + status = 499 if self._response == None else self._response.status + while relocationCount > 0 and (status >= 301 and status < 400): + relocationCount -= 1 + url = self.getHeaderField('location', '') + self._handleSingleRequest(url, method) + status = 499 if self._response == None else self._response.status + return url + +if __name__ == '__main__': + logger = base.Logger.Logger('/tmp/httpclient.log', True) + client = HttpClient(1, logger) + url = 'https://wiki.hamatoma.de' + data = client.getHead(url) + print('content-length of {}: {}'.format(url, data)) + client.close() + pass \ No newline at end of file diff --git a/net/LinuxTcpTaskHandler.py b/net/LinuxTcpTaskHandler.py new file mode 100644 index 0000000..1113450 --- /dev/null +++ b/net/LinuxTcpTaskHandler.py @@ -0,0 +1,176 @@ +''' +Created on 21.04.2018 + +@author: hm +''' + +import os.path +import net.TcpTaskHandler +import time +import base.LinuxUtils + +class LinuxTcpTaskHandler(net.TcpTaskHandler.TcpTaskHandler): + '''Handler for linux specific operations. + ''' + def __init__(self, logger): + '''Constructor. + @param logger: a logger, type ThreadLogger + ''' + net.TcpTaskHandler.TcpTaskHandler.__init__(self, logger) + + def argumentError(self, message, serverThread, exception = None): + '''Handles an argument error, e.g. wrong number of arguments. + @param message: the error message + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + @param exception: None or the raised exception + ''' + info = '' + if exception != None and len(exception.args) > 1: + info = ' ' + exception.args[1] + serverThread.send('+++ "file": ' + message + info) + + def fulfill(self, args, serverThread): + '''Fulfills the tasks 'file': (move, copy, delete, mkdir, rmdir, put, get). + @param args: the command arguments. args[0] contains the command + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + @return: True: request is handled False: request is unknown, next chain member should be asked + ''' + rc = True + if args[0] == 'diskfree': + infos = base.LinuxUtils.diskFree() + answer = 'OK\n' + for info in infos: + answer += base.StringUtils.join('\t', infos) + '\n' + serverThread.send(answer) + elif args[0] == 'users': + infos = base.LinuxUtils.users() + answer = 'OK\n' + for info in infos: + answer += base.StringUtils.join('\t', info) + '\n' + serverThread.send(answer) + elif args[0] == 'load': + info = base.LinuxUtils.load() + answer = 'OK\n' + base.StringUtils.join('\t', info) + elif args[0] == 'memoryinfo': + info = base.LinuxUtils.memoryInfo() + answer = 'OK\n' + base.StringUtils.join('\t', info) + serverThread.send(answer) + else: + rc = False + return rc + + def get(self, args, serverThread): + '''Copies a local file to the client. + Format of the answer: + 'OK' SIZE OFFSET LENGTH MTIME MODE OWNER GROUP '\n' + All numbers are hexadecimal. MTIME: microseconds + @param args: the command arguments: source and offset + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + if len(args) < 2: + self.argumentError('get: offset', serverThread) + elif not self.validPath(args[0]): + self.argumentError('get: invalid path', serverThread) + else: + try: + offset = int(args[1]) + info = os.stat(args[0]) + length = self._blocksize if info.st_size - offset > self._blocksize else info.st_size - offset + head = 'OK\n{:x} {:x} {:x} {:x} {:x} {:x} {:x}\n'.format(info.st_size, offset, length, int(info.st_mtime_ns / 1000), info.st_mode, info.st_uid, info.st_gid) + with open(args[0], 'rb') as fp: + fp.seek(offset) + data = head.encode() + fp.read(self._blocksize) + serverThread.sendBinary(data) + except Exception as exc: + self.argumentError('get: file not found', serverThread, exc) + pass + def mkDir(self, args, serverThread): + '''Creates a subdirectory. + @param args: the command arguments: source [mtime mode uid gid] + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + name = args[0] + if not self.validPath(name): + self.argumentError('mkdir: invalid path', serverThread) + else: + try: + mode = 0o777 if len(args) < 3 else int(args[2], 16) + os.makedirs(name, mode, True) + if len(args) >= 5: + atime = int(time.time() * 1000000000) + mtime = int(args[1], 16) * 1000 + os.utime(name, ns=(atime, mtime)) + os.chmod(name, mode) + os.setuid(name, int(args[3], 16)) + os.setuid(name, int(args[4], 16)) + serverThread.send('OK') + except OSError as exc: + self.argumentError('mkdir failed', serverThread, exc) + + def move(self, args, serverThread): + '''Moves a file. + @param args: the command arguments: source and target + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + if len(args) < 2: + self.argumentError('move: missing target', serverThread) + elif not self.validPath(args[0]) or not self.validPath(args[1]): + self.argumentError('move: invalid path', serverThread) + else: + try: + os.replace(args[0], args[1]) + serverThread.send('OK') + except OSError as exc: + self.argumentError('move failed', serverThread, exc) + + def put(self, args, serverThread): + '''Receives a file from the client. + @param args: the command arguments: source SIZE OFFSET LENGTH MTIME MODE OWNER GROUP + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + ''' + name = args[0] + if len(args) < 8: + self.argumentError('put: missing group', serverThread) + elif not self.validPath(args[0]): + self.argumentError('put: invalid path', serverThread) + else: + try: + offset = args[2] + mode = 'wb' if offset == 0 else 'ab' + fp = open(name, mode) + if fp == None: + self.argumentError('put cannot open for writing: ' + name, serverThread) + else: + fp.write(serverThread._firstBlock) + while True: + data = serverThread.readBinary() + if not data: + break + else: + fp.write(data) + atime = int(time.time() * 1000000000) + mtime = int(args[4], 16) * 1000 + fileNo = fp.fileno() + os.fchmod(fileNo, int(args[5], 16)) + os.fchown(fileNo, int(args[6], 16), int(args[7], 16)) + fp.close() + os.utime(name, ns=(atime, mtime)) + serverThread.send('OK') + except Exception as exc: + self.argumentError('put failed', serverThread, exc) + + def validPath(self, path): + '''Tests whether the path is inside a directory tree specified in _directories[] + @param path: path to test + @return: True: path is valid False: invalid path + ''' + rc = False + path = os.path.normpath(path) + for aDir in self._directories: + if path.startswith(aDir): + rc = True + break + return rc + +if __name__ == '__main__': + pass \ No newline at end of file diff --git a/net/TcpClient.py b/net/TcpClient.py new file mode 100644 index 0000000..8542562 --- /dev/null +++ b/net/TcpClient.py @@ -0,0 +1,82 @@ +''' +Created on 14.04.2018 + +@author: hm +''' +import socket +from base.Logger import Logger +import traceback +import base.StringUtils + +class TcpClient: + '''A client for communication with TcpServer. + ''' + def __init__(self, host, port, logger): + '''Constructor. + @param host: the host of the server + @param port: the port of the server + @param logger: the logger + ''' + self._host = host + self._port = port + self._logger = logger + self._verbose = True + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._socket.connect((self._host, self._port)) + + def send(self, command, args = None): + '''Sends a command with args. + @param command: command name + @param args: arguments as list + ''' + msg = command + if args != None: + msg += '\t' + base.StringUtils.join('\t', args) + msg += '\n' + try: + self._socket.sendall(msg.encode()) + except Exception as exc: + # + ' '.join(exc.args) + self._logger.error(exc.__class__.__name__ + ': ' + base.StringUtils.join(' ', exc.args)) + traceback.print_exc(3) + + def sendAndReceive(self, command, args = None): + '''Sends a command with args. + @param command: command name + @param args: arguments as list + @return: answer from the server + ''' + msg = command + if args != None: + for item in args: + msg += '\t' + str(item) + msg += '\n' + data = None + try: + data = msg.encode() + self._socket.sendall(data) + data = self._socket.recv(0x10000).decode() + except Exception as exc: + self._logger.error(exc.__class__.__name__ + ': ' + base.StringUtils.join(' ', exc.args)) + return data + + def close(self): + self._socket.close() + self._socket = None + +if __name__ == '__main__': + logger = Logger('/tmp/client.log', 1) + client = TcpClient('localhost', 58112, logger) + data = client.sendAndReceive('echo', ['A', 1, 'C']) + logger.log('echo answer: ' + str(data)) + data2 = client.sendAndReceive('time') + logger.log('time answer: ' + data2) + data = client.sendAndReceive('echo', ['B', 2, 'C']) + logger.log('echo answer: ' + str(data)) + client.send('bye') + client.send('stop') + client.close() + print('starting client2:') + client2 = TcpClient('localhost', 58112, logger) + client2.send('stop') + pass \ No newline at end of file diff --git a/net/TcpServer.py b/net/TcpServer.py new file mode 100644 index 0000000..15c52f4 --- /dev/null +++ b/net/TcpServer.py @@ -0,0 +1,151 @@ +''' +Created on 12.04.2018 + +@author: hm +''' + +import sys +import socket +import traceback +import threading +import base.ThreadLogger +from net.TcpTaskHandler import TcpTaskHandler + +class ServerThread(threading.Thread): + '''Handles a connection to one client in an own thread. + ''' + def __init__(self, threadId, ip, port, clientSocket, server, verboseLevel = 0): + '''Constructor. + @param threadId: a unique number of the thread + @param ip: the client address + @param port: the client port + @param socket: the client specific socket + @param server: the 'parent' of the thread, type TcpServer + @param verboseLevel: detail level of logging 0: none 1: summary 2: normal 3: often + ''' + self._id = threadId + self._ip = ip + self._verboseLevel = verboseLevel + self._port = port + self._server = server + self._socket = clientSocket + self._shouldStop = False + self._request = None + self._firstBlock = None + self._locker = threading.Lock() + server._logger.log('{:d}: thread started for {:s}:{:d}'.format(threadId, ip, port), 1) + threading.Thread.__init__(self, name='T{:03d}'.format(threadId)) + + def lock(self): + '''Locks the thread for exclusive access. + ''' + self._locker.acquire() + + def send(self, data): + '''Sends an answer to the client. + @param data: the data to send, type: string + ''' + self._socket.send(data.encode()) + + def sendBinary(self, data): + '''Sends an answer to the client. + @param data: the data to send, type: bytes + ''' + self._socket.send(data) + + def run(self): + '''Does the job of a thread. + Receives data from the client and gives them to the task chain. + ''' + self._server._logger.log('Starting thread {:d}'.format(self._id), 1) + separator = '\n'.encode() + self._server._handlerChain.fulfill(['threadstart'], self) + while not self._shouldStop : + try: + data = self._socket.recv(self._server._bufferSize) + ix = data.find(separator) + self._request = data[0:ix].decode() + if ix < len(data) - 1: + self._firstBlock = data[ix+1:] + if self._request == '': + self._server._logger.error('empty command. aborting...') + break + if self._request.find('\t') < 0: + args = [self._request] + else: + args = self._request.split('\t') + self._server._logger.log('Thread {}: received: {}'.format(self._id, base.StringUtils.limitItemLength(args, 40)), 3) + handler = self._server._handlerChain + while not handler.fulfill(args, self): + handler = handler._nextHandler + if handler == None: + self._server._logger.error('unknown request (no handler found) from {:s}:{:d} {}'.format(self._ip, self._port, self._request[0:50])) + break + except Exception as exc: + self._server._logger.error('exception {:s} raised:'.format(exc.__class__.__name__)) + traceback.print_exc(2) + raise exc + self._server._handlerChain.fulfill(['threadend'], self) + self._socket.close() + self._socket = None + self._server._logger.log('Thread {:d}: finished'.format(self._id), 2) + + def unlock(self): + '''Ends the locking. + ''' + self._locker.release() + +class TcpServer: + '''Implements a TCP server + ''' + def __init__(self, port, logger, handler, host='0.0.0.0', verboseLevel=0): + '''Constructor. + @param port: port for listening + @param logger: logging unit, type ThreadLogger + @param handler: the task handler, type TcpTaskHandler + @param host: listening host + @param pollInterval: number of seconds (as float) between two polls of shutdown requests + @param verboseLevel: detail level of logging 0: none 1: summary 2: normal 3: often + ''' + if sys.version_info[0] < 3: + logger.error('python version 3 expected, found: ' + str(sys.version_info[0])) + self._port = port + self._host = host + self._logger = logger + self._handlerChain = handler + self._verboseLevel = verboseLevel + self._connection = None + self._bufferSize = 64*1024 + self._shouldHalt = False + logger.info('listening on {:s}:{:d}'.format(host, port)) + self.listen() + + def listen(self): + tcpServer = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcpServer.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + tcpServer.bind((self._host, self._port)) + threads = [] + threadId = 0 + + while not self._shouldHalt: + threadId += 1 + tcpServer.listen(4) + self._logger.log('Multithreaded Python server : Waiting for connections from TCP clients...', 1) + (clientSocket, (ip, port)) = tcpServer.accept() + newthread = ServerThread(threadId, ip, port, clientSocket, self, self._verboseLevel) + newthread.start() + maxIx = len(threads) - 1 + for ix in range(maxIx + 1): + if not threads[maxIx - ix].is_alive(): + self._logger.log('removed from threads: {:d}'.format(threads[maxIx - ix]._id), 3) + del threads[maxIx - ix] + threads.append(newthread) + self._logger.log('waiting for finish of all ({:d}) threads ...'.format(len(threads)), 2) + for t in threads: + t.join() + tcpServer.close() + +if __name__ == '__main__': + logger = base.ThreadLogger.ThreadLogger('/tmp/TcpServer.log', True) + handler = TcpTaskHandler(logger) + server = TcpServer(58112, logger, handler) diff --git a/net/TcpTaskHandler.py b/net/TcpTaskHandler.py new file mode 100644 index 0000000..7141ca7 --- /dev/null +++ b/net/TcpTaskHandler.py @@ -0,0 +1,66 @@ +''' +Created on 22.04.2018 + +@author: hm +''' + +import datetime + +class TcpTaskHandler: + '''Implements a base class which "knows" one ore more commands sent to the server. + Derived classes must implement fulfill(self, args, serverThread). + fulfill() should use args[0] to identify the current command. + Writing an answer is done by using serverThread.send() + This base class knows the commands 'echo', 'time', 'stop' and 'bye' + Some task handlers can build a chain to handle many commands with different handlers: use setNextHandler() + ''' + def __init__(self, logger): + self._nextHandler = None + self._logger = logger + self._verboseLevel = 0 + + def fulfill(self, args, serverThread): + '''FulFills the tasks "echo", "time", "stop" and "bye". + @param args: the command arguments. args[0] contains the command + @param serverThread: the handler with type ServerThread or derived. Used for writing an answer to the client + @return: True: request is handled False: request is unknown, next chain member should be asked + ''' + rc = True + + if args[0] == 'time': + now = datetime.datetime.now() + serverThread.send( now.strftime('%Y.%m.%d %H:%M:%S %s')) + elif args[0] == 'echo': + serverThread.send("\t".join(args[1:])) + elif args[0] == 'stop': + serverThread._shouldStop = True + elif args[0] == 'bye': + serverThread._server._shouldHalt = True + else: + rc = False + return rc + + def setNextHandler(self, handler): + '''Sets the next handler in the handler chain. + @param handler: the next handler, derived from TcpBaseStreamHandler + ''' + if handler != None: + handler.setHandler(self._nextHandler) + self._nextHandler = handler + + def close(self): + '''Finishes the work: all resources will be freed. + ''' + pass + + def setVerboseLevel(self, level, recursive=True): + '''Sets the detail level of logging. + @param level: the new level + @param recursive: True: all handlers in the chain should set this level + ''' + self._verboseLevel = level + if recursive and self._nextHandler != None: + self._nextHandler.setVerboseLevel(level, True) + +if __name__ == '__main__': + pass \ No newline at end of file diff --git a/net/__init__.py b/net/__init__.py new file mode 100644 index 0000000..4bd6f9a --- /dev/null +++ b/net/__init__.py @@ -0,0 +1,5 @@ +''' +Created on 22.04.2018 + +@author: hm +''' diff --git a/unittest/PackageTest.py b/unittest/PackageTest.py new file mode 100644 index 0000000..8a09a25 --- /dev/null +++ b/unittest/PackageTest.py @@ -0,0 +1,25 @@ +''' +Created on 22.04.2018 + +@author: hm +''' + +class PackageTest: + def __init__(self, name): + self._name = name + self._modules = 0 + self._errors = 0 + self._asserts = 0 + + def run(self, clazz): + tester = clazz() + tester.run() + self._modules += 1 + self._errors += tester._errors + self._asserts += tester._asserts + + def _finish(self): + print('= {:s}: {:d} modules with {:d} asserts and {:d} error(s)'.format(self._name, self._modules, self._asserts, self._errors)) + +if __name__ == '__main__': + pass \ No newline at end of file diff --git a/unittest/UnitTestCase.py b/unittest/UnitTestCase.py new file mode 100644 index 0000000..f183f92 --- /dev/null +++ b/unittest/UnitTestCase.py @@ -0,0 +1,402 @@ +''' +Created on 12.04.2018 + +@author: hm +''' + +import re +import traceback +import os.path +import tempfile +import posix +import fnmatch + +import base.FileHelper +import base.BaseTool +import fnmatch + +class UnitTestAppInfo: + def __init__(self, usage): + self._usage = usage + +class UnitTestCase: + def __init__(self): + self._userId = posix.getuid() + self._isRoot = self._userId == 0 + self._inTestSuite = False + self._errors = 0 + self._name = type(self).__name__ + self._asserts = 0 + self._application = UnitTestAppInfo(self.usage) + base.BaseTool.baseToolUnitTestActive = True + self._logger = base.MemoryLogger.MemoryLogger(True) + self._silentLogger = base.MemoryLogger.MemoryLogger(False) + + def _describeDifference(self, str1, str2, prefix = ''): + '''Logs the difference of two strings. + @param str1: first string to compare + @param str2: second string to compare + @param prefix: '' or a prefix + @return: a string describing the difference + ''' + count1 = len(str1) + count2 = len(str2) + ixDiff = -1 + for ix in range(min(count1, count2)): + if str1[ix] != str2[ix]: + ixDiff = ix + break; + if ixDiff != -1: + rc = '+++ {:s}different at pos {:d}: {:s}/{:s}\n'.format(prefix, ixDiff + 1, str1[ixDiff:ixDiff+5], str2[ixDiff:ixDiff+5]) + elif count1 > count2: + rc = '+++ expected is longer:\n' + else: + rc = '+++ expected is shorter:\n' + rc += str1 + '\n' + str2 + if ixDiff != -1: + rc += '\n' + ('=' * ixDiff) + '^' + pass + return rc + + def assertDirExists(self, current): + '''Tests whether a given directory exists. + @param current: the directory to inspect + @returns: True: the directory exists + ''' + self._asserts += 1 + rc = True + if not os.path.exists(current): + rc = self.error('directory does not exist: ' + current) + elif not os.path.isdir(current): + rc = self.error('file {} exists but it is not a directory'.format(current)) + return rc + + def assertEquals(self, expected, current): + '''Tests whether two values are equal. + @param expected: the expected value + @param current: the value to test + @return: True: values are equal + ''' + rc = False + self._asserts += 1 + if type(expected) != type(current): + self.error('different types: ' + str(type(expected)) + ' / ' + str(type(current))) + else: + rc = expected == current + if not rc: + if type(expected) is str: + if expected.find('\n') < 0: + self.error(self._describeDifference(expected, current)) + else: + listExpected = expected.splitlines() + listCurrent = current.splitlines() + ixDiff = -1 + for ix in range(min(len(listExpected), len(listCurrent))): + if listExpected[ix] != listCurrent[ix]: + self.error(self._describeDifference(listExpected[ix], listCurrent[ix], 'in line {:d}: '.format(ix + 1))) + ixDiff = ix + break + if ixDiff == -1 and len(listExpected) != len(listCurrent): + if len(listExpected) < len(listCurrent): + msg = 'expected has fewer lines: {:d}/{:d}\n'.format(len(listExpected), len(listCurrent)) + listCurrent[len(listExpected)] + else: + msg = 'expected has more lines: {:d}/{:d}\n'.format(len(listExpected), len(listCurrent)) + listExpected[len(listCurrent)] + self.error(msg) + elif type(expected) is int: + self.error('different: {:d}/{:d} [0x{:x}/0x{:x}]'.format(expected, current, expected, current)) + else: + self.error('different: {:s} / {:s}'.format(str(expected), str(current))) + return rc + + def assertFalse(self, current): + '''Tests whether a value is False. + @param current: value to test + @returns: True: the value is False + ''' + rc = True + self._asserts += 1 + if current != False: + rc = self.error('+++ False expected, found: ' + str(current)) + return rc + + def assertFileContent(self, expectedFile, currentContent): + '''Compares a given string with an expected file content. + @param expectedFile: the name of the file with the expected content, e.g. 'data/abc.xyz' + @param currentContent: the content to compare + @returns: True: file content is the expected + ''' + rc = True + self._asserts += 1 + full = expectedFile if expectedFile.startswith(os.sep) else os.path.dirname(__file__) + os.sep + expectedFile + if not os.path.exists(full): + rc =self.error('missing file: ' + expectedFile) + else: + with open(full, 'r') as fp: + expected = fp.read() + rc = self.assertEquals(expected, currentContent) + if not rc: + tempFile = self.tempFile(os.path.basename(expectedFile)) + with open(tempFile, "w") as fp: + fp.write(currentContent) + self.log('meld {} {}'.format(tempFile, full)) + return rc + + def assertFileContains(self, expected, currentFile): + '''Tests whether a given file contains a given content. + @param expected: content to search. May be a string or a re.RegExpr instance + @param currentFile: file to inspect + @returns: True: the file contains the expected string + ''' + rc = True + self._asserts += 1 + if not os.path.isfile(currentFile): + rc = self.error('missing file ' + currentFile) + else: + found = False + with open(currentFile) as fp: + # lineNo = 0 + for line in fp: + #lineNo += 1 + #if lineNo == 126: + # lineNo = 126 + if type(expected) == str: + if line.find(expected) >= 0: + found = True + break + else: + if expected.search(line) != None: + found = True + break + if not found: + text = expected if type(expected) == str else expected.pattern + rc = self.error('missing content {:s} in {:s}'.format(text[0:20], currentFile)) + return rc + + def assertFileExists(self, filename): + '''Tests whether a given file exists. + @param filename: the name of the file to test + @returns: True: the file exists + ''' + rc = True + self._asserts += 1 + if not os.path.exists(filename): + rc = self.error('file does not exist: ' + filename) + return rc + + def assertFileNotContains(self, unexpectedContent, currentFile): + '''Tests whether a given file contains not a given content. + @param unexpectedContent: content to search + @param currentFile: file to inspect + ''' + rc = True + self._asserts += 1 + if not os.path.isfile(currentFile): + rc = self.error('missing file ' + currentFile) + else: + found = False + with open(currentFile) as fp: + for line in fp: + if line.find(unexpectedContent) >= 0: + found = True + break + if found: + rc = self.error('unexpected content {:s} in {:s}'.format(unexpectedContent[0:20], currentFile)) + return rc + + def assertFileNotExists(self, filename): + '''Tests whether a given file does not exist. + @param filename: the name of the file to test + @returns: True: the file does not exist + ''' + rc = True + self._asserts += 1 + if os.path.exists(filename): + rc = self.error('file exists: ' + filename) + return rc + + def assertMatches(self, expectedRegExpr, current, flags=0): + '''Tests whether a string matches a given regular expression. + @param expectedRegExpr: regular expression + @param current: string to test + @param flags: flags for re.match, e.g. re.IGNORECASE or re.MULTILINE + @returns: True: the value matches the expected + ''' + rc = True + self._asserts += 1 + if not re.search(expectedRegExpr, current, flags): + rc = self.error('+++ does not match\n' + expectedRegExpr + '\n' + current) + return rc + + def assertNodeExists(self, path, nodePattern): + '''Tests whether at least one file exists in a given path with a given node pattern. + @param path: the directory to inspect + @param nodePattern: a pattern with unix wildcards to inspect + @return True: node found + ''' + nodes = os.listdir(path) + found = False + for node in nodes: + if fnmatch.fnmatch(node, nodePattern): + found = True + break + if not found: + self.error('node {} not found in {}'.format(nodePattern, path)) + return found + + def assertNone(self, current): + '''Tests whether a value is None. + @param current: value to test + @returns: True: the value is None + ''' + rc = True + self._asserts += 1 + if current != None: + rc = self.error('+++ None expected, found: ' + str(current)) + return rc + + def assertNotNone(self, current): + '''Tests whether a value is not None. + @param current: value to test + @returns: True: the value is not None + ''' + self._asserts += 1 + if current == None: + self.error('+++ unexpected None found') + + def assertTrue(self, current): + '''Tests whether a value is True. + @param current: value to test + @returns: True: the value is True + ''' + rc = True + self._asserts += 1 + if current != True: + rc = self.error('+++ True expected, found: ' + str(current)) + return rc + + def clearDirectory(self, path): + '''Removes all files and subdirs in a given directory. + @param path: name of the directory + ''' + base.FileHelper.clearDirectory(path, self._logger) + + def ensureDirectory(self, directory): + '''Ensures that the given directory exists. + @param directory: the complete name + @return: None: could not create the directory + otherwise: the directory's name + ''' + rc = base.FileHelper.ensureDirectory(directory, self._logger) + return rc + + def ensureFileDoesNotExist(self, filename): + '''Ensures that a file does not exist. + @param filename: the file to delete if it exists. + ''' + base.FileHelper.ensureFileDoesNotExist(filename, self._logger) + + def error(self, message): + '''Displays an error with backtrace. + @param message: error message + @returns: False + ''' + self._errors += 1 + print(message) + info = traceback.extract_stack() + # ignore runtime methods: + while len(info) > 1 and (info[0].filename.find('/pysrc/') > 0 or info[0]._line.startswith('tester.run()') + or info[0].filename.find('UnitTestCase.py') > 0 and info[0]._line.startswith('method()')): + del info[0] + # ignore UnitTest internals: + length = len(info) + while length > 1 and info[length - 1].filename.find('UnitTestCase.py') > 0: + del info[length - 1] + length -= 1 + for entry in info: + print('{:s}:{:d} {:s}'.format(entry.filename, entry.lineno, entry.line)) + return False + + def log(self, message): + '''Displays a message + @param message: message + ''' + print(message) + + def getSummary(self): + '''Return the summary message. + @return the summary message + ''' + return self._summary + + def setInTestSuite(self, value): + '''Sets the attribute. + @param value: the new value of _inTestSuite + ''' + self._inTestSuite = value + + def run(self): + '''Searches the methods starting with 'test' and call them. + ''' + for item in self.__dir__(): + if item.startswith('test'): + method = getattr(self, item) + print('= ' + item) + method() + self._summary = '=== unit {:s}: {:d} assert(s) with {:d} error(s)'.format(self._name, self._asserts, self._errors) + print(self._summary) + if hasattr(self, '_finish'): + self._finish() + + def tempDir(self, node, subdir = None): + '''Builds the name of a directory and ensures that the directory exists. + @param node: the directory's name (without path) + @param subdir: None or the name of a directory inside the temp dir + @return: None: cannot create directory + otherwise: the name of an existing temporary directory (with path) + ''' + rc = tempfile.gettempdir() + if subdir != None: + rc += os.sep + subdir + if node != None: + if rc[-1] != os.sep: + rc += os.sep + rc += node + rc = self.ensureDirectory(rc) + return rc + + def tempFile(self, node, subdir = None, subdir2 = None): + '''Builds the name of a temporary file and ensures that the parent directories exist. + @param node: the file's name (without path) + @param subdir: None or the name of a directory inside the temp dir + @param subdir2: None or the name of a directory inside subdir + @return: the name of a temporary file (with path), e.g. /tmp/subdir/subdir2/node + ''' + rc = tempfile.gettempdir() + os.sep + if subdir != None: + rc += subdir + rc = self.ensureDirectory(rc) + if subdir[-1] != os.sep: + rc += os.sep + if subdir2 != None: + rc += subdir2 + self.ensureDirectory(rc) + if subdir2[-1] != os.sep: + rc += os.sep + rc += node + return rc + + def usage(self, message): + print(message) + self.assertFalse(True) + + def xtestMyself(self): + self.assertEquals(3, 4) + self.assertEquals('Hello', 'Hallo') + self.assertEquals('abc\nhallo', 'abc\nhello') + self.assertNone('not none') + self.assertNotNone(None) + +if __name__ == '__main__': + tester = UnitTestCase() + tester.run() + pass \ No newline at end of file diff --git a/unittest/UnitTestSuite.py b/unittest/UnitTestSuite.py new file mode 100644 index 0000000..9f3819b --- /dev/null +++ b/unittest/UnitTestSuite.py @@ -0,0 +1,116 @@ +''' +Created on 22.04.2018 + +@author: hm +''' + +import os.path +import re +import sys +import importlib + +sys.path.insert(0, '/usr/share/pyrshell') + +class UnitTestSuite: + '''Tests a group of test cases. + ''' + def __init__(self, name): + '''Constructor. + @param name: name of the suite (for logging) + ''' + self._name = name + self._testCases = [] + self._imports = [] + self._base = '/home/ws/py/pyrshell/' + self._summary = [] + + def addByPattern(self, relPath, pattern = r'.*[.]py$'): + '''Adds the test cases given by a directory and a filename pattern (of modules, not test cases). + @param relPath: the directory containing the modules to test, relative to the parent of 'unittest' + @param pattern: a regular expression for selecting the modules + ''' + basePath = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + path = basePath + os.sep + relPath + files = os.listdir(path) + regExpr = re.compile(pattern) + for node in files: + if regExpr.match(node): + self.addFromFile(relPath, node) + + def addFromFile(self, package, node): + '''Adds a test case given by the name of the module. + @param package: the package name of the module + @param node: the file containing the module + ''' + moduleName = node[0:-3] + 'Test' + #moduleWithPackage = package + '.' + moduleName + #self._imports.append([moduleName, moduleWithPackage]) + full = self._base + 'unittest/' + package + '/' + moduleName + '.py' + if os.path.exists(full): + self._imports.append([moduleName, package]) + self._testCases.append(moduleName) + + def addList(self, testCases): + '''Adds a list of test cases for inspecting. + @param testCases: a list of class names + ''' + for item in testCases: + if item not in self._testCases: + self._testCases.append(item) + + def instantiate(self, clazz ): + '''Instantiate a class object given by name + @param clazz: the classes name + @return the instance + ''' + parts = clazz.split('.') + moduleName = ".".join(parts[:-1]) + className = parts[-1] + module = importlib.import_module(moduleName) + instance = getattr(module, className) + return instance + + def process(self): + '''Instantiate the classes collected in _imports and call the class.run() method. + ''' + tests = [] + for name, package in self._imports: + if not name.startswith('__'): + if name.find('ToolTest') > 0: + package = 'appl' + instance = self.instantiate('unittest.' + package + '.' + name + '.' + name)() + clazz = self.instantiate('unittest.' + package + '.' + name + '.' + name) + tests.append((clazz, instance)) + for (clazz, instance) in tests: + clazz.__init__(instance) + clazz.setInTestSuite(instance, True) + clazz.run(instance) + self._summary.append(clazz.getSummary(instance)) + def summary(self): + print('=== Summary ===') + errors = '' + countErrors = 0 + asserts = 0 + units = 0 + for item in self._summary: + print(item) + units += 1 + # 0...1....2...............3..4.........5....6 + # === unit LinuxUtilsTest: 84 assert(s) with 0 error(s) + parts = item.split() + asserts += int(parts[3]) + countErrors += int(parts[6]) + if parts[6] != '0': + errors += '{}: {} '.format(parts[2][0:-1], parts[6]) + if errors != '': + print ('=== {} units with {} assert(s) and {} error(s) in:\n{}'.format( + units, asserts, countErrors, errors)) + +if __name__ == '__main__': + paths = sys.path + suite = UnitTestSuite('base') + suite.addByPattern('base') + suite.addByPattern('net') + suite.addByPattern('appl') + suite.process() + suite.summary() \ No newline at end of file diff --git a/unittest/__init__.py b/unittest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/unittest/appl/BackupToolTest.py b/unittest/appl/BackupToolTest.py new file mode 100644 index 0000000..e7e2279 --- /dev/null +++ b/unittest/appl/BackupToolTest.py @@ -0,0 +1,508 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import os.path +import time +import datetime +import shutil + +import base.MemoryLogger +import base.StringUtils +import appl.BackupTool +import appl.DirTool +import appl.TextTool + +debug = False + +def usage(msg=None): + return 'test usage' + +class BackupToolTest(UnitTestCase): + + def __init__(self): + UnitTestCase.__init__(self) + config = base.JavaConfig.JavaConfig('/etc/pyrshell/dbtool.conf', self._logger) + self._admin = config.getString('admin.user', 'root') + self._adminCode = config.getString('admin.code', '') + self._textTool = appl.TextTool.TextTool(self.getOptions('TextTool')) + + def findBackupStorage(self, path): + rc = None + nodes = os.listdir(path) + for node in nodes: + full = path + os.sep + node + if os.path.isdir(full): + rc = full + return rc + + def getOptions(self, application='BackupTool'): + appInfo = base.BaseTool.ApplicationInfo(application.lower(), 'appl.' + application, usage) + rc = base.BaseTool.GlobalOptions(appInfo) + rc._logFiles.append('-') + return rc + + def makeBackupConfig(self, clear=True): + baseDir = self.tempDir('config', 'unittest') + if clear: + self.clearDirectory(baseDir) + targetDir = self.makeTargetDir() + appDataDir = self.tempDir('blog', 'unittest') + base.StringUtils.toFile(appDataDir + os.sep + 'index.php', '<?php\ninfo();') + base.StringUtils.toFile(appDataDir + os.sep + 'new.php', '<?php\nnew();') + oldFile = appDataDir + os.sep + 'old.php' + base.StringUtils.toFile(oldFile, '<?php\nold();') + base.FileHelper.setModified(oldFile, None, datetime.datetime(2018,1,2,3,4,5)) + + appDir = self.ensureDirectory(baseDir + os.sep + 'webapps.d') + dirDir = self.ensureDirectory(baseDir + os.sep + 'dirs.d') + base.StringUtils.toFile(appDir + os.sep + 'testdb.conf', ''' +db=backuptooltest +user={} +password={} +sql.file=testdb +directory={:s} +'''.format(self._admin, self._adminCode, appDataDir)) + base.StringUtils.toFile(baseDir + os.sep + 'backup.conf', ''' +target.path={:s} +job.dayly=job.sysinfo job.sys job.db job.minimal.1 job.minimal.2 job.gitlab.backup +job.weekly=job.sysinfo job.sys job.db.all job.full +job.monthly=job.sysinfo job.sys job.full +job.sys=&saveDirByTar @%etc%/dirs.d/sys.*.conf:directory:excluded +job.full=&saveDirByZip @%etc%/dirs.d/std.*.conf:directory +job.db=&saveMysql @%etc%/webapps.d/*.conf:db:user:password:sql.file +job.minimal.1=&saveDirByZipLatest 7 @%etc%/webapps.d/*.conf:directory +job.minimal.2=&saveDirByZipLatest 7 @%etc%/dirs.d/std.*.conf:directory +job.save.clean=&deleteFile @%etc%/webapps.d/*.conf:directory:sql.file +job.db.all=&saveAllDatabases +job.gitlab.backup=&gitlabBackup +job.sysinfo=&systemInfo +# Reading metadata from mysql: +mysql.user={} +mysql.code={} +# Email data: +# Receiver of the error messages: may be a blank separated list of email addresses +send.always=False +location=unittest +admin.email=hm.neutral@gmx.de +smtp.host=smtp.gmx.de +smtp.port=587 +smtp.sender=hm.unittest@gmx.de +smtp.user=hm.unittest@gmx.de +smtp.code=33S.e.c.r.e.t +smtp.tls=True +'''.format(targetDir, self._admin, self._adminCode)) + appl.BackupTool.BackupTool.createDirsD(dirDir, None, False) + return baseDir + + def makeConfigSet(self): + baseDir = self.tempDir('configset', 'unittest') + base.StringUtils.toFile(baseDir + os.sep + 'example1.conf', 'db=adam\nuser=alice') + base.StringUtils.toFile(baseDir + os.sep + 'example2.conf', 'db=bob\nuser=berta') + base.StringUtils.toFile(baseDir + os.sep + 'bad.conf~', 'db=charly\nuser=carol') + return baseDir + + def makeDirDirectSync(self): + baseDir = self.tempDir('rsyncdirect', 'unittest') + syncTarget = self.tempDir('trgdir', 'unittest') + syncSource = self.tempDir('syncsrc', 'unittest') + base.StringUtils.toFile(baseDir + os.sep + 'backup.conf', ''' +target.path={:s} +job.often=&synchronize --delete {:s} {:s} +'''.format(syncTarget, syncSource, syncTarget)) + srcFile = syncSource + os.sep + 'file.sync.txt' + trgFile = syncTarget + os.sep + 'trg.only.txt' + base.StringUtils.toFile(srcFile, 'src') + base.StringUtils.toFile(trgFile, 'trg') + return (baseDir, srcFile, trgFile) + + def makeDirToSync(self): + srcDir = self.tempDir('srcsync', 'unittest') + trgDir = self.tempDir('trgsync', 'unittest') + shutil.rmtree(trgDir, False) + srcSub = srcDir + os.sep + 'subdir' + self.ensureDirectory(srcSub) + trgSub = trgDir + os.sep + 'subdir' + self.ensureDirectory(trgSub) + for ix in range(1, 3+1): + full = srcDir + os.sep + 'src_only{:d}.txt'.format(ix) + base.StringUtils.toFile(full, 'src{:d}: {:s}'.format(ix, full)) + full = trgDir + os.sep + 'trg_only{:d}.txt'.format(ix) + base.StringUtils.toFile(full, 'trg{:d}: {:s}'.format(ix, full)) + full = srcSub + os.sep + 'src_only{:d}.txt'.format(ix) + base.StringUtils.toFile(full, 'src{:d}: {:s}'.format(ix, full)) + full = trgSub + os.sep + 'trg_only{:d}.txt'.format(ix) + base.StringUtils.toFile(full, 'trg{:d}: {:s}'.format(ix, full)) + for ix in range(1, 2+1): + full = srcDir + os.sep + 'src_younger{:d}.txt'.format(ix) + base.StringUtils.toFile(full, 'src{:d}: {:s}'.format(ix, full)) + time.sleep(1) + full = trgDir + os.sep + 'src_younger{:d}.txt'.format(ix) + base.StringUtils.toFile(full, 'trg{:d}: {:s}'.format(ix, full)) + for ix in range(1, 2+1): + full = trgDir + os.sep + 'src_older{:d}.txt'.format(ix) + base.StringUtils.toFile(full, 'trg{:d}: {:s}'.format(ix, full)) + time.sleep(1) + full = srcDir + os.sep + 'src_older{:d}.txt'.format(ix) + base.StringUtils.toFile(full, 'src{:d}: {:s}'.format(ix, full)) + return (srcDir, trgDir) + + def makeDirIndirectSync(self): + baseDir = self.tempDir('syncindirect', 'unittest') + syncTarget = self.tempDir('trgdir', 'unittest') + syncSource = self.tempDir('sync2src', 'unittest') + dirDir = baseDir + os.sep + 'dirs.d' + self.ensureDirectory(dirDir) + base.StringUtils.toFile(dirDir + os.sep + 'sync.test.conf', ''' +source={:s} +target={:s} +delete=True +'''.format(syncSource, syncTarget)) + base.StringUtils.toFile(baseDir + os.sep + 'backup.conf', ''' +target.path={:s} +job.often=&synchronize @%etc%/dirs.d/sync.*.conf:source:target:delete +'''.format(syncTarget, syncSource, syncTarget)) + srcFile = syncSource + os.sep + 'file.sync.txt' + trgFile = syncTarget + os.sep + 'trg.only.txt' + base.StringUtils.toFile(srcFile, 'src') + base.StringUtils.toFile(trgFile, 'trg') + return (baseDir, srcFile, trgFile) + + def makeMinimalConfigDir(self): + dirName = self.tempFile('pyrshell', 'etc') + os.makedirs(dirName, 0o777, True) + base.StringUtils.toFile(dirName + os.sep + 'backup.conf', '') + return dirName + + def makeTree(self, baseDir = None): + baseDir = self.tempFile('unittest') if baseDir == None else baseDir + text = 'Dies ist ein langer Text, der hoffentlich ein wenig komprimiert werden kann. Wenn nicht, kann man nichts machen' * 1000 + dirName = baseDir + os.sep + 'srcdir' + os.makedirs(dirName, 0o777, True) + # /tmp/unittest/srcdir: file[1-5].txt dir[123] .git linkConfig + # /tmp/unittest/srcdir/dirX: levelX.txt subdir1 + # /tmp/unittest/srcdir/dirX/subdir1: a.jpg b.txt c.odt + # /tmp/unittest/srcdir/.git: config aa bb + # /tmp/unittest/srcdir/.git/xx: xx1234 xx2345 xx3456 + for ix in range(1, 5+1): + fn = dirName + os.sep + 'file{:d}.txt'.format(ix) + base.StringUtils.toFile(fn, 'no{:d}: {:s}'.format(ix, text)) + for ix in range(1, 3+1): + subdir = dirName + os.sep + 'dir' + str(ix) + os.makedirs(subdir, 0o777, True) + fn = subdir + os.sep + 'level{:d}'.format(ix) + base.StringUtils.toFile(fn, 'level{:d}: {:s}'.format(ix, text)) + dir2 = subdir + os.sep + 'subdir1' + os.makedirs(dir2, 0o777, True) + for node in ['a.jpg', 'b.txt', 'c.odt']: + fn = dir2 + os.sep + node + base.StringUtils.toFile(fn, node + ': ' + text) + return dirName + + def makeTargetDir(self): + targetDir = self.tempDir('trgdir', 'unittest') + self.clearDirectory(targetDir) + return targetDir + + def testEnsureDir(self): + if debug: + return + options = self.getOptions() + options._configDir = self.makeMinimalConfigDir() + baseDir = self.tempFile('tmp111') + subdir = baseDir + os.sep + 'tmp222' + if os.path.isdir(subdir): + os.rmdir(subdir) + if os.path.isdir(baseDir): + os.rmdir(baseDir) + self.assertFalse(os.path.isdir(baseDir)) + tool = appl.BackupTool.BackupTool(options) + tool.ensureDirectory(subdir) + self.assertTrue(os.path.isdir(subdir)) + self.assertEquals(0, tool._logger._errors) + + def testEnsureFileNotExist(self): + if debug: + return + fn = self.tempFile('file', 'unittest') + if os.path.exists(fn): + os.unlink(fn) + options = self.getOptions() + options._configDir = self.makeMinimalConfigDir() + tool = appl.BackupTool.BackupTool(options) + tool.ensureFileDoesNotExist(fn) + base.StringUtils.toFile(fn, 'Hi') + self.assertTrue(os.path.exists(fn)) + tool.ensureFileDoesNotExist(fn) + self.assertFalse(os.path.exists(fn)) + self.assertEquals(0, tool._logger._errors) + + def testUsage(self): + if debug: + return + appl.BackupTool.main(['backuptool', '-q']) + + def testSaveDirByTar(self): + if debug: + return + source = self.makeTree() + target = self.tempDir('trgdir', 'unittest') + name = 'test' + archive = target + os.sep + name + '.tgz' + if os.path.exists(archive): + os.unlink(archive) + options = self.getOptions() + options._configDir = self.makeMinimalConfigDir() + tool = appl.BackupTool.BackupTool(options) + tool.saveDirectoryByTar(name, source, target, None) + self.assertTrue(os.path.exists(archive)) + self.assertEquals(0, tool._logger._errors) + + def testSaveDirByZip(self): + if debug: + return + source = self.makeTree() + target = self.tempDir('trgdir', 'unittest') + name = 'test' + archive = target + os.sep + name + '.zip' + if os.path.exists(archive): + os.unlink(archive) + options = self.getOptions() + options._configDir = self.makeMinimalConfigDir() + tool = appl.BackupTool.BackupTool(options) + tool.saveDirectoryByZip(name, source, target, None) + self.assertTrue(os.path.exists(archive)) + self.assertEquals(0, tool._logger._errors) + + def testLogFile(self): + options = self.getOptions() + options._configDir = self.makeMinimalConfigDir() + tool = appl.BackupTool.BackupTool(options) + fn = self.tempFile('example.txt', 'unittest') + base.StringUtils.toFile(fn, 'Hi') + ftime = time.mktime((2018,1,2,3,4,5,0,0,0)) + os.utime(fn, (ftime, ftime)) + self.assertEquals('f: /tmp/unittest/example.txt s: 2 Byte t: 2018.01.02 03:04:05 r: 0 sec', tool.logFile(fn, 'f: %f s: %s t: %t r: %r', time.time())) + + def testConfigurationSet(self): + if debug: + return + configDir = self.makeConfigSet() + logger = base.MemoryLogger.MemoryLogger(1) + configSet = appl.BackupBase.ConfigurationSet(configDir + os.sep + '*.conf', logger) + count = 0 + files = dict() + for filename in configSet.nextFile(): + count += 1 + files[filename] = 1 + self.assertEquals(2, count) + self.assertTrue(configDir + os.sep + 'example1.conf' in files) + self.assertTrue(configDir + os.sep + 'example2.conf' in files) + self.assertFalse(configDir + os.sep + 'bad.conf~' in files) + + configSet = appl.BackupBase.ConfigurationSet(configDir + os.sep + '*.conf:db:user', logger) + count = 0 + dbs = dict() + users = dict() + for params in configSet.nextParameters(): + count += 1 + dbs[params[0]] = 1 + users[params[1]] = 1 + self.assertEquals(2, count) + self.assertTrue('adam' in dbs) + self.assertTrue('bob' in dbs) + self.assertTrue('alice' in users) + self.assertTrue('berta' in users) + self.assertEquals(0, logger._errors) + + def testBackupDayly(self): + if debug: + return + target = self.makeTargetDir() + configDir = self.makeBackupConfig() + self._textTool.replace([r'=/etc', '=/etc/nginx', + os.path.dirname(target) + os.sep + 'config/dirs.d/sys.etc.conf']) + # options._errorFilter = 'Keine Berechtigung' + appl.BackupTool.main(['backuptool', '-v3', '--configuration-directory=' + configDir, + 'backup', 'dayly']) + storage = self.findBackupStorage(target + os.sep + 'dayly') + self.assertFileExists(storage + os.sep + 'testdb.sql.gz') + + def testBackupWeekly(self): + if debug: + return + target = self.makeTargetDir() + configDir = self.makeBackupConfig() + self._textTool.replace([r'=/etc', '=/etc/nginx', + os.path.dirname(target) + os.sep + 'config/dirs.d/sys.etc.conf']) + # options._errorFilter = 'Keine Berechtigung' + appl.BackupTool.main(['backuptool', '-v3', '--configuration-directory=' + configDir, + 'backup', 'weekly']) + storage = self.findBackupStorage(target + os.sep + 'weekly') + self.assertFileExists(storage + os.sep + 'srv.zip') + self.assertFileExists(storage + os.sep + 'etc=nginx.tgz') + self.assertFileExists(storage + os.sep + 'winfothek.sql.gz') + self.assertFileExists(storage + os.sep + 'system.info.gz') + self.assertFileExists(storage + os.sep + 'usr=local.tgz') + + def testBackupMonthly(self): + if debug: + return + target = self.makeTargetDir() + configDir = self.makeBackupConfig() + self._textTool.replace([r'=/etc', '=/etc/nginx', + os.path.dirname(target) + os.sep + 'config/dirs.d/sys.etc.conf']) + # options._errorFilter = 'Keine Berechtigung' + appl.BackupTool.main(['backuptool', '-v3', '--configuration-directory=' + configDir, + 'backup', 'monthly']) + storage = self.findBackupStorage(target + os.sep + 'monthly') + + self.assertFileExists(storage + os.sep + 'srv.zip') + self.assertFileExists(storage + os.sep + 'etc=nginx.tgz') + self.assertFileExists(storage + os.sep + 'system.info.gz') + self.assertFileExists(storage + os.sep + 'usr=local.tgz') + + def testExample(self): + if debug: + return + configDir = self.makeBackupConfig(True) + logFile = self.tempFile('example.log', 'unittest') + argv = ['dummy', '--configuration-directory=' + configDir, '-l' + logFile, 'example'] + appl.BackupTool.main(argv) + self.assertFileExists(configDir + os.sep + 'backup.conf') + self.assertFileExists(configDir + os.sep + 'webapps.d/testdb.conf') + + def testWebapps(self): + if debug: + return + logFile = self.tempFile('webapps.log', 'unittest') + argv = ['dummy', '-l' + logFile, 'webapps', 'Secret', 'next', 'shop'] + appl.BackupTool.main(argv) + + def testSync(self): + if debug: + return + + (srcDir, trgDir) = self.makeDirToSync() + options = self.getOptions() + tool = appl.BackupTool.BackupTool(options) + tool.synchronize(srcDir, trgDir, True) + + for ix in range(1, 3+1): + full = trgDir + os.sep + 'src_only{:d}.txt'.format(ix) + self.assertTrue(os.path.exists(full)) + full = trgDir + os.sep + 'trg_only{:d}.txt'.format(ix) + self.assertFalse(os.path.exists(full)) + + full = trgDir + os.sep + 'subdir' + os.sep + 'src_only{:d}.txt'.format(ix) + self.assertTrue(os.path.exists(full)) + full = trgDir + os.sep + 'subdir' + os.sep + 'trg_only{:d}.txt'.format(ix) + self.assertFalse(os.path.exists(full)) + + for ix in range(1, 2+1): + full = trgDir + os.sep + 'src_younger{:d}.txt'.format(ix) + content = base.StringUtils.fromFile(full) + self.assertTrue(content.startswith('src')) + full = trgDir + os.sep + 'src_older{:d}.txt'.format(ix) + content = base.StringUtils.fromFile(full) + self.assertTrue(content.startswith('trg')) + self.assertEquals(0, tool._logger._errors) + + def testSyncJobDirect(self): + if debug: + return + + (baseDir, srcFile, trgFile) = self.makeDirDirectSync() + options = self.getOptions() + options._configDir = baseDir + tool = appl.BackupTool.BackupTool(options) + tool.doBackup('often') + file2 = os.path.dirname(trgFile) + os.sep + os.path.basename(srcFile) + self.assertFileExists(file2) + self.assertFileNotExists(trgFile) + self.assertEquals(0, tool._logger._errors) + + def testSyncJobIndirect(self): + if debug: + return + + (baseDir, srcFile, trgFile) = self.makeDirIndirectSync() + options = self.getOptions() + options._configDir = baseDir + tool = appl.BackupTool.BackupTool(options) + tool.doBackup('often') + file2 = os.path.dirname(trgFile) + os.sep + os.path.basename(srcFile) + self.assertFileExists(file2) + self.assertFileNotExists(trgFile) + self.assertEquals(0, tool._logger._errors) + + def testFindDbInfoWinfothek(self): + if debug: + return + options = self.getOptions() + options._configDir = self.makeMinimalConfigDir() + baseDir = self.tempDir('unittest') + config = self.tempFile('winfothek.conf', 'unittest', 'resources') + base.StringUtils.toFile(config, '''# Unittest +.dburl=mysql:host=localhost;dbname=testdb +.dbuser=jonny +.dbcode=Sekret + ''') + tool = appl.BackupTool.BackupTool(options) + (db, user, code) = tool.findDbInfo(baseDir, 'xbackup', 'xbackup2') + self.ensureFileDoesNotExist(config) + self.assertEquals('testdb', db) + self.assertEquals('jonny', user) + self.assertEquals('Sekret', code) + + def testFindDbInfoWordPress(self): + if debug: + return + options = self.getOptions() + options._configDir = self.makeMinimalConfigDir() + baseDir = self.tempDir('unittest') + config = self.tempFile('wp-config.php', 'unittest') + base.StringUtils.toFile(config, '''# Unittest +// ** MySQL settings ** // +define('DB_NAME', 'testdb'); // Der Name der Datenbank, die du benutzt. +define('DB_USER', 'eva'); // Dein MySQL-Datenbank-Benutzername. +define('DB_PASSWORD', 'sEkret'); // Dein MySQL-Passwort + ''') + tool = appl.BackupTool.BackupTool(options) + (db, user, code) = tool.findDbInfo(baseDir, 'xbackup', 'xbackup2') + self.ensureFileDoesNotExist(config) + self.assertEquals('testdb', db) + self.assertEquals('eva', user) + self.assertEquals('sEkret', code) + + def testSnapshot(self): + if debug: + return + dataDir = self.makeTree() + snapshotBase = os.path.dirname(dataDir) + os.sep + 'dayly' + self.ensureDirectory(snapshotBase) + self.clearDirectory(snapshotBase) + appl.BackupTool.main(['backuptool', '-v3', 'snapshot', 'dayly', dataDir]) + dirTool = appl.DirTool.DirTool(self.getOptions()) + snaptshotDir = self.findBackupStorage(snapshotBase) + dirTool.compare([dataDir, snaptshotDir, '--test-content']) + + def testSnapshotClearIfMissing(self): + if debug: + return + dataDir = self.makeTree() + snapshotBase = os.path.dirname(dataDir) + os.sep + 'monthly' + self.ensureFileDoesNotExist(snapshotBase) + appl.BackupTool.main(['backuptool', '-v3', 'snapshot', 'monthly', dataDir, '--create-if-missing']) + dirTool = appl.DirTool.DirTool(self.getOptions()) + nodes = os.listdir(snapshotBase) + if self.assertEquals(1, len(nodes)): + dirTool._testContent = True + dirTool.compare([dataDir, snapshotBase + os.sep + nodes[0], '--test-content']) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = BackupToolTest() + tester.run() diff --git a/unittest/appl/BaseToolTest.py b/unittest/appl/BaseToolTest.py new file mode 100644 index 0000000..ab8c93b --- /dev/null +++ b/unittest/appl/BaseToolTest.py @@ -0,0 +1,191 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import os +import shutil +import base.BaseTool +import sys +# import from base.BaseTool BaseTool + +debug = False + +vBaseToolTestUsage = '?' +def usage(msg = None): + if msg != None: + print('+++ usage() is called with an argument: ' + msg) + return 'usage: this is an pseudo usage' + +def getUsage(): + global vBaseToolTestUsage + return vBaseToolTestUsage + +class BaseToolTest(UnitTestCase): + def __init__(self): + UnitTestCase.__init__(self) + self._appInfo = base.BaseTool.ApplicationInfo('basetooltest', 'BaseToolTest', usage) + self._options = base.BaseTool.GlobalOptions(self._appInfo, 3) + self._options._logFiles = ['-'] + self._tool = base.BaseTool.BaseTool(self._options, 'testbasetool') + self._tool._logger = self._silentLogger + + def testEnsureDirectory(self): + if debug : return + dir1 = self.tempDir('dir1', 'basetooltest') + shutil.rmtree(dir1, True) + self._tool.ensureDirectory(dir1) + self.assertFileExists(dir1) + self._tool.ensureDirectory(dir1) + self.assertFileExists(dir1) + + def testEnsureDirectoryStatic(self): + if debug : return + dir2 = self.tempDir('dir2', 'basetooltest') + shutil.rmtree(dir2, True) + base.BaseTool.BasicStatics.ensureDirectory(dir2) + self.assertFileExists(dir2) + base.BaseTool.BasicStatics.ensureDirectory(dir2) + self.assertFileExists(dir2) + + def testEnsureFileNotExists(self): + if debug : return + file1 = self.tempFile('file1.txt', 'basetooltest') + base.StringUtils.toFile(file1, 'not allowed') + self.assertFileExists(file1) + tool = base.BaseTool.BaseTool(self._options, 'unittest.conf') + tool.ensureFileDoesNotExist(file1) + self.assertFileNotExists(file1) + tool.ensureFileDoesNotExist(file1) + + def testErrorFileToLog(self): + if debug : return + file1 = self.tempFile('file1.txt', 'basetooltest') + base.StringUtils.toFile(file1, 'xy failed\nmissing blub') + tool = base.BaseTool.BaseTool(self._options, 'unittest.conf') + tool._logger = logger = self._silentLogger + tool.errorFileToLog(file1, 'UnitTest!') + self.assertTrue(logger.contains('UnitTest!')) + self.assertTrue(logger.contains('xy failed')) + self.assertTrue(logger.contains('missing blub')) + + def testGrep(self): + if debug : return + file1 = self.tempFile('file1.txt', 'basetooltest') + base.StringUtils.toFile(file1, 'öäü Umlaut\na=b') + aList = base.BaseTool.BasicStatics.grep('*', file1) + self.assertEquals(2, len(aList)) + self.assertEquals('öäü Umlaut\n', aList[0]) + self.assertEquals('a=b', aList[1]) + aList = base.BaseTool.BasicStatics.grep('a=', file1) + self.assertEquals(1, len(aList)) + self.assertEquals('a=b', aList[0]) + aList = base.BaseTool.BasicStatics.grep('U*ut\n', file1) + self.assertEquals(1, len(aList)) + self.assertEquals('öäü Umlaut\n', aList[0]) + + def testClearDirectory(self): + if debug : return + file1 = self.tempFile('x1.txt', 'basetooltest') + base.StringUtils.toFile(file1, 'xy failed\nmissing blub') + self.assertTrue(os.path.exists(file1)) + file2 = self.tempFile('x1.txt', 'basetooltest', 'dir2') + base.StringUtils.toFile(file2, 'xy failed\nmissing blub') + self.assertTrue(os.path.exists(file1)) + baseDir = os.path.dirname(file1) + tool = base.BaseTool.BaseTool(self._options, 'unittest.conf') + tool.clearDirectory(baseDir) + self.assertFalse(os.path.exists(file1)) + self.assertFalse(os.path.exists(file2)) + files = os.listdir(baseDir) + self.assertEquals(0, len(files)) + self.assertTrue(os.path.exists(baseDir)) + + def testFullPathToName(self): + tool = base.BaseTool.BaseTool(self._options, 'unittest.conf') + self.assertEquals('etc=ssh=sshd_config', tool.fullPathToName('/etc/ssh/sshd_config')) + + def testCreateSystemDScript(self): + tool = base.BaseTool.InstallTool('sample', self._options) + tool._systemDPath = self.tempDir('', 'testsystemd') + tool.createSystemDScript('systest', 'systeststarter', 'usample', 'gsample', 'sample script') + self.assertFileContent(tool._systemDPath + 'sample.service', '''[Unit] +Description=sample script. +After=syslog.target +[Service] +Type=simple +User=usample +Group=gsample +WorkingDirectory=/etc/pyrshell +EnvironmentFile=-/etc/pyrshell/systest.env +ExecStart=/usr/local/bin/systeststarter daemon +ExecReload=/usr/local/bin/systeststarter reload +SyslogIdentifier=systest +StandardOutput=syslog +StandardError=syslog +Restart=always +RestartSec=3 +[Install] +WantedBy=multi-user.target +''') + + def testEnsureEnvironmentFile(self): + tool = base.BaseTool.InstallTool('sample', self._options) + tool._configPath = self.tempDir('testbasetool') + os.sep + tool.ensureEnvironmentFile('SAMPLE_', '/abc/def', '/var/log/local/test.log', 'A=B\nC=D') + fn = tool._configPath + 'sample.env' + self.assertFileContains('PYTHONPATH=', fn) + self.assertFileContains('SAMPLE__CONFIG=/abc/def', fn) + self.assertFileContains('SAMPLE__APPL=sample', fn) + self.assertFileContains('SAMPLE__LOGFILE=/var/log/local/test.log', fn) + self.assertFileContains('A=B', fn) + self.assertFileContains('C=D', fn) + + def testIntegerArgument(self): + self.assertEquals(22, self._tool.integerOption('--count=22')) + + def testIntegerArgumentError(self): + self.log('expecting error "not an integer"') + self.assertNone(self._tool.integerOption('--count=22x')) + + self.log('expecting error "missing integer"') + self.assertEquals(-1, self._tool.integerOption('--count:33', -1)) + + def testEnsureSymbolicLink(self): + tempDir = self.tempDir('jail', 'unittest') + target = tempDir + os.sep + 'parent' + # creating base dir and target: + self.ensureFileDoesNotExist(tempDir) + sibling = self.tempDir('sibling', 'unittest') + tool = base.BaseTool.BaseTool(self._options, 'unittest.conf') + tool.ensureSymbolicLink('../../sibling', target) + self.assertTrue(os.path.islink(target)) + self.assertEquals('../../sibling', os.readlink(target)) + # changing link source: + sibling2 = self.tempDir('sibling2', 'unittest') + tool.ensureSymbolicLink('../../sibling2', target) + self.assertTrue(os.path.islink(target)) + self.assertEquals('../../sibling2', os.readlink(target)) + # removing existing target: + self.ensureFileDoesNotExist(target) + base.StringUtils.toFile(target, 'anything') + tool.ensureSymbolicLink('../../sibling2', target) + self.assertTrue(os.path.islink(target)) + self.assertEquals('../../sibling2', os.readlink(target)) + + def testEnsureSymbolicLinkErrors(self): + tempDir = self.tempDir('jail', 'unittest') + target = tempDir + os.sep + 'parent' + self.ensureDirectory(target) + # creating base dir and target: + self.ensureFileDoesNotExist(tempDir) + sibling = self.tempDir('sibling', 'unittest') + tool = base.BaseTool.BaseTool(self._options, 'unittest.conf') + self._logger.log('= expecting error is directory') + tool.ensureSymbolicLink('../../sibling', target) + +if __name__ == '__main__': + # import sys;sys.argv = ['', 'Test.testName'] + tester = BaseToolTest() + tester.run() diff --git a/unittest/appl/BenchToolTest.py b/unittest/appl/BenchToolTest.py new file mode 100644 index 0000000..1a85aff --- /dev/null +++ b/unittest/appl/BenchToolTest.py @@ -0,0 +1,48 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase + +import shutil +import os + +import appl.BenchTool +import base.BaseTool + +debug = False + +def usage(msg=None): + return 'test usage' + +class BenchToolTest(UnitTestCase): + def __init__(self): + UnitTestCase.__init__(self) + self._finish() + self._baseDir = self.tempDir('bench', 'unittest') + shutil.copytree('/home/ws/py/pyrshell/appl', self._baseDir + os.sep + 'pyrshell/appl') + shutil.copytree('/home/ws/py/pyrshell/base', self._baseDir + os.sep + 'pyrshell/base') + shutil.copytree('/home/ws/py/pyrshell/unittest', self._baseDir + os.sep + 'pyrshell/unittest') + + def _finish(self): + shutil.rmtree(self.tempDir('unittest')) + + def testBenchCalculate(self): + if False and debug: + return + appl.BenchTool.main(['benchtool', '-v3', 'calculate', self._baseDir, '--max-loops=20', '--factor-print=10']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testBenchFiles(self): + if debug: + return + appl.BenchTool.main(['benchtool', '-v3', 'files', self._baseDir, '--max-loops=20', '--max-written=100', '--factor-print=10']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + +if __name__ == '__main__': + # import sys;sys.argv = ['', 'Test.testName'] + tester = BenchToolTest() + tester.run() diff --git a/unittest/appl/DbToolTest.py b/unittest/appl/DbToolTest.py new file mode 100644 index 0000000..eb1a631 --- /dev/null +++ b/unittest/appl/DbToolTest.py @@ -0,0 +1,390 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase + +import os + +import appl.DbTool +import base.BaseTool +import shutil + +debug = False + +def usage(msg=None): + return 'test usage' + +class DbToolTest(UnitTestCase): + + def __init__(self): + UnitTestCase.__init__(self) + config = base.JavaConfig.JavaConfig('/etc/pyrshell/dbtool.conf', self._logger) + self._admin = config.getString('admin.user') + self._adminCode = config.getString('admin.code') + self._tool = appl.DbTool.DbTool(self.getOptions()) + self._processHelper = base.ProcessHelper.ProcessHelper(3, self._logger) + + def _finish(self): + shutil.rmtree(self.tempDir('unittest')) + + def buildSqlFile(self, filename, directory = None, table = None): + '''Creates a *.sql.gz file + @param filename: the filename (without trailing '.gz') + @param directory: None: it is chosen automatically otherwise: the "home" of the file + @param table: None or the name of a table + @return: the full filename e.g. "/tmp/unittest/dbtool/example.sql.gz" + ''' + if table == None: + table = 'dummytable' + if directory == None: + directory = self.tempDir('dbtool', 'unittest') + fnSql = directory + os.sep + filename + base.FileHelper.ensureFileExists(fnSql, '''-- MySQL dump 10.16 Distrib 10.1.38-MariaDB, for debian-linux-gnu (x86_64) +DROP TABLE IF EXISTS `{}`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `dummytable` ( + `id` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; +'''.format(table)) + self._processHelper.execute(['gzip', fnSql], True) + return fnSql + '.gz' + + def getOptions(self): + appInfo = base.BaseTool.ApplicationInfo('dbtool', 'appl.DbTool', usage) + rc = base.BaseTool.GlobalOptions(appInfo) + rc._logFiles.append('-') + rc._verboseLevel = 0 + return rc + + def dbExists(self, db, testOnly = False): + dbs = self._tool.findAllDbs(self._admin, self._adminCode) + rc = db in dbs + if not testOnly: + self.assertTrue(rc) + return rc + + def userExists(self, user, testOnly = False): + users = self._tool.findAllUsers(self._admin, self._adminCode) + rc = user in users + if not testOnly: + self.assertTrue(rc) + return rc + + def deleteDbAndUser(self, db, user): + self._tool._forceDeleting = True + if db != None: + self._tool.deleteDb(db, self._admin, self._adminCode) + if user != None: + self._tool.deleteUser(user, self._admin, self._adminCode) + self._tool._forceDeleting = False + + def testCreateDbAndUser(self): + if debug: + return + db = 'dbdummy2' + user = 'uwordpress' + self.deleteDbAndUser(db, user) + appl.DbTool.main(['dbtool', 'create-db-and-user', db, user, 'TopSecret']) + self.dbExists(db) + self.userExists(user) + self.deleteDbAndUser(db, user) + + def testCreateBackupUser(self): + if debug: + return + # <user> <passwd> <user-adm> <passwd-adm> + user = 'ubackup' + self.deleteDbAndUser(None, user) + appl.DbTool.main(['dbtool', + 'create-backup-user', + user, + 'StaySecret']) + self.userExists(user) + self.deleteDbAndUser(None, user) + + def testAllDbs(self): + if debug: + return + # all-dbs <admin> <admin-passwd> [<internal-too>] + appl.DbTool.main(['dbtool', + 'all-dbs']) + dbs = appl.DbTool.result() + self.assertTrue(dbs != None and 'timetracking' in dbs) + + def testCreateWebappDbs(self): + if debug: + return + # create-webapp-dbs <admin> <admin-passwd> + appl.DbTool.main(['dbtool', + 'create-webapp-dbs', self._admin, self._adminCode]) + + def testImportWebApp(self): + if debug: + return + self.deleteDbAndUser('dbexample', 'uexample') + configDir = self.tempDir('webapps.d', 'unittest') + testDir = os.path.dirname(configDir) + fn = configDir + os.sep + 'example.infeos.de.conf' + base.StringUtils.toFile(fn, """db=dbexample +user=uexample +password=hi + """) + fn = self.tempFile('example.infeos.de.sql') + base.StringUtils.toFile(fn, '''create table tdummy ( +id integer, +name varchar(255) +); +drop table tdummy;''') + self._tool.createDbAndUser('dbexample', 'uexample', 'hi', self._admin, self._adminCode) + # <domain> <sql-file> + appl.DbTool.main(['dbtool', + '--configuration-directory=' + testDir, + 'import-webapp', + 'example.infeos.de', + fn]) + self.dbExists('dbexample') + self.userExists('uexample') + self.deleteDbAndUser('dbexample', 'uexample') + + def testImportAllWebApps(self): + if debug: + return + dirSql = self.tempDir('dbtooltest') + for node in ['example1.sql', 'example2.sql']: + fn = self.tempFile(node, 'dbtooltest') + base.StringUtils.toFile(fn, """ + """) + # <sql-directory> + appl.DbTool.main(['dbtool', + '-v3', + 'import-all-webapps', + dirSql]) + if base.BaseTool.BaseTool(self.getOptions(), 'dbtool.conf')._isRoot: + self.dbExists('?') + + def testExportDb(self): + if debug: + return + fn = self.tempFile('db.sql', 'dbtooltest') + self.ensureFileDoesNotExist(fn) + appl.DbTool.main(['dbtool', '-q', + 'export-db', 'timetracking', self._admin, self._adminCode, fn]) + self.assertFileContains('CREATE TABLE', fn) + + def testExportWebApp(self): + if debug: + return + fn = self.tempFile('db.sql.gz', 'dbtooltest') + self.ensureFileDoesNotExist(fn) + appl.DbTool.main(['dbtool', '-q', + 'export-webapp', 'timetracking', fn]) + self.assertFileExists(fn) + + def testCreateAdmin(self): + if debug: + return + self.deleteDbAndUser(None, 'admin2') + appl.DbTool.main(['dbtool', '-v3', 'create-admin', 'admin2', 'NeverGood', self._admin, self._adminCode]) + self.deleteDbAndUser(None, 'admin2') + + def testDeleteAdmin(self): + if debug: + return + appl.DbTool.main(['dbtool', '-v3', 'create-admin', 'testadmin', 'NeverGood']) + appl.DbTool.main(['dbtool', '-v3', 'delete-user', 'testadmin', self._admin, self._adminCode]) + + def testExampleAllUsers(self): + if debug: + return + appl.DbTool.main(['dbtool', 'all-users', self._admin, self._adminCode]) + lines = appl.DbTool.result() + self.assertTrue(self._admin in lines) + + def testExampleSaveWebApp(self): + if debug: + return + dirConfig = self.tempDir('webapps.d', 'unittest') + domain = 'example.com' + home = self.tempDir('example.com', 'unittest') + base.StringUtils.toFile(home + os.sep + 'phpinfo.php', '<?php\nphpinfo();') + base.StringUtils.toFile(dirConfig + os.sep + domain + '.conf', '''db=testdb +user=testuser +password=TopSecret +directory={} +excluded= +'''.format(home)) + archive = self.tempFile('example.com.tgz', 'unittest') + appl.DbTool.main(['dbtool', '-v3', '--test-target-dir=' + dirConfig, + 'save-webapp', domain, archive]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + current = '\n'.join(tool.tableOfContent(archive)) + self.assertEquals('''./ +./phpinfo.php +./db/ +./db/example.com.save-webapp.sql.gz''', current) + + #dbtool -v3 create-webapp-configuration example.com /home/example.com dbexample usrexample NeverUsed + def testExampleCreateWebAppConfiguration(self): + if debug: + return + dirConfig = self.tempDir('webapps.d', 'unittest') + domain = 'example.com' + home = self.tempDir('example.com', 'unittest') + appl.DbTool.main(['dbtool', '-v3', '--test-target-dir=' + dirConfig, + 'create-webapp-configuration', 'example.com', home, 'dbexample', 'usrexample', 'NeverUsed']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileContent(dirConfig + os.sep + domain + '.conf', '''db=dbexample +user=usrexample +password=NeverUsed +sql.file=example.com_dbexample +directory=/tmp/unittest/example.com +excluded=''') + + def testExampleCreateAndImportAllWebapps(self): + if debug: + return + self._finish() + dirConfig = self.tempDir('webapps.d', 'unittest') + domain = 'example.com' + home = self.tempDir('example.com', 'unittest') + self.buildSqlFile('example.com_dbexample.sql', home) + configFile = dirConfig + os.sep + domain + '.conf' + base.StringUtils.toFile(configFile, '''db=dbexample +user=usrexample +password=NeverUsed +sql.file=example.com_dbexample +directory={} +excluded='''.format(home), self._logger) + appl.DbTool.main(['dbtool', '-v3', '--test-target-dir=' + dirConfig, + 'create-and-import-all-webapps', home]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleCreateAndImportWebApp(self): + if False and debug: + return + self._finish() + dirConfig = self.tempDir('webapps.d', 'unittest') + domain = 'example2.com' + home = self.tempDir(domain, 'unittest') + self.buildSqlFile('example2.com_dbexample2.sql', home) + configFile = dirConfig + os.sep + domain + '.conf' + base.StringUtils.toFile(configFile, '''db=dbexample2 +user=usrexample2 +password=NeverUsed +sql.file=example2.com_dbexample2 +directory={} +excluded='''.format(home), self._logger) + appl.DbTool.main(['dbtool', '-v3', '--test-target-dir=' + dirConfig, + 'create-and-import-webapp', domain, home, 'admin', '*Never77Forget']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleFilterRecordsFromSql(self): + if debug: + return + fnIn = self.tempFile('input.sql', 'unittest', 'sql') + fnOut = self.tempFile('output.sql', 'unittest', 'sql') + base.StringUtils.toFile(fnIn, '''-- MySQL dump 10.17 Distrib 10.3.17-MariaDB, for debian-linux-gnu (x86_64) + +DROP TABLE IF EXISTS `user`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `user` (`user_id` int(10) unsigned NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=104 DEFAULT CHARSET=utf8; +LOCK TABLES `user` WRITE; +/*!40000 ALTER TABLE `user` DISABLE KEYS */; +INSERT INTO `user` VALUES (1,'root',NULL,NULL),(2,'jonny',33,NULL),(3,'eva',44,NULL),(4,'admin',55,NULL),(5,'jussuf',64,NULL); +UNLOCK TABLES; +DROP TABLE IF EXISTS `usergroup`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `usergroup` (`user_id` int(10) unsigned NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=104 DEFAULT CHARSET=utf8; +LOCK TABLES `user` WRITE; +/*!40000 ALTER TABLE `usergroup` DISABLE KEYS */; +INSERT INTO `usergroup` VALUES (1,'root',NULL,NULL),(2,'gjonny',33,NULL),(3,'geva',44,NULL),(4,'gadmin',55,NULL),(5,'gjussuf',64,NULL); +UNLOCK TABLES; +''') + appl.DbTool.main(['dbtool', '-v3', 'filter-records', fnIn, 'user', fnOut, "--exclude=,'root',"]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileContent(fnOut, '''-- MySQL dump 10.17 Distrib 10.3.17-MariaDB, for debian-linux-gnu (x86_64) + +LOCK TABLES `user` WRITE; + +INSERT INTO `user` VALUES +(2,'jonny',33,NULL), +(3,'eva',44,NULL), +(4,'admin',55,NULL), +(5,'jussuf',64,NULL); +UNLOCK TABLES;''') + + def testExampleExportAllDbs(self): + if debug: + return + target = self.tempDir('sql', 'unittest') + appl.DbTool.main(['dbtool', '-v3', 'export-all-dbs', target]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + nodes = os.listdir(target) + self.assertTrue(len(nodes) > 0) + + def testExampleDeleteDbAndUser(self): + if False and debug: + return + appl.DbTool.main(['dbtool', '-v3', 'create-db-and-user', 'dbtestdelete', 'usrtestdelete', 'ForgetIt']) + appl.DbTool.main(['dbtool', '-v3', 'delete-db-and-user', 'dbtestdelete', 'usrtestdelete', '--no-saving', '--no-confirmation']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleDeleteDbAndUser2(self): + if debug: + return + # tool.deleteDbAndUser() stopps for user input (confirmation) + if self._inTestSuite: + return + appl.DbTool.main(['dbtool', '-v3', 'create-db-and-user', 'd42', 'u42', 'ForgetIt']) + appl.DbTool.main(['dbtool', '-v3', 'delete-db-and-user', 'd42', 'u42']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertNodeExists(self.tempDir(None), 'dbtestdelete2.sql.[0-9]*.gz') + + def testExampleArchiveWebapp(self): + if False and debug: + return + domain = 'huber42.de' + dirArchive = self.tempDir('archive', 'unittest') + dirWebapp = self.tempDir(domain, 'unittest') + base.StringUtils.toFile(dirWebapp + os.sep + 'index.html', '<html><body>Hi!</body></html>') + dirConf = self.tempDir('archive.conf', 'unittest') + fnConfig = dirConf + os.sep + domain + '.conf' + base.StringUtils.toFile(fnConfig, '''db=dbhuber42 +user=usrhuber42 +password=ForgetIt +directory={} +sql.file=dbhuber42_sql +excluded= +'''.format(dirWebapp)) + fnNginx = dirConf + os.sep + domain + base.StringUtils.toFile(fnNginx, '# nginx file ' + domain) + appl.DbTool.main(['dbtool', '-v3', 'create-db-and-user', 'dbhuber42', 'usrhuber42', 'ForgetIt']) + appl.DbTool.main(['dbtool', '-v3', '--test-target-dir=' + dirConf, + 'archive-webapp', domain, dirArchive]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + fnScript = self.tempFile('Remove.' + domain) + self.assertFileContent(fnScript, '''#! /bin/bash +dbtool -v3 delete-db-and-user dbhuber42 usrhuber42 --no-saving $1 +rm -Rf /tmp/unittest/huber42.de /tmp/unittest/archive.conf/huber42.de /tmp/unittest/archive.conf/huber42.de.conf +rm -f /etc/ssl/certs/huber42.de.pem /etc/ssl/private/huber42.de.pem /etc/nginx/sites-enabled/huber42.de +''') + tool._processHelper.execute(['/bin/bash', fnScript, '--no-confirmation'], True) + +if __name__ == '__main__': + tester = DbToolTest() + tester.run() diff --git a/unittest/appl/DirToolTest.py b/unittest/appl/DirToolTest.py new file mode 100644 index 0000000..39cbf3f --- /dev/null +++ b/unittest/appl/DirToolTest.py @@ -0,0 +1,418 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase + +import shutil +import os +import datetime + +import appl.DirTool +import base.BaseTool +import base.StringUtils + +debug = False + +def usage(msg=None): + return 'test usage' + +class DirToolTest(UnitTestCase): + def __init__(self): + UnitTestCase.__init__(self) + self._finish() + self._logger = base.MemoryLogger.MemoryLogger(1) + base.FileHelper.clearDirectory(self.tempDir('unittest')) + appl.DirTool.main(['dirtool', '-q', 'build-examples']) + self._exampleBase = '/tmp/%examples' + self._dir1 = self._exampleBase + '/dir1' + self._dir2 = self._exampleBase + '/dir2' + self._sourceDir = self._dir1 + self._targetDir = self._dir2 + self._safeDir = self._exampleBase + '/safe' + self.ensureDirectory(self._safeDir) + + def _finish(self): + shutil.rmtree(self.tempDir('unittest')) + + def createFile(self, baseDir, path, node, number): + fn = baseDir + os.sep + ('' if path == '' else path + os.sep) + node.format(number) + base.StringUtils.toFile(fn, self._fileContent) + base.FileHelper.setModified(fn, None, datetime.datetime(2018, 3 + number % 7, 1 + number%28, 4+number % 18, 22-number % 22, 3*number%60)) + + def _makeMirror(self): + appl.DirTool.main(['dirtool', '-q', 'build-examples']) + + def _makeMirror2(self): + appl.DirTool.main(['dirtool', '-q', 'build-examples']) + + def getOptions(self): + rc = base.BaseTool.GlobalOptions() + rc._exampleText = 'log.file=/var/log/local/wikitool.log' + rc._logFiles.append('-') + rc._verboseLevel = 0 + rc._appInfo = self._applicationName + return rc + + def testSync(self): + if debug: + return + self._makeMirror() + ''' + fn = '/usr/share/pyrshell/examples/dir2/file5.txt' + content = base.StringUtils.fromFile(fn) + aTime = os.path.getmtime(fn) + build(fn, content.replace('e', 'E'), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(aTime))) + ''' + self.assertFileNotExists(self._targetDir + '/file2.txt') + self.assertFileNotExists(self._targetDir + '/subdir1/data_6.txt') + self.assertFileContains('is a file instead of the name', self._targetDir + '/subdir2') + self.assertFileContains('line1', self._targetDir + '/new.txt') + self.assertFileContains('line3', self._targetDir + '/subdir1/new2.txt') + self.assertFileContains('newer content in file4.txt', self._targetDir + '/file4.txt') + self.assertFileContains('bla bla', self._targetDir + '/subdir3/data_5.txt') + self.assertFileContains('contEnt of thE filE filE4.txt', self._targetDir + '/file5.txt') + self.assertFileContains('contEnt of thE filE filE5.txt', self._targetDir + '/subdir3/data_6.txt') + + base.FileHelper.clearDirectory(self._safeDir, self._logger) + appl.DirTool.main(['filetool', '-v4', 'sync', self._sourceDir, self._targetDir, self._safeDir]) + + appl.DirTool.main(['filetool', 'compare', self._sourceDir, self._targetDir]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileContains('is a file instead of the name', self._safeDir + '/subdir2') + self.assertFileContains('line1', self._safeDir + '/new.txt') + self.assertFileContains('line3', self._safeDir + '/subdir1/new2.txt') + self.assertFileContains('content of the file file4.txt', self._safeDir + '/subdir1/data_5.txt') + self.assertFileContains('content of the file file0.txt', self._safeDir + '/file1.txt') + self.assertFileContains('line2', self._safeDir + '/subdir3/data5.txt') + self.assertFileContains('newer content in file4.txt', self._safeDir + '/file4.txt') + + def testSnapshot(self): + if debug: + return + self._makeMirror() + target = os.path.dirname(self._targetDir) + os.sep + 'snapshot' + if os.path.exists(target): + shutil.rmtree(target) + base.FileHelper.clearDirectory(self._safeDir, self._logger) + appl.DirTool.main(['filetool', '-v4', 'snapshot', self._sourceDir, target]) + appl.DirTool.main(['filetool', 'compare', self._sourceDir, target]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testInfo(self): + if debug: + return + self._makeMirror() + appl.DirTool.main(['filetool', '-v4', 'info', self._sourceDir, '--count=2', self._targetDir]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 7 Files: 27 / 1.226 KB +Ignored: 0 file(s) / 0 dir(s) +The smallest files: +2017.07.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_7.txt +2017.05.03 07:03:53 69 Byte /tmp/%examples/dir1/subdir3/data_5.txt +The oldest files: +2017.05.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_5.txt +2017.05.02 07:03:52 45 Byte /tmp/%examples/dir1/subdir2/data_5.txt +The largest files: +2018.02.03 07:03:51 45 Byte /tmp/%examples/dir1/file2.txt +2017.05.03 07:03:53 69 Byte /tmp/%examples/dir1/subdir3/data_5.txt +The youngest files: +2018.11.03 09:03:53 <dir> /tmp/%examples/dir2/subdir3 +2018.12.09 07:03:53 <dir> /tmp/%examples/dir1/subdir2''', out) + appl.DirTool.main(['filetool', '-v4', 'info', self._sourceDir, self._targetDir, + '--max-largest=3', '--max-oldest=2', '--max-youngest=7']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 7 Files: 27 / 1.226 KB +Ignored: 0 file(s) / 0 dir(s) +The smallest files: +2017.06.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_6.txt +2017.05.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_5.txt +2017.07.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_7.txt +2017.07.03 07:03:53 45 Byte /tmp/%examples/dir1/subdir3/data_7.txt +2017.05.03 07:03:53 69 Byte /tmp/%examples/dir1/subdir3/data_5.txt +The oldest files: +2017.05.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_5.txt +2017.05.02 07:03:52 45 Byte /tmp/%examples/dir1/subdir2/data_5.txt +The largest files: +2017.05.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_5.txt +2018.02.03 07:03:51 45 Byte /tmp/%examples/dir1/file2.txt +2017.05.03 07:03:53 69 Byte /tmp/%examples/dir1/subdir3/data_5.txt +The youngest files: +2018.05.03 07:03:54 69 Byte /tmp/%examples/dir1/file5.txt +2018.05.03 07:03:54 69 Byte /tmp/%examples/dir2/file5.txt +2018.09.03 09:03:53 <dir> /tmp/%examples/dir2/subdir1 +2018.10.03 09:03:53 26 Byte /tmp/%examples/dir2/file4.txt +2018.10.03 09:07:53 11 Byte /tmp/%examples/dir2/subdir3/data5.txt +2018.11.03 09:03:53 <dir> /tmp/%examples/dir2/subdir3 +2018.12.09 07:03:53 <dir> /tmp/%examples/dir1/subdir2''', out) + + def testExampleLargest(self): + if debug: + return + self._makeMirror() + appl.DirTool.main(['filetool', '-v4', 'largest', self._sourceDir, '--pattern=.*1.*']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 4 Files: 1 / 37 Byte +Ignored: 13 file(s) / 0 dir(s) +The largest files: +2018.01.03 07:03:50 37 Byte /tmp/%examples/dir1/file1.txt''', out) + + def testExampleYoungest(self): + if debug: + return + self._makeMirror() + appl.DirTool.main(['filetool', '-v4', 'youngest', self._sourceDir, '--max-depth=0']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 1 Files: 5 / 265 Byte +Ignored: 0 file(s) / 3 dir(s) +The youngest files: +2018.03.08 07:03:53 <dir> /tmp/%examples/dir1/subdir1 +2018.03.10 07:03:53 <dir> /tmp/%examples/dir1/subdir3 +2018.04.03 07:03:53 61 Byte /tmp/%examples/dir1/file4.txt +2018.05.03 07:03:54 69 Byte /tmp/%examples/dir1/file5.txt +2018.12.09 07:03:53 <dir> /tmp/%examples/dir1/subdir2''', out) + + def testExampleOldest(self): + if debug: + return + self._makeMirror() + appl.DirTool.main(['filetool', '-v4', 'oldest', self._sourceDir, '--file-only']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 4 Files: 14 / 718 Byte +Ignored: 0 file(s) / 0 dir(s) +The oldest files: +2017.05.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_5.txt +2017.05.02 07:03:52 45 Byte /tmp/%examples/dir1/subdir2/data_5.txt +2017.05.03 07:03:53 69 Byte /tmp/%examples/dir1/subdir3/data_5.txt +2017.06.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_6.txt +2017.06.02 07:03:52 53 Byte /tmp/%examples/dir1/subdir2/data_6.txt''', out) + + def testCheck(self): + if debug: + return + self._makeMirror() + appl.DirTool.main(['filetool', '-v4', 'check', self._sourceDir]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testCompare(self): + if debug: + return + self._makeMirror2() + appl.DirTool.main(['filetool', '-v4', 'compare', self._sourceDir, self._targetDir, '--test-content']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(tool._differences) + self.assertEquals('''additional: /tmp/%examples/dir1/file2.txt +date: 2017.05.01 07:03:51 / 2018.01.03 09:03:53 /tmp/%examples/dir1/subdir1/data_5.txt +additional: /tmp/%examples/dir1/subdir1/data_6.txt +missing counterpart: /tmp/%examples/dir1/subdir1/new2.txt +different at position [5]: /tmp/%examples/dir1/subdir3/data_6.txt +missing counterpart: /tmp/%examples/dir1/subdir3/data5.txt +different types: dir / file /tmp/%examples/dir1/subdir2 +different at position [5]: /tmp/%examples/dir1/file5.txt +size: 61 / 26 /tmp/%examples/dir1/file4.txt +date: 2018.01.03 07:03:50 / 2018.01.03 09:03:53 /tmp/%examples/dir1/file1.txt +missing counterpart: /tmp/%examples/dir1/new.txt''', out) + def testCompareShort(self): + if debug: + return + self._makeMirror2() + appl.DirTool.main(['dirtool', '-v4', 'compare', self._sourceDir, self._targetDir, '--short', '--test-content']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(tool._differences) + self.assertEquals('''+ /tmp/%examples/dir1/file2.txt +T 2017.05.01 07:03:51 / 2018.01.03 09:03:53 /tmp/%examples/dir1/subdir1/data_5.txt ++ /tmp/%examples/dir1/subdir1/data_6.txt +- /tmp/%examples/dir1/subdir1/new2.txt +C [5] /tmp/%examples/dir1/subdir3/data_6.txt +- /tmp/%examples/dir1/subdir3/data5.txt +! dir / file /tmp/%examples/dir1/subdir2 +C [5] /tmp/%examples/dir1/file5.txt +S 61 / 26 /tmp/%examples/dir1/file4.txt +T 2018.01.03 07:03:50 / 2018.01.03 09:03:53 /tmp/%examples/dir1/file1.txt +- /tmp/%examples/dir1/new.txt''', out) + + def testExampleCheck(self): + if debug: + return + appl.DirTool.main(['dirtool', '-v3', 'check', self._dir1, self._dir2]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleCompare1(self): + if debug: + return + appl.DirTool.main(['dirtool', '-v3', 'compare', self._dir1, self._dir2]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(tool._differences) + self.assertEquals('''additional: /tmp/%examples/dir1/file2.txt +date: 2017.05.01 07:03:51 / 2018.01.03 09:03:53 /tmp/%examples/dir1/subdir1/data_5.txt +additional: /tmp/%examples/dir1/subdir1/data_6.txt +missing counterpart: /tmp/%examples/dir1/subdir1/new2.txt +missing counterpart: /tmp/%examples/dir1/subdir3/data5.txt +different types: dir / file /tmp/%examples/dir1/subdir2 +size: 61 / 26 /tmp/%examples/dir1/file4.txt +date: 2018.01.03 07:03:50 / 2018.01.03 09:03:53 /tmp/%examples/dir1/file1.txt +missing counterpart: /tmp/%examples/dir1/new.txt''', out) + + def testExampleCompare2(self): + if debug: + return + + appl.DirTool.main(['dirtool', '-v3', 'compare', self._dir1, self._dir2, '--ignore-time', '--short', '--test-content']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(tool._differences) + self.assertEquals('''+ /tmp/%examples/dir1/file2.txt ++ /tmp/%examples/dir1/subdir1/data_6.txt +- /tmp/%examples/dir1/subdir1/new2.txt +C [5] /tmp/%examples/dir1/subdir3/data_6.txt +- /tmp/%examples/dir1/subdir3/data5.txt +! dir / file /tmp/%examples/dir1/subdir2 +C [5] /tmp/%examples/dir1/file5.txt +S 61 / 26 /tmp/%examples/dir1/file4.txt +- /tmp/%examples/dir1/new.txt''', out) + + def testExampleCompare3(self): + if debug: + return + + appl.DirTool.main(['dirtool', '-v4', 'compare', self._dir1, self._dir2, '-s', '--exlude=sub.*2', '--pattern=[^u]*[1-3].*', '-i', '--ignore-case']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(tool._differences) + self.assertEquals('''+ /tmp/%examples/dir1/file2.txt +! dir / file /tmp/%examples/dir1/subdir2 +T 2018.01.03 07:03:50 / 2018.01.03 09:03:53 /tmp/%examples/dir1/file1.txt +- /tmp/%examples/dir1/new.txt''', out) + + +#dirtool info /usr/share/pyrshell/examples/dir1 --max-largest=2 --max-youngest=3 --file-only + def testExampleInfo1(self): + if debug: + return + appl.DirTool.main(['dirtool', '-v3', 'info', self._dir1, + '--max-largest=2', '--max-youngest=3', '--max-oldest=1', '--file-only']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 4 Files: 14 / 718 Byte +Ignored: 0 file(s) / 0 dir(s) +The smallest files: +2017.06.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_6.txt +2017.05.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_5.txt +2017.07.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_7.txt +2017.07.03 07:03:53 45 Byte /tmp/%examples/dir1/subdir3/data_7.txt +2017.05.03 07:03:53 69 Byte /tmp/%examples/dir1/subdir3/data_5.txt +The oldest files: +2017.05.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_5.txt +The largest files: +2018.02.03 07:03:51 45 Byte /tmp/%examples/dir1/file2.txt +2017.05.03 07:03:53 69 Byte /tmp/%examples/dir1/subdir3/data_5.txt +The youngest files: +2018.03.03 07:03:52 53 Byte /tmp/%examples/dir1/file3.txt +2018.04.03 07:03:53 61 Byte /tmp/%examples/dir1/file4.txt +2018.05.03 07:03:54 69 Byte /tmp/%examples/dir1/file5.txt''', out) + + def testExampleInfo2(self): + if debug: + return + appl.DirTool.main(['dirtool', '-v3', 'info', self._dir1, + '--max-largest=0', '--max-smallest=0', '--max-youngest=3', '--max-oldest=4', '--dir-only']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 4 Files: 0 / 0 Byte +Ignored: 14 file(s) / 0 dir(s) +The oldest files: +2018.03.08 07:03:53 <dir> /tmp/%examples/dir1/subdir1 +2018.03.10 07:03:53 <dir> /tmp/%examples/dir1/subdir3 +2018.12.09 07:03:53 <dir> /tmp/%examples/dir1/subdir2 +The youngest files: +2018.03.08 07:03:53 <dir> /tmp/%examples/dir1/subdir1 +2018.03.10 07:03:53 <dir> /tmp/%examples/dir1/subdir3 +2018.12.09 07:03:53 <dir> /tmp/%examples/dir1/subdir2''', out) + + def testExampleInfo3(self): + if debug: + return + appl.DirTool.main(['dirtool', '-v3', 'info', self._dir1, + '--max-largest=0', '--max-smallest=0', '--max-youngest=3', '--max-oldest=4', '--file-only']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 4 Files: 14 / 718 Byte +Ignored: 0 file(s) / 0 dir(s) +The oldest files: +2017.05.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_5.txt +2017.05.02 07:03:52 45 Byte /tmp/%examples/dir1/subdir2/data_5.txt +2017.05.03 07:03:53 69 Byte /tmp/%examples/dir1/subdir3/data_5.txt +2017.06.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_6.txt +The youngest files: +2018.03.03 07:03:52 53 Byte /tmp/%examples/dir1/file3.txt +2018.04.03 07:03:53 61 Byte /tmp/%examples/dir1/file4.txt +2018.05.03 07:03:54 69 Byte /tmp/%examples/dir1/file5.txt''', out) + + def testExampleListTar1(self): + if False and debug: + return + appl.DirTool.main(['dirtool', '-v3', 'list', '/usr/share/pyrshell/unittest/data/examples2.tgz']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 9 Files: 27 / 1.226 KB +Ignored: 0 file(s) / 0 dir(s) +''', out) + out = tool._rawOutput + self.assertEquals(''' +''', out) + + def testExampleListDir(self): + if True: + return + if debug: + return + appl.DirTool.main(['dirtool', '-v3', 'list', '/usr/share/pyrshell/unittest/data', '--order-date-size', '--file-only', '--mbyte-size']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + out = '\n'.join(base.BaseTool.result()) + self.assertEquals('''Directories: 4 Files: 14 / 718 Byte +Ignored: 0 file(s) / 0 dir(s) +The oldest files: +2017.05.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_5.txt +2017.05.02 07:03:52 45 Byte /tmp/%examples/dir1/subdir2/data_5.txt +2017.05.03 07:03:53 69 Byte /tmp/%examples/dir1/subdir3/data_5.txt +2017.06.01 07:03:51 37 Byte /tmp/%examples/dir1/subdir1/data_6.txt +The youngest files: +2018.03.03 07:03:52 53 Byte /tmp/%examples/dir1/file3.txt +2018.04.03 07:03:53 61 Byte /tmp/%examples/dir1/file4.txt +2018.05.03 07:03:54 69 Byte /tmp/%examples/dir1/file5.txt''', out) + + def testExampleImageResize(self): + if False and debug: + return + target = self.tempDir('images', 'unittest') + appl.DirTool.main(['dirtool', '-v3', 'image-resize', '/usr/share/pyrshell/unittest/img', target, '--max-width=100', '--max-height=50']) + self.assertFileExists(target + os.sep + 'landscape.jpg') + self.assertFileExists(target + os.sep + 'portrait.jpg') + +if __name__ == '__main__': + # import sys;sys.argv = ['', 'Test.testName'] + tester = DirToolTest() + tester.run() diff --git a/unittest/appl/FtpToolTest.py b/unittest/appl/FtpToolTest.py new file mode 100644 index 0000000..56ade62 --- /dev/null +++ b/unittest/appl/FtpToolTest.py @@ -0,0 +1,154 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import os.path +import time +import appl.FtpTool +import base.StringUtils +import shutil + +def usage(msg=None): + return 'test usage' + +debug = False + +class FtpToolTest(UnitTestCase): + def __init__(self): + UnitTestCase.__init__(self) + self._baseDir = self.tempDir('ftptool', 'unittest') + + def _finish(self): + shutil.rmtree(self._baseDir) + + def getOptions(self): + appInfo = base.BaseTool.ApplicationInfo('ftptool', 'appl.FtpTool.py', usage) + rc = base.BaseTool.GlobalOptions(appInfo) + rc._logFiles.append('-') + return rc + + def makeSourceTree(self): + text = 'Dies ist ein langer Text, der hoffentlich ein wenig komprimiert werden kann. Wenn nicht, kann man nichts machen' * 1000 + dirName = self._baseDir + os.sep + 'src' + os.makedirs(dirName, 0o777, True) + os.chmod(dirName, 0o777) + # /tmp/zippertest/srcdir: file[1-5].txt dir[123] .git linkConfig + # /tmp/zippertest/srcdir/dirX: levelX.txt subdir1 + # /tmp/zippertest/srcdir/dirX/subdir1: a.jpg b.txt c.odt + # /tmp/zippertest/srcdir/.git: config aa bb + # /tmp/zippertest/srcdir/.git/xx: xx1234 xx2345 xx3456 + for ix in range(1, 5+1): + fn = dirName + os.sep + 'file{:d}.txt'.format(ix) + base.StringUtils.toFile(fn, 'no{:d}: {:s}'.format(ix, text)) + os.chmod(fn, 0o666) + if not os.path.exists(fn): + os.symlink('file1.txt', fn) + for ix in range(1, 3+1): + subdir = dirName + os.sep + 'dir' + str(ix) + os.makedirs(subdir, 0o777, True) + os.chmod(subdir, 0o777) + fn = subdir + os.sep + 'level{:d}'.format(ix) + base.StringUtils.toFile(fn, 'level{:d}: {:s}'.format(ix, text)) + dir2 = subdir + os.sep + 'subdir1' + os.makedirs(dir2, 0o777, True) + os.chmod(dir2, 0o777) + for node in ['a.jpg', 'b.txt', 'c.odt']: + fn = dir2 + os.sep + node + base.StringUtils.toFile(fn, node + ': ' + text) + os.chmod(fn, 0o666) + return dirName + + def makeMinimalConfigDir(self): + dirName = self.tempFile('pyrshell', 'etc') + os.makedirs(dirName, 0o777, True) + base.StringUtils.toFile(dirName + os.sep + 'backup.conf', '') + return dirName + + def makeTargetDir(self): + targetDir = self._baseDir + 'trgdir' + self.clearDirectory(targetDir) + return targetDir + + def makeFtpConfig(self, clear=True): + baseDir = self.tempDir('config', 'backuptest') + if clear: + self.clearDirectory(baseDir) + targetDir = self.makeTargetDir() + appDataDir = self.tempDir('blog', 'backuptest') + base.StringUtils.toFile(appDataDir + os.sep + 'index.php', '<?php\ninfo();') + base.StringUtils.toFile(appDataDir + os.sep + 'new.php', '<?php\nnew();') + oldFile = appDataDir + os.sep + 'old.php' + base.StringUtils.toFile(oldFile, '<?php\nold();') + ftime = time.mktime((2018,1,2,3,4,5,0,0,0)) + os.utime(oldFile, (ftime, ftime)) + + appDir = self.ensureDirectory(baseDir + os.sep + 'ftp.d') + base.StringUtils.toFile(appDir + os.sep + 'unittest.conf', ''' +host=localhost +user=unittest +port=21 +code=Secret +'''.format(appDataDir)) + base.StringUtils.toFile(baseDir + os.sep + 'ftptool.conf', '''# created by unittest +log.file=/tmp/ftptooltest.log +'''.format(targetDir, appDir)) + return baseDir + + def testFtpInfo(self): + if debug: + return + configDir = self.makeFtpConfig() + logFile = self.tempFile('ftpinfo.log', 'unittest') + self.ensureFileDoesNotExist(logFile) + argv = ['ftptool', '-c' + configDir, '-l' + logFile, 'info', 'unittest', 'localhost'] + appl.FtpTool.main(argv) + lines = base.StringUtils.fromFile(logFile) + self.assertTrue(len(lines) > 0) + self.assertTrue(lines.find('MLST modify*;') > 0) + + def testFtpRmTree(self): + if debug: + return + source = self.makeSourceTree() + configDir = self.makeFtpConfig() + logFile = self.tempFile('ftpinfo.log', 'ftptest') + self.ensureFileDoesNotExist(logFile) + ftpDir = '/home/unittest/unittest/' + os.path.basename(source) + argv = ['ftptool', '-c' + configDir, '-l' + logFile, '-v3', 'rmtree', 'unittest', ftpDir] + appl.FtpTool.main(argv) + self.assertFileNotExists(source) + lines = base.StringUtils.fromFile(logFile) + self.assertTrue(len(lines) > 0) + self.assertTrue(lines.find('/home/unittest/unittest/src/file2.txt') > 0) + + def testFtpLsTree(self): + if False and debug: + return + source = self.makeSourceTree() + configDir = self.makeFtpConfig() + logFile = self.tempFile('ftpinfo.log', 'ftptest') + self.ensureFileDoesNotExist(logFile) + ftpDir = '/home/unittest/unittest/' + os.path.basename(source) + argv = ['ftptool', '-c' + configDir, '-l' + logFile, '-v3', 'lstree', 'unittest', ftpDir] + appl.FtpTool.main(argv) + lines = base.StringUtils.fromFile(logFile) + self.assertTrue(len(lines) > 0) + self.assertTrue(lines.find('/home/unittest/unittest/src/file2.txt') > 0) + + def testExample(self): + if debug: + return + configDir = self.makeFtpConfig() + logFile = self.tempFile('ftpinfo.log', 'ftptest') + self.ensureFileDoesNotExist(logFile) + argv = ['ftptool', '-c' + configDir, '-l' + logFile, 'example'] + appl.FtpTool.main(argv) + self.assertFileExists(configDir + os.sep + 'ftptool.conf.example') + self.assertFileExists(configDir + os.sep + 'ftp.d' + os.sep + 'example.conf') + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = FtpToolTest() + tester.run() diff --git a/unittest/appl/MonitorTest.py b/unittest/appl/MonitorTest.py new file mode 100644 index 0000000..92ea528 --- /dev/null +++ b/unittest/appl/MonitorTest.py @@ -0,0 +1,276 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +import os.path +from unittest.UnitTestCase import UnitTestCase +import appl.Monitor +import base.Logger +import base.MemoryLogger + +def usage(msg=None): + return 'test usage' + +debug = False + +class MonitorTest(UnitTestCase): + + def __init__(self): + UnitTestCase.__init__(self) + appInfo = base.BaseTool.ApplicationInfo('pymonitor', 'appl/Monitor.py', usage) + self._options = self.getOptions() + self._options._logFiles = [] + + def _finish(self): + self.clearDirectory(self.tempDir('unittest')) + + def getOptions(self): + appInfo = base.BaseTool.ApplicationInfo('pymonitor', 'appl/Monitor.py', usage) + options = base.BaseTool.GlobalOptions(appInfo) + options._logFiles = [] + return options + + def checkObservedProperty(self, parent, name, warnLimit, errorLimit): + self.assertTrue(name in parent._observed) + if name in parent._observed: + prop = parent._observed[name] + self.assertEquals(name, prop._id) + self.assertEquals(warnLimit, prop._warnLimit) + self.assertEquals(errorLimit, prop._errorLimit) + + def checkProperty(self, parent, name, value): + self.assertTrue(name in parent._properties) + if name in parent._properties: + prop = parent._properties[name] + self.assertEquals(name, prop._id) + if isinstance(value, appl.Monitor.ReceiverGroup): + self.assertEquals(value._properties['error'], prop._value._properties['error']) + self.assertEquals(value._properties['warning'], prop._value._properties['warning']) + else: + self.assertEquals(value, prop._value) + + def checkReceivers(self, receiver1, receiver2): + self.assertTrue(isinstance(receiver1, appl.Monitor.ReceiverGroup)) + self.assertTrue(isinstance(receiver2, appl.Monitor.ReceiverGroup)) + self.assertEquals(receiver1._name, receiver2._name) + self.assertEquals(receiver1._properties['error']._value, receiver2._properties['error']._value) + self.assertEquals(receiver1._properties['warning']._value, receiver2._properties['warning']._value) + + def checkDisk(self, host, name, usedWarn, usedError, receivers = None): + self.assertTrue(name in host._disks) + disk = host._disks[name] + self.assertEquals('Disk', disk._type) + self.assertEquals(name, disk._name) + self.assertEquals(usedWarn, disk._observed['used']._warnLimit) + self.assertEquals(usedError, disk._observed['used']._errorLimit) + if receivers != None: + self.assertEquals(receivers, disk._properties['receivers']._name) + + def checkHost(self, monitor, name, address, receivers, load1Warn, load1Error, load5Warn, load5Error, load10Warn, load10Error, procWarn, procError): + self.assertTrue(name in monitor._hosts) + host = monitor._hosts[name] + self.checkProperty(host, 'address', address) + self.checkReceivers(receivers, host._properties['receivers']) + self.checkReceivers(receivers, host._properties['receivers']) + self.checkObservedProperty(host, 'load1', load1Warn, load1Error) + self.checkObservedProperty(host, 'load5', load5Warn, load5Error) + self.checkObservedProperty(host, 'load10', load10Warn, load10Error) + self.checkObservedProperty(host, 'processes', procWarn, procError) + return host + + def testDump(self): + if debug: return + configDir = os.path.dirname(os.path.dirname(__file__)) + '/configuration' + options = self.getOptions() + options._additionalConfigDir = configDir + options._logFile = ['/tmp/unittest.log'] + monitor = appl.Monitor.Monitor(options, configDir) + dump = monitor.dump() + self.assertMatches(r'Disk "/":', dump) + pass + + def testBasic(self): + if debug: return + configDir = os.path.dirname(os.path.dirname(__file__)) + '/configuration' + options = self.getOptions() + options._additionalConfigDir = configDir + options._logFile = ['/tmp/unittest.log'] + monitor = appl.Monitor.Monitor(options, configDir) + host = self.checkHost(monitor, '', 'localhost', monitor._receiverGroups[''], '4.0', '5.0', '3.1', '4.1', '2.2', '3.2', '500', '1000') + self.checkDisk(host, '', '81%', '91%', '') + + host = self.checkHost(monitor, 'localhost', '127.0.0.1', monitor._receiverGroups['group1'], '3.3', '5.3', '2.4', '3.4', '1.5', '2.5', '502', '1002') + self.checkDisk(host, '', '81%', '91%', '') + self.checkDisk(host, '/', '70%', '80%', 'globalGroup') + self.checkDisk(host, '/home', '83%', '93%', 'group1') + self.checkDisk(host, '/opt', '84%', '94%', '') + self.assertEquals('globalGroup', host._disks['/']._properties['receivers']._name) + self.assertEquals('group1', host._observed['load1']._receivers._name) + self.assertEquals('group1', host._observed['load5']._receivers._name) + self.assertEquals('globalGroup', host._observed['load10']._receivers._name) + self.assertEquals('group1', host._observed['processes']._receivers._name) + + def testFormatErrors(self): + if debug: return + configDir = os.path.dirname(os.path.dirname(__file__)) + '/configuration/error' + options = self.getOptions() + options._additionalConfigDir = configDir + options._logFiles = [] + monitor = appl.Monitor.Monitor(options, configDir) + self._logger.log('=== Note many errors follow...') + self.assertTrue(monitor._logger.contains('unexpected "}"')) + self.assertTrue(monitor._logger.contains('nested definition of host')) + self.assertTrue(monitor._logger.contains('disk not inside a host definition')) + self.assertTrue(monitor._logger.contains('disk not inside a host definition. Found:')) + self.assertTrue(monitor._logger.contains('property outside a container')) + self.assertTrue(monitor._logger.contains('unknown property dummyProp')) + self.assertTrue(monitor._logger.contains('unknown receivergroup: unknownGroup1')) + self.assertTrue(monitor._logger.contains('unknown receivergroup: unknownGroup2')) + self.assertTrue(monitor._logger.contains('receiver group definition inside a container')) + self.assertTrue(monitor._logger.contains('unexpected input')) + self.assertTrue(monitor._logger.contains('missing "}"')) + + def testObserveDisks(self): + if debug: return + configDir = os.path.dirname(os.path.dirname(__file__)) + '/configuration/observed' + options = self.getOptions() + options._additionalConfigDir = configDir + options._logFile = ['/tmp/unittest.log'] + monitor = appl.Monitor.Monitor(options, configDir) + with open(self.tempFile('monitor.conf'), 'w') as fp: + fp.write(monitor.dump()) + monitor._currentHostname = 'localhost' + rc = monitor.observeDisks() + if self.assertEquals(2, len(rc)): + # W |localhost:disk:/| disk usage warning free: 8.685 GiByte (29.5%) of 29.404 GiByte Limit: 10%||1568545510 + self.assertMatches(r'W\|localhost:disk:/\|disk usage warning free: [0-9.]+ \S+ \([.0-9]+%\) of [0-9.]+ \S+ Limit: 10%\|', rc[0]) + # E|localhost:disk:/home |disk usage error free: 28.331 GiByte (45.3%) of 62.496 GiByte Limit: 11%|monitor@hamatoma.de pit@hamatoma.de|1568555108 + self.assertMatches(r'E\|localhost:disk:/home\|disk usage error free: [0-9.]+ \S+ \([.0-9]+%\) of [0-9.]+ \S+ Limit: 11%\|monitor\@hamatoma.de pit\@hamatoma.de', rc[1]) + #self.assertMatches(r'W\|localhost:disk:/media/work\|disk usage warning: free: [0-9.]+ \S+ \([.0-9]+%\) of [0-9.]+ \S+ Limit: 81%\|', rc[2]) + + def testObserveCore(self): + if debug: return + configDir = os.path.dirname(os.path.dirname(__file__)) + '/configuration/observed' + options = self.getOptions() + options._additionalConfigDir = configDir + options._logFile = ['/tmp/unittest.log'] + monitor = appl.Monitor.Monitor(options, configDir) + monitor._currentHostname = 'localhost' + rc = monitor.observeCore() + if self.assertEquals(5, len(rc)): + self.assertMatches(r'W\|localhost:core:load1\|core detection warning\s+\S+ Limit: \S+\s*\|one@hamatoma.de two@hamatoma.de', rc[0]) + self.assertMatches(r'E\|localhost:core:load5\|core detection error\s+\S+ Limit: \S+\s*\|critical@hamatoma.de', rc[1]) + self.assertMatches(r'W\|localhost:core:load10\|core detection warning\s+\S+ Limit: \S+\s*\|one@hamatoma.de two@hamatoma.de', rc[2]) + self.assertMatches(r'E\|localhost:core:processes\|core detection error\s+\S+ Limit: \S+\s*\|critical@hamatoma.de', rc[3]) + self.assertMatches(r'W\|localhost:core:swap\|core detection warning 0.0 Limit: 0\|one@hamatoma.de two@hamatoma.de\|', rc[4]) + + def testObserve(self): + if debug: return + configDir = os.path.dirname(os.path.dirname(__file__)) + '/configuration/observed' + options = self.getOptions() + options._additionalConfigDir = configDir + options._logFile = ['/tmp/unittest.log'] + monitor = appl.Monitor.Monitor(options, configDir) + (newProblems, closedProblems) = monitor.observe() + self.assertEquals(9, len(newProblems)) + self.assertEquals(0, len(closedProblems)) + self.assertEquals(9, len(monitor._openProblems)) + no = -1 + keys = sorted(list(monitor._openProblems.keys())) + for key in keys: + no += 1 + if no % 2 == 0: + del monitor._openProblems[key] + host = monitor._hosts['localhost'] + host._observed['processes'] = appl.Monitor.ObservedProperty('processes', 9999, 9999, None) + (newProblems, closedProblems) = monitor.observe() + self.assertEquals(5, len(newProblems)) + self.assertEquals(1, len(closedProblems)) + self.assertEquals(8, len(monitor._openProblems)) + self._logger.log('- openProblems:\n' + '\n'.join(monitor._openProblems.values())) + + def testExampleSiteCommand(self): + if debug: + return + nginxDir = os.path.dirname(os.path.dirname(__file__)) + '/configuration/sites' + appl.Monitor.main(['pymonitor', '-l-', '-v3', '--test-source-dir=' + nginxDir, + 'site', '--scan', nginxDir]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + lines = base.BaseTool.result() + if self.assertEquals(2, len(lines)): + self.assertEquals('site "monitor.infeos.eu" {\n\turl: http://monitor.infeos.eu:10116/works\n}', lines[0]) + self.assertEquals('site "wiki.hamatoma.de" {\n\turl: https://wiki.hamatoma.de/index.php?title=Hauptseite\n}', lines[1]) + + def testExampleCreateSiteServer(self): + if debug: + return + self.tempDir('sites-enabled', 'unittest') + self.tempDir('sites-available', 'unittest') + baseDir = self.tempDir('www', 'unittest') + tmpDir = self.tempDir('unittest') + appl.Monitor.main(['pymonitor', '-v3', '--test-target-dir=' + tmpDir, + 'create-site-server', '--ip=208.33.99.5', '--domain=gustl.example.com', '--overwrite']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + expected = '''server { + listen 80; + server_name gustl.example.com 208.33.99.5; + root /tmp/unittest/www/gustl.example.com; + location / { + allow all; + } +}''' + self.assertFileContent(tmpDir + '/sites-enabled/gustl.example.com', expected) + self.assertFileContent(tmpDir + '/sites-available/gustl.example.com', expected) + baseDir = baseDir + os.sep + 'gustl.example.com' + self.assertFileContains('<html>', baseDir + os.sep + 'index.html') + self.assertFileContains('<?php', baseDir + os.sep + 'index.php') + self.assertFileContains('<html>', baseDir + os.sep + 'index.php') + self.assertFileContains('gustl.example.com', baseDir + os.sep + 'domain.txt') + + def testExampleCreateSiteServerApache(self): + if debug: + return + self.clearDirectory(self.tempDir('unittest')) + self.tempDir('sites-enabled', 'unittest') + self.tempDir('sites-available', 'unittest') + baseDir = self.tempDir('www', 'unittest') + tmpDir = self.tempDir('unittest') + appl.Monitor.main(['pymonitor', '-v3', '--test-target-dir=' + tmpDir, + 'create-site-server', '--overwrite', '--apache']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + expectedDomain = base.BaseTool.BasicStatics.hostname(True) + expected = '''<VirtualHost *:80> + ServerName {} + ServerAlias 192.168.2.50 + ServerAdmin webmaster@localhost + DocumentRoot /tmp/unittest/www/{} + <Directory /tmp/unittest/www/{}> + AllowOverride all + Require all granted + Order allow,deny + allow from all + </Directory> +</VirtualHost>'''.format(expectedDomain, expectedDomain, expectedDomain) + self.assertFileContent(tmpDir + '/sites-enabled/{}.conf'.format(expectedDomain), expected) + self.assertFileContent(tmpDir + '/sites-available/{}.conf'.format(expectedDomain), expected) + baseDir = baseDir + os.sep + expectedDomain + self.assertFileContains('<html>', baseDir + os.sep + 'index.html') + self.assertFileContains('<?php', baseDir + os.sep + 'index.php') + self.assertFileContains('<html>', baseDir + os.sep + 'index.php') + self.assertFileContains(expectedDomain, baseDir + os.sep + 'domain.txt') + + def testExampleTest(self): + if debug: + return + appl.Monitor.main(['pymonitor', '-v3', 'test']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = MonitorTest() + tester.run() diff --git a/unittest/appl/RestoreToolTest.py b/unittest/appl/RestoreToolTest.py new file mode 100644 index 0000000..705e0d4 --- /dev/null +++ b/unittest/appl/RestoreToolTest.py @@ -0,0 +1,694 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +import sys +sys.path.insert(0, '/usr/share/pyrshell') +from unittest.UnitTestCase import UnitTestCase +import os +import time +import shutil +import re + +import base.StringUtils +import appl.RestoreTool +import appl.BackupTool + +def usage(msg=None): + return 'test usage' + +debug = False + +class RestoreToolTest(UnitTestCase): + + def __init__(self): + UnitTestCase.__init__(self) + self._targetDir = self.tempDir('trgdir', 'unittest') + + def makeTargetDir(self): + self.clearDirectory(self._targetDir) + return self._targetDir + + def _finish(self): + shutil.rmtree('/tmp/unittest', True) + shutil.rmtree('/tmp/restoretool', True) + + def makeTree(self): + text = 'Dies ist ein langer Text, der hoffentlich ein wenig komprimiert werden kann. Wenn nicht, kann man nichts machen' * 1000 + dirName = self.tempFile('srcdir', 'unittest') + os.makedirs(dirName, 0o777, True) + # /tmp/zippertest/srcdir: file[1-5].txt dir[123] .git linkConfig + # /tmp/zippertest/srcdir/dirX: levelX.txt subdir1 + # /tmp/zippertest/srcdir/dirX/subdir1: a.jpg b.txt c.odt + # /tmp/zippertest/srcdir/.git: config aa bb + # /tmp/zippertest/srcdir/.git/xx: xx1234 xx2345 xx3456 + for ix in range(1, 5+1): + fn = dirName + os.sep + 'file{:d}.txt'.format(ix) + base.StringUtils.toFile(fn, 'no{:d}: {:s}'.format(ix, text)) + for ix in range(1, 3+1): + subdir = dirName + os.sep + 'dir' + str(ix) + os.makedirs(subdir, 0o777, True) + fn = subdir + os.sep + 'level{:d}'.format(ix) + base.StringUtils.toFile(fn, 'level{:d}: {:s}'.format(ix, text)) + dir2 = subdir + os.sep + 'subdir1' + os.makedirs(dir2, 0o777, True) + for node in ['a.jpg', 'b.txt', 'c.odt']: + fn = dir2 + os.sep + node + base.StringUtils.toFile(fn, node + ': ' + text) + return dirName + + def makeBackup(self): + target = self.makeTargetDir() + options = self.getOptions() + options._configDir = self.makeBackupConfig() + options._errorFilter = 'Keine Berechtigung' + tool = appl.BackupTool.BackupTool(options) + base.FileHelper.ensureFileExists('/srv/.srv.exists') + tool.doBackup('dayly') + relPath = tool.relativeBackupPath('dayly') + rc = target + relPath + return rc + + def makeBackupConfig(self, clear=True): + baseDir = self.tempDir('config', 'backuptest') + if clear: + self.clearDirectory(baseDir) + sourceDir = self.makeTree() + targetDir = self.makeTargetDir() + appDataDir = self.tempDir('blog', 'backuptest') + base.StringUtils.toFile(appDataDir + os.sep + 'index.php', '<?php\ninfo();') + base.StringUtils.toFile(appDataDir + os.sep + 'new.php', '<?php\nnew();') + oldFile = appDataDir + os.sep + 'old.php' + base.StringUtils.toFile(oldFile, '<?php\nold();') + ftime = time.mktime((2018,1,2,3,4,5,0,0,0)) + os.utime(oldFile, (ftime, ftime)) + + appDir = self.ensureDirectory(baseDir + os.sep + 'webapps.d') + dirDir = self.ensureDirectory(baseDir + os.sep + 'dirs.d') + base.StringUtils.toFile(dirDir + os.sep + 'sys.etc.conf', ''' +directory=/etc/systemd +excluded= +''') + base.StringUtils.toFile(dirDir + os.sep + 'std.source.conf', ''' +directory={} +excluded= +'''.format(sourceDir)) + base.StringUtils.toFile(baseDir + os.sep + 'backup.conf', ''' +target.path={:s} +job.dayly=job.sysinfo job.sys job.full +job.sys=&saveDirByTar @%etc%/dirs.d/sys.*.conf:directory:excluded +job.full=&saveDirByZip @%etc%/dirs.d/std.*.conf:directory +job.sysinfo=&systemInfo +# Reading metadata from mysql: +mysql.user=backup +mysql.code=backup2backup +# Email data: +# Receiver of the error messages: may be a blank separated list of email addresses +send.always=False +location=unittest +admin.email=hm.neutral@gmx.de +smtp.host=smtp.gmx.de +smtp.port=587 +smtp.sender=hm.unittest@gmx.de +smtp.user=hm.unittest@gmx.de +smtp.code=33S.e.c.r.e.t +smtp.tls=True +'''.format(targetDir, appDir)) + appl.BackupTool.BackupTool.createDirsD(dirDir, None, False) + return baseDir + + def makeMinimalConfigDir(self): + dirName = self.tempFile('pyrshell', 'etc') + os.makedirs(dirName, 0o777, True) + base.StringUtils.toFile(dirName + os.sep + 'backup.conf', '') + return dirName + + def getOptions(self): + appInfo = base.BaseTool.ApplicationInfo('wiki', 'appl.WikiTool', usage) + rc = base.BaseTool.GlobalOptions(appInfo) + rc._logFiles.append('-') + return rc + + def testRestoreByTar(self): + if debug: + return + target = self.makeBackup() + archive = target + os.sep + 'etc=systemd.tgz' + options = self.getOptions() + tool = appl.RestoreTool.RestoreTool(options) + target2 = self.tempDir('trgdir2', 'unittest') + fn = target2 + os.sep + 'toDelete.txt' + base.StringUtils.toFile(fn, 'to delete') + dir2 = target2 + os.sep + 'dir.to.remove' + tool.ensureDirectory(dir2) + tool.restoreDirectoryByTar(archive, target2, None, True) + self.assertFileNotExists(fn) + self.assertFileNotExists(dir2) + self.assertFileExists(target2 + os.sep + 'system.conf') + + def testRestoreByZip(self): + if debug: + return + target = self.makeBackup() + archive = target + os.sep + 'tmp=unittest=srcdir.zip' + options = self.getOptions() + tool = appl.RestoreTool.RestoreTool(options) + target2 = self.tempDir('trgdir2', 'unittest') + fn = target2 + os.sep + 'toDelete.txt' + base.StringUtils.toFile(fn, 'to delete') + dir2 = target2 + os.sep + 'dir.to.remove' + tool.ensureDirectory(dir2) + tool.restoreDirectoryByZip(archive, target2, None) + self.assertFileNotExists(fn) + self.assertFileNotExists(dir2) + self.assertFileExists(target2 + os.sep + 'file2.txt') + self.assertFileExists(target2 + os.sep + 'dir2' + os.sep + 'level2') + + def testMergeUsers(self): + if False and debug: + return + self._finish() + options = self.getOptions() + dirSource = self.tempDir('etc', 'unittest') + dirSecond = self.tempDir('etc2', 'unittest') + options._testSourceDir = os.path.dirname(dirSource) + options._testTargetDir = os.path.dirname(dirSource) + tool = appl.RestoreTool.RestoreTool(options) + fnPasswd = dirSource + os.sep + 'passwd' + base.StringUtils.toFile(fnPasswd, '''# test passwd +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +wk:x:1003:1003:,,,:/home/wk:/bin/bash +redis:x:129:136::/var/lib/redis:/usr/sbin/nologin +''') + base.StringUtils.toFile(dirSource + os.sep + 'shadow', '''# test shadow +root:$6$ovR8mtHu$RXx8SzG1k5fYe9DIecA3nM7744otO9lyVwCUIKo.gMY0LziEb53dRRQRCyGlNBFqx/XULgrXQfDuYc59REHdY.:17732:0:99999:7::: +daemon:*:17708:0:99999:7::: +bin:*:17708:0:99999:7::: +wk:$6$Fz9IlJZv$c7BY4/D5LeJrXpd.313wBsH3hAwU9xyPe7p322nf3g3X4kNO1dkLoh32rWEeIRCFqIt/MgglOBD/UGUoMR6rJ.:17777:0:99999:7::: +redis:*:17962:0:99999:7::: +''') + base.StringUtils.toFile(dirSource + os.sep + 'group', '''# test group +root:x:0: +daemon:x:1: +bin:x:2: +root:x:0: +daemon:x:1: +bin:x:2: +''') + # === Merge files: + base.StringUtils.toFile(dirSecond + os.sep + 'passwd', '''# test passwd +bin2:x:502:502:bin:/bin:/usr/sbin/nologin +wk:x:1003:1003:,,,:/home/wk:/bin/bash +user1:x:129:136::/var/lib/redis:/usr/sbin/nologin +''') + base.StringUtils.toFile(dirSecond + os.sep + 'shadow', '''# test shadow +bin2:*:17708:0:99999:7::: +wk:$6$Fz9IlJZv$c7BY4/D5LeJrXpd.313wBsH3hAwU9xyPe7p322nf3g3X4kNO1dkLoh32rWEeIRCFqIt/MgglOBD/UGUoMR6rJ.:17777:0:99999:7::: +user1:*:17962:0:99999:7::: +''') + base.StringUtils.toFile(dirSecond + os.sep + 'group', '''# test group +bin2:x:502: +wk:x:1003: +group1:x:2:user1 +''') + tool.restoreMergeUsersAndGroups(dirSecond) + dirTarget = dirSource + expected = '''# test passwd +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +wk:x:1003:1003:,,,:/home/wk:/bin/bash +redis:x:129:136::/var/lib/redis:/usr/sbin/nologin +bin2:x:502:502:bin:/bin:/usr/sbin/nologin +user1:x:129:136::/var/lib/redis:/usr/sbin/nologin''' + current = base.StringUtils.fromFile(dirTarget + os.sep + 'passwd') + self.assertEquals(expected, current) + expected = '''# test shadow +root:$6$ovR8mtHu$RXx8SzG1k5fYe9DIecA3nM7744otO9lyVwCUIKo.gMY0LziEb53dRRQRCyGlNBFqx/XULgrXQfDuYc59REHdY.:17732:0:99999:7::: +daemon:*:17708:0:99999:7::: +bin:*:17708:0:99999:7::: +wk:$6$Fz9IlJZv$c7BY4/D5LeJrXpd.313wBsH3hAwU9xyPe7p322nf3g3X4kNO1dkLoh32rWEeIRCFqIt/MgglOBD/UGUoMR6rJ.:17777:0:99999:7::: +redis:*:17962:0:99999:7::: +# test shadow +bin2:*:17708:0:99999:7::: +user1:*:17962:0:99999:7::: +''' + current = base.StringUtils.fromFile(dirTarget + os.sep + 'shadow') + self.assertEquals(expected, current) + expected = '''# test group +root:x:0: +daemon:x:1: +bin:x:2: +root:x:0: +daemon:x:1: +bin:x:2: +bin2:x:502: +wk:x:1003: +group1:x:2:user1 +''' + current = base.StringUtils.fromFile(dirTarget + os.sep + 'group') + self.assertEquals(expected, current) + + def testInitNetStatic(self): + if debug: + return + tempDir = self.tempDir('trgdir', 'unittest') + fn = tempDir + os.sep + 'interfaces' + base.StringUtils.toFile(fn, '''# interfaces(5) file used by ifup(8) and ifdown(8) +auto lo +iface lo inet loopback + +auto wlan0 +iface wlan0 inet dhcp +''') + appl.RestoreTool.main(['restoretool', '--test-target-dir={}'.format(self._targetDir), 'net', 'static']) + self.assertFileContains('auto enp2s0', fn) + self.assertFileContains('iface enp2s0 inet static', fn) + self.assertFileContains(' address 10.10.10.100', fn) + self.assertFileContains(' netmask 255.0.0.0', fn) + self.assertFileContains(' gateway 10.10.10.1', fn) + + def testInitNetStatic2(self): + if debug: + return + fn = self._targetDir + os.sep + 'interfaces' + base.StringUtils.toFile(fn, '''# interfaces(5) file used by ifup(8) and ifdown(8) +auto lo +iface lo inet loopback +iface enp2s0 inet static + address 192.168.3.100 + netmask 255.255.255.0 + gateway 192.168.3.1 +auto wlan0 +iface wlan0 inet dhcp +''') + appl.RestoreTool.main(['restoretool', '--test-target-dir={}'.format(self._targetDir), + 'net', 'static', '101']) + self.assertFileContains('auto enp2s0', fn) + self.assertFileContains('iface enp2s0 inet static', fn) + self.assertFileContains(' address 10.10.10.101', fn) + self.assertFileContains(' netmask 255.0.0.0', fn) + self.assertFileContains(' gateway 10.10.10.1', fn) + + def testInitNetStatic3(self): + if debug: + return + fn = self._targetDir + os.sep + 'interfaces' + base.StringUtils.toFile(fn, '''# interfaces(5) file used by ifup(8) and ifdown(8) +auto lo +iface lo inet loopback +iface wlp4s0 inet static + address 192.168.3.100 + netmask 255.255.255.0 + gateway 192.168.3.1 +auto wlan0 +iface wlan0 inet dhcp +''') + appl.RestoreTool.main(['restoretool', '--test-target-dir={}'.format(self._targetDir), + 'net', 'static', '25', 'w*']) + self.assertFileContains('auto wlp4s0', fn) + self.assertFileContains('iface wlp4s0 inet static', fn) + self.assertFileContains(' address 10.10.10.25', fn) + self.assertFileContains(' netmask 255.0.0.0', fn) + self.assertFileContains(' gateway 10.10.10.1', fn) + + def testInitNetStatic4(self): + if debug: + return + fn = self._targetDir + os.sep + 'interfaces' + base.StringUtils.toFile(fn, '''# empty +''') + appl.RestoreTool.main(['restoretool', '--test-target-dir={}'.format(self._targetDir), + 'net', 'static', '192.25', 'e*']) + self.assertFileContains('auto enp2s0', fn) + self.assertFileContains('iface enp2s0 inet static', fn) + self.assertFileContains(' address 192.10.10.25', fn) + self.assertFileContains(' netmask 255.255.255.0', fn) + self.assertFileContains(' gateway 192.10.10.1', fn) + + def testNetNameserver(self): + if debug: + return + tmpDir = self.tempDir('unittest') + shutil.copy2('/etc/systemd/resolved.conf', tmpDir) + appl.RestoreTool.main(['restoretool', '--test-target-dir={}'.format(tmpDir), + 'net', 'nameserver', '192.168.178.254']) + self.assertFileContains('DNS=192.168.178.254', tmpDir + os.sep + 'resolved.conf') + + def testNetNameserver2(self): + if debug: + return + tmpDir = self.tempDir('unittest') + shutil.copy2('/etc/systemd/resolved.conf', tmpDir) + appl.RestoreTool.main(['restoretool', '--test-target-dir={}'.format(tmpDir), + 'net', 'nameserver']) + self.assertFileContains('DNS=9.9.9.9', tmpDir + os.sep + 'resolved.conf') + + def testExampleStorageDisk1(self): + if debug: + return + appl.RestoreTool.main(['restoretool', 'storage', 'disks', 'sd[a-f]']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + lines = tool._output + if self.assertTrue(lines != None and len(lines) > 0): + for line in lines: + self.assertMatches(r'UUID', line) + + def testExampleStorageAutoFsDisk(self): + if debug: + return + tmpDir = self.tempDir('etc', 'unittest') + appl.RestoreTool.main(['restoretool', '--test-target-dir=' + tmpDir, + 'storage', 'autofs', 'usb-backup', '/', 'disk|*|/media/usb-backup|rw', '--auto-select=sd[a].*ntfs']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + fn = tmpDir + os.sep + 'auto.master' + self.assertFileContains(re.compile(r'^/-.*auto.usb-backup'), fn) + fn = tmpDir + os.sep + 'auto.usb-backup' + self.assertFileContains(re.compile(r'^[0-9A-Fa-f-]+\s+-fstype=\w+,rw,umask=002\s+:/media/usb-backup'), fn) + + def testExampleInitApache(self): + if debug: + return + tmpDir = self.tempDir('etc', 'unittest') + fn = tmpDir + os.sep + 'ports.conf' + base.StringUtils.toFile(fn, '''# If you just change the port or add more ports here, you will likely also +# have to change the VirtualHost statement in +# /etc/apache2/sites-enabled/000-default.conf +Listen 80 +<IfModule ssl_module> + Listen 443 +</IfModule> +<IfModule mod_gnutls.c> + Listen 443 +</IfModule> +''') + appl.RestoreTool.main(['restoretool', '-v3', '--test-target-dir=' + tmpDir, + 'init', 'apache', '--ports=81,444']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileContains('Listen 81', fn) + self.assertFileContains('Listen 444', fn) + + def testExampleInitNginx(self): + if debug: + return + tmpDir = self.tempDir('etc', 'unittest') + dirWellKnown = self.tempDir('letsencrypt', 'unittest') + fn = tmpDir + os.sep + 'letsencrypt.conf' + self.ensureFileDoesNotExist(fn) + appl.RestoreTool.main(['restoretool', '-v3', '--test-target-dir=' + tmpDir, + 'init', 'nginx', '--well-known=' + dirWellKnown]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertDirExists(dirWellKnown + os.sep + '.well-known') + self.assertFileContains('location ^~ /.well-known/acme-challenge/', fn) + self.assertFileContains(re.compile(r'root\s+' + dirWellKnown), fn) + + def testExampleNetInfo(self): + if debug: + return + lines = base.BaseTool.result() + appl.RestoreTool.main(['restoretool', 'net', 'info']) + if self.assertTrue(lines != None and len(lines) > 0): + self.assertTrue(lines[0].startswith('sda3')) + + def testExampleBtrFsCreateFsLoop(self): + if debug: + return + if not self._isRoot: + self._logger.log('not root. ignoring testExampleBtrFsCreateFsLoop') + else: + tempDir = self.tempDir('storage', 'unittest') + base.StringUtils.toFile(tempDir + os.sep + 'auto.master', '# auto.master') + image = tempDir + os.sep + 'fs.unittest.img' + appl.RestoreTool.main(['restoretool', '-v3', '--test-target-dir=' + tempDir, + 'btrfs', 'create-fs', '/dev/loop33', 'fs.unittest', '--image={}:150M'.format(image), '--force']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + fn = tempDir + os.sep + 'auto.btrfs.fs.unittest' + self.assertFileContains('LABEL=fs.unittest\t-fstype=btrfs\t:fs.unittest', fn) + self.assertFileContains(re.compile(r'/-\s+' + fn), tempDir + os.sep + 'auto.master') + + def testExampleBtrCreateFsMove(self): + if debug: + return + if not self._isRoot: + self._logger.log('not root: ignoring testExampleBtrCreateFsMove') + else: + tempDir = self.tempDir('storage', 'unittest') + self.ensureDirectory(tempDir + os.sep + 'fs.unittest') + fnTab = tempDir + os.sep + 'fstab' + base.StringUtils.toFile(fnTab, '''# FSTab +/dev/sda2 /boot ext4 defaults,noatime 0 2 +LABEL=swap swap swap defaults,noatime 0 2 +UUID=74a63fee-8fca-412e-9533-4d02547fe978 / btrfs subvol=@,defaults,noatime,space_cache,autodefrag 0 1 +UUID=74a63fee-8fca-412e-9533-4d02547fe978 /home btrfs subvol=@home,defaults,noatime,space_cache,autodefrag 0 2 +''') + appl.RestoreTool.main(['restoretool', '-v3', '--test-target-dir=' + tempDir, + 'btrfs', 'create-subvol', 'fs.unittest', 'db', tempDir + os.sep + 'db', '--options=nodatacow', '--move-files', '--force']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + line = 'UUID=74a63fee-8fca-412e-9533-4d02547fe978 /tmp/unittest/storage/db btrfs subvol=db,defaults,noatime,space_cache,autodefrag,nodatacow 0 0' + self.assertFileContains(line, fnTab) + + def testExampleInitGrub(self): + if debug: + return + fnGrub = self.tempFile('grub.cfg', 'unittest', 'grub') + base.StringUtils.toFile(fnGrub, '''# Dummy grub file +menuentry 'Buster XFCE on /dev/nvme0n1p6 NVSAMS' --class debian --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-simple-381089c0-2913-4623-9358-1ad56766b80b' { + # Text 1 +} +menuentry 'Entry2' { + # Text 2 +} +''', self._logger) + fnBlkId = self.tempFile('blkid.out', 'unittest', 'grub') + base.StringUtils.toFile(fnBlkId, '''# Dummy output of lsblk +/dev/nvme0n1p1: LABEL="SYSTEM" UUID="184C-BC88" TYPE="vfat" PARTLABEL="EFI system partition" PARTUUID="801ef4fd-3afc-4378-b112-296be3955024" +/dev/nvme0n1p3: LABEL="Windows" UUID="FA844F35844EF3A3" TYPE="ntfs" PARTLABEL="Basic data partition" PARTUUID="e7239890-b477-4d50-aa82-4fb6de1f5205" +/dev/nvme0n1p6: LABEL="nvsams" UUID="381089c0-2913-4623-9358-1ad56766b80b" UUID_SUB="7ea15cb3-6a84-48f7-8a61-ffadc21733b8" TYPE="btrfs" PARTLABEL="Basic data partition" PARTUUID="99c411d2-6660-4f02-b855-aa189fb71daf" +/dev/nvme0n1p7: LABEL="WinRE_DRV" UUID="C8C650B7C650A808" TYPE="ntfs" PARTLABEL="Basic data partition" PARTUUID="4891912c-4d85-4b3f-9fea-00be938a0f57" +/dev/sda3: LABEL="source" UUID="16BE0656BE062F35" TYPE="ntfs" PARTUUID="979453a9-03" +/dev/sda6: LABEL="system" UUID="7a497274-d30b-4860-a06b-c94e83d8df76" UUID_SUB="229f5912-31b6-4f70-a18f-063742429e23" TYPE="btrfs" PARTUUID="979453a9-06" +/dev/nvme0n1: PTUUID="d3966240-6a16-484c-9d3f-498c9dcdf296" PTTYPE="gpt" +/dev/nvme0n1p2: PARTLABEL="Microsoft reserved partition" PARTUUID="372f5f3e-1ac7-492c-9bc8-ba73e132e358" +/dev/sda5: PARTUUID="979453a9-05" +''', self._logger) + appl.RestoreTool.main(['restoretool', '-v3', '--test-target-dir=' + os.path.dirname(fnGrub), + 'init', 'grub']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + current = base.StringUtils.fromFile(fnGrub, '\n') + self.assertEquals('''# Dummy grub file +menuentry 'Buster XFCE on /dev/nvme0n1p6 NVSAMS' --class debian --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-simple-381089c0-2913-4623-9358-1ad56766b80b' { + # Text 1 +} +menuentry 'Entry2' { + # Text 2 +} +'''.split('\n'), current) + + def testExampleRestoreClone(self): + if debug: + return + self._finish() + tar = '/usr/share/pyrshell/unittest/data/example.tgz' + target = self.tempDir('bin', 'unittest') + target2 = os.path.dirname(target) + os.sep + 'bin2' + self.ensureFileDoesNotExist(target2) + appl.RestoreTool.main(['restoretool', '-v3', + 'restore', 'clone', tar, target]) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertDirExists(target2) + self.assertFileExists(target2 + os.sep + 'All.sh') + + def testExampleRestorePyrshell(self): + if debug: + return + self._finish() + tar = '/usr/share/pyrshell/unittest/data/etc.tgz' + target = self.tempDir('unittest') + target1 = self.tempDir('etc', 'unittest') + target2 = self.tempDir('etc2', 'unittest') + self.ensureFileDoesNotExist(target2) + appl.RestoreTool.main(['restoretool', '-v3', + 'restore', 'clone', tar, target1]) + appl.RestoreTool.main(['restoretool', '-v3', + '--test-source-dir={}'.format(os.path.dirname(target2)), + '--test-target-dir={}'.format(os.path.dirname(target1)), + 'restore', 'pyrshell']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + target3 = self.tempDir('pyrshell', 'unittest') + self.assertFileExists(target3 + os.sep + 'backup.conf') + self.assertDirExists(target3 + os.sep + 'webapps.d') + unwanted = target3 + os.sep + 'unwanted.data' + base.StringUtils.toFile(unwanted, 'unwanted') + appl.RestoreTool.main(['restoretool', '-v3', + '--test-source-dir={}'.format(os.path.dirname(target2)), + '--test-target-dir={}'.format(os.path.dirname(target1)), + 'restore', 'pyrshell']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileNotExists(target3 + os.sep + 'unwanted.data') + + def testExampleRestoreNginx(self): + if debug: + return + self._finish() + tar = '/usr/share/pyrshell/unittest/data/etc.tgz' + target = self.tempDir('unittest') + target1 = self.tempDir('etc', 'unittest') + source = self.tempDir('etc2', 'unittest') + self.ensureFileDoesNotExist(source) + appl.RestoreTool.main(['restoretool', '-v3', + 'restore', 'clone', tar, target1]) + base.FileHelper.unpack('/usr/share/pyrshell/unittest/data/etc.work.tgz', target, self._logger) + appl.RestoreTool.main(['restoretool', '-v3', + '--test-source-dir={}'.format(os.path.dirname(source)), + '--test-target-dir={}'.format(os.path.dirname(target1)), + 'restore', 'nginx']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + target = target1 + os.sep + 'nginx' + self.assertFileExists(target + os.sep + 'nginx.conf') + self.assertFileExists(target + os.sep + 'snippets/letsencrypt.conf') + self.assertFileExists(target + os.sep + 'sites-enabled/life.caribou') + self.assertFileExists(target + os.sep + 'sites-available/life.caribou') + target = target1 + os.sep + 'ssl' + self.assertFileExists(target + os.sep + 'private/wiki.hamatoma.de.key') + self.assertFileExists(target + os.sep + 'certs/wiki.hamatoma.de.pem') + + def testExampleRestoreEtc(self): + if debug: + return + self._finish() + target = self.tempDir('unittest') + tar = '/usr/share/pyrshell/unittest/data/etc.tgz' + target1 = self.tempDir('etc', 'unittest') + source = self.tempDir('unittest') + appl.RestoreTool.main(['restoretool', '-v3', + 'restore', 'clone', tar, target1]) + base.FileHelper.unpack('/usr/share/pyrshell/unittest/data/etc.work.tgz', target, self._logger) + appl.RestoreTool.main(['restoretool', '-v3', + '--test-source-dir={}'.format(source), + '--test-target-dir={}'.format(target), + 'restore', 'etc']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + target = target1 + os.sep + 'nginx' + self.assertFileExists(target + os.sep + 'nginx.conf') + self.assertFileExists(target + os.sep + 'snippets/letsencrypt.conf') + self.assertFileExists(target + os.sep + 'sites-enabled/life.caribou') + self.assertFileExists(target + os.sep + 'sites-available/life.caribou') + target = target1 + os.sep + 'ssl' + self.assertFileExists(target + os.sep + 'private/wiki.hamatoma.de.key') + self.assertFileExists(target + os.sep + 'certs/wiki.hamatoma.de.pem') + dirTarget = self.tempDir('etc', 'unittest') + expected = '''# test passwd +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +wk:x:1003:1003:,,,:/home/wk:/bin/bash +redis:x:129:136::/var/lib/redis:/usr/sbin/nologin +bin2:x:502:502:bin:/bin:/usr/sbin/nologin +user1:x:129:136::/var/lib/redis:/usr/sbin/nologin''' + current = base.StringUtils.fromFile(dirTarget + os.sep + 'passwd') + self.assertEquals(expected, current) + + def testExampleRestoreLetsencryptOff(self): + if debug: + return + self._finish() + tempDir = self.tempDir('sites-enabled', 'unittest') + domain = 'www.example.de' + fn = tempDir + os.sep + domain + base.StringUtils.toFile(fn, '''# with + ssl_certificate /etc/letsencrypt/live/latest/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/latest/privkey.pem; + #ssl_certificate /etc/ssl/certs/www.example.de.pem; + #ssl_certificate_key /etc/ssl/private/www.example.de.key; +''') + appl.RestoreTool.main(['restoretool', '-v3', + '--test-target-dir={}'.format(tempDir), + 'reconfigure', 'letsencrypt', 'off', domain]) + self.assertFileContent(fn, '''# with +# ssl_certificate /etc/letsencrypt/live/latest/fullchain.pem; +# ssl_certificate_key /etc/letsencrypt/live/latest/privkey.pem; + ssl_certificate /etc/ssl/certs/www.example.de.pem; + ssl_certificate_key /etc/ssl/private/www.example.de.key; +''') + def testExampleRestoreLetsencryptOn(self): + if debug: + return + self._finish() + tempDir = self.tempDir('sites-enabled', 'unittest') + domain = 'www.example.de' + fn = tempDir + os.sep + domain + base.StringUtils.toFile(fn, '''# with + ## ssl_certificate /etc/letsencrypt/live/latest/fullchain.pem; + ## ssl_certificate_key /etc/letsencrypt/live/latest/privkey.pem; + ssl_certificate /etc/ssl/certs/www.example.de.pem; + ssl_certificate_key /etc/ssl/private/www.example.de.key; +''') + appl.RestoreTool.main(['restoretool', '-v3', + '--test-target-dir={}'.format(tempDir), + 'reconfigure', 'letsencrypt', 'on', domain]) + self.assertFileContent(fn, '''# with + ssl_certificate /etc/letsencrypt/live/latest/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/latest/privkey.pem; +# ssl_certificate /etc/ssl/certs/www.example.de.pem; +# ssl_certificate_key /etc/ssl/private/www.example.de.key; +''') + + def testExampleRestoreLetsencryptAll(self): + if debug: + return + self._finish() + tempDir = self.tempDir('sites-enabled', 'unittest') + domain = 'www.example.de' + fn = tempDir + os.sep + domain + base.StringUtils.toFile(fn, '''# with + ## ssl_certificate /etc/letsencrypt/live/latest/fullchain.pem; + ## ssl_certificate_key /etc/letsencrypt/live/latest/privkey.pem; + ssl_certificate /etc/ssl/certs/www.example.de.pem; + ssl_certificate_key /etc/ssl/private/www.example.de.key; +''') + appl.RestoreTool.main(['restoretool', '-v3', + '--test-target-dir={}'.format(tempDir), + 'reconfigure', 'letsencrypt', 'on', '--all']) + self.assertFileContent(fn, '''# with + ssl_certificate /etc/letsencrypt/live/latest/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/latest/privkey.pem; +# ssl_certificate /etc/ssl/certs/www.example.de.pem; +# ssl_certificate_key /etc/ssl/private/www.example.de.key; +''') + + def testLetsencryptAdapt(self): + if debug: + return + live = self.tempDir('live', 'unittest') + huber1 = live + os.sep + 'huber.de.01' + self.ensureDirectory(huber1) + time.sleep(1) + self.ensureDirectory(live + os.sep + 'huber.de.02') + latest = live + os.sep + 'latest' + self.ensureFileDoesNotExist(latest) + os.symlink('huber.de.01', latest) + options = self.getOptions() + tool = appl.RestoreTool.RestoreTool(options) + tool._globalOptions._testTargetDir = os.path.dirname(live) + tool.reconfigureLetsencryptAdapt() + self.assertEquals('huber.de.02', os.readlink(latest)) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = RestoreToolTest() + tester.run() diff --git a/unittest/appl/SvgToolTest.py b/unittest/appl/SvgToolTest.py new file mode 100644 index 0000000..ce342b2 --- /dev/null +++ b/unittest/appl/SvgToolTest.py @@ -0,0 +1,176 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase + +import appl.SvgTool +import base.BaseTool + +debug = False + +def usage(msg=None): + return 'test usage' + +class SvgToolTest(UnitTestCase): + + def getOptions(self): + appInfo = base.BaseTool.ApplicationInfo('svgtool', 'appl.SvgTool', usage) + rc = base.BaseTool.GlobalOptions(appInfo) + rc._logFiles.append('-') + return rc + def stringToFloatArray(self, text): + rc = [] + for y in text.split(';'): + rc.append(float(y)) + return rc + def floatArrayToString(self, data): + rc = '' + for y in data: + rc += str(y) + ';' + return rc[0:-1] + + def makeSet(self, xValues, yValues): + rc = [] + xCol = appl.SvgTool.Column('x', rc) + yCol = appl.SvgTool.Column('y', rc) + for x in xValues.split(';'): + # time or datetime values will be untouched: + xCol._values.append(x if x.find(':') >= 0 else int(x)) + yCol._values = self.stringToFloatArray(yValues) + rc.append(xCol) + rc.append(yCol) + return rc + + def makeTool(self, xValues, yValues): + options = self.getOptions() + tool = appl.SvgTool.SvgTool(options) + tool._columns = self.makeSet(xValues, yValues) + return tool + + def dumpSet(self, array): + xValues = '' + yValues = '' + self.assertEquals(2, len(array)) + self.assertEquals(len(array[0]._values), len(array[1]._values)) + for ix in range(len(array[0]._values)): + xValues += ';{}'.format(array[0]._values[ix]) + yValues += ';{}'.format(array[1]._values[ix]) + return [xValues, yValues] + + def testShrinkData(self): + if debug: + return + tool = self.makeTool('1;2;3;4;5;6;7;8;9;10;11;12;13;14;15', '-1;2;0;0;-2;3;5;5;5;-3;4;0;0;0;2') + array = tool.shrinkData(5) + [x,y] = self.dumpSet(array) + self.assertEquals(';-1.0;2.0;5.0;-3.0;2.0', y) + self.assertEquals(';1;4.5;8.0;11.5;15.0', x) + tool = self.makeTool('2019-02-03 00:11;2019.02.04T12:13:14', '-1;2') + fn = self.tempFile('data.csv', 'svgtest') + fn2 = self.tempFile('data.html', 'svgtest') + tool.putCsv(fn) + argv = [fn, fn2] + tool.diagram(argv, None) + + def testNormalize(self): + if debug: + return + tool = self.makeTool('1;2;3;4;5;6;7;8;9;10', '10;20;10;20;90;30;20;30;20;100') + fn = self.tempFile('data.csv', 'svgtest') + tool.putCsv(fn) + fn2 = self.tempFile('data.html', 'svgtest') + tool.putCsv(fn) + argv = [fn, fn2] + tool.diagram(argv, None) + + def testMinMax(self): + if debug: + return + tool = self.makeTool('1;12;13;14;15;16;17;18;19;26', '3;-12;-13;-14;-15;-16;-17;-18;-19;-26') + tool._columns[0].findMinMax(80, 1.1); + self.assertEquals(18, tool._columns[0]._max) + self.assertEquals(13, tool._columns[0]._min) + tool._columns[1].findMinMax(80, 1.1); + self.assertEquals(-13.0, tool._columns[1]._max) + self.assertEquals(-18.0, tool._columns[1]._min) + + def testMinMaxSpreadFactor(self): + if debug: + return + tool = self.makeTool('1;2;3;4;5;6;7;8;9;11', '-11;-12;-13;-14;-15;-16;-17;-18;-19;-21') + tool._columns[0].findMinMax(80, 1.401); + self.assertEquals(8, tool._columns[0]._max) + self.assertEquals(1, tool._columns[0]._min) + tool._columns[1].findMinMax(80, 1.401); + self.assertEquals(-11.0, tool._columns[1]._max) + self.assertEquals(-18.0, tool._columns[1]._min) + + def testDiagram(self): + if debug: + return + fnCsv = self.tempFile('sinus.csv') + fnHtml = self.tempFile('sinus.html') + self.ensureFileDoesNotExist(fnCsv) + self.ensureFileDoesNotExist(fnHtml) + appl.SvgTool.main(['svgtool', 'example']) + self.assertFileExists(fnCsv) + self.assertFileContains('x;sin', fnCsv) + self.assertFileContains('12.30726762886598;-0.2562135827507867;0.9666201942924666;-0.2650612766665054', fnCsv) + appl.SvgTool.main(['svgtool', '-v3', 'x-y-diagram', fnCsv, fnHtml, '--width=512']) + self.assertFileContains('x;sin', fnCsv) + self.assertFileContains('</html>', fnHtml) + self.assertFileContains('<line x1="12" y1="97" x2="18" y2="97" stroke="green" stroke-width="1" />', fnHtml) + + def testConvertToMovingAverage(self): + if debug: + return + tool = self.makeTool('1', '10') + + dataAsString = '10;30;50;160;200;300' + data = self.stringToFloatArray(dataAsString) + tool.convertToMovingAverage(data, 4) + current = self.floatArrayToString(data) + self.assertEquals('30.0;62.5;110.0;177.5;220.0;250.0', current) + + dataAsString = '10;20;30;40;50;160;170;180;190;200;300' + data = self.stringToFloatArray(dataAsString) + tool.convertToMovingAverage(data, 3) + current = self.floatArrayToString(data) + self.assertEquals('15.0;20.0;30.0;40.0;83.33333333333333;126.66666666666667;170.0;180.0;190.0;230.0;250.0', current) + + def testDiagramMovingAverage(self): + if debug: + return + fnCsv = self.tempFile('sinus.csv') + fnHtml = self.tempFile('sinus.html') + self.ensureFileDoesNotExist(fnCsv) + self.ensureFileDoesNotExist(fnHtml) + appl.SvgTool.main(['svgtool', 'example', '12']) + self.assertFileExists(fnCsv) + self.assertFileContains('x;sin', fnCsv) + self.assertFileContains('36.92180288659794;-0.7013637757662801;0.7128035171369929;-0.9839510593092174', fnCsv) + appl.SvgTool.main(['svgtool', '-v3', 'x-y-diagram', fnCsv, fnHtml, '--moving-average=201']) + self.assertFileContains('x;sin', fnCsv) + self.assertFileContains('</html>', fnHtml) + self.assertFileContains('<tr style="color: blue"><td>x:</td><td></td><td>0.00</td><td>37.62</td><td>485 Werte</td></tr>', fnHtml) + self.assertFileContains('<tr style="color: black"><td>sin(x):</td><td>0.000092</td><td>-0.215542</td><td>0.213731</td><td></td></tr>', fnHtml) + self.assertFileContains('<tr style="color: red"><td>cos(x):</td><td>-0.002358</td><td>-0.113892</td><td>0.132246</td><td></td></tr>', fnHtml) + self.assertFileContains('<tr style="color: green"><td>tan(x):</td><td>0.000221</td><td>-0.152102</td><td>0.150606</td><td></td></tr>', fnHtml) + + def testConvertToMovingAverage2(self): + if False and debug: + return + tool = self.makeTool('1', '10') + + dataAsString = '10;30;50;0;0;0;0;200;300' + data = self.stringToFloatArray(dataAsString) + tool.convertToMovingAverage(data, 3) + current = self.floatArrayToString(data) + self.assertEquals('20.0;30.0;26.666666666666668;16.666666666666668;0.0;0.0;66.66666666666667;166.66666666666666;250.0', current) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = SvgToolTest() + tester.run() diff --git a/unittest/appl/TextToolTest.py b/unittest/appl/TextToolTest.py new file mode 100644 index 0000000..c390e1d --- /dev/null +++ b/unittest/appl/TextToolTest.py @@ -0,0 +1,812 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase + +import os +import shutil +import re +import time + +import appl.TextTool +import base.BaseTool +import base.StringUtils + +debug = False + +def usage(msg=None): + return 'test usage' + +class TextToolTest(UnitTestCase): + def __init__(self): + UnitTestCase.__init__(self) + self._finish() + base.FileHelper.clearDirectory(self.tempDir('unittest')) + self._tempDir = self.tempDir('texttool', 'unittest') + self._fn = self._tempDir + os.sep + 'first.txt' + base.StringUtils.toFile(self._fn, "line 1\nline 2\nThis text is in line 3") + self._fn2 = self._tempDir + os.sep + 'second.txt' + base.StringUtils.toFile(self._fn2, "How are you") + self._tool = appl.TextTool.TextTool(self.getOptions()) + + def _finish(self): + shutil.rmtree(self.tempDir('unittest')) + + def getOptions(self): + appInfo = base.BaseTool.ApplicationInfo('wiki', 'appl.WikiTool', usage) + rc = base.BaseTool.GlobalOptions(appInfo, 1) + rc._exampleText = 'log.file=/var/log/local/wikitool.log' + rc._logFiles.append('-') + rc._verboseLevel = 0 + return rc + + def testGrep(self): + if debug: + return + appl.TextTool.main(['texttool', + '-v4', + 'grep', + 'line|How', + self.tempFile('*.txt', 'unittest')]) + result = base.BaseTool.result() + self.assertEquals("{}: How are you\n{}: line 1\n{}: line 2\n{}: This text is in line 3".format(self._fn2, self._fn, self._fn, self._fn), result) + + def testGrepMiddleOfTheLine(self): + if debug: + return + appl.TextTool.main(['texttool', + '-v4', + 'grep', + 'ine|ow', + self.tempFile('*.txt', 'unittest')]) + result = base.BaseTool.result() + self.assertEquals("{}: How are you\n{}: line 1\n{}: line 2\n{}: This text is in line 3".format(self._fn2, self._fn, self._fn, self._fn), result) + + def testGrepExcludedFile(self): + if debug: + return + appl.TextTool.main(['texttool', + '-v4', + 'grep', + 'How', + self.tempFile('*.txt', 'unittest'), + '--excluded=:f*']) + result = base.BaseTool.result() + self.assertEquals("{}: How are you".format(self._fn2), result) + + def testGrepExcludedDir(self): + if debug: + return + appl.TextTool.main(['texttool', + '-v4', + '--exit-with-0', + 'grep', + 'How', + self.tempFile('*.txt', 'unittest'), + '--excluded=:/tmp/*/unit*']) + result = base.BaseTool.result() + self.assertEquals('', result) + + def testGrepLineNo(self): + if debug: + return + appl.TextTool.main(['texttool', + 'grep', + 'line', + self.tempFile('first*txt'), + '--line-no']) + result = base.BaseTool.result() + self.assertEquals("{}-1: line 1\n{}-2: line 2\n{}-3: This text is in line 3".format(self._fn, self._fn, self._fn), result) + + def testGrep2(self): + if debug: + return + appl.TextTool.main(['texttool', + 'grep', + r'text\sis', + self.tempFile('first*txt')]) + self.assertEquals("{}: This text is in line 3".format(self._fn, self._fn, self._fn), base.BaseTool.result()) + + def testGrepList(self): + if debug: + return + appl.TextTool.main(['texttool', + 'grep', + r'text\sis', + self.tempFile('first*txt'), + '--list']) + self.assertEquals(self._fn, base.BaseTool.result()) + + def testGrepOnlyMatching(self): + if debug: + return + appl.TextTool.main(['texttool', + 'grep', + r'text\sis', + self.tempFile('first*txt'), + '--only-matching']) + result = base.BaseTool.result() + self.assertEquals("{}: text is".format(self._fn, self._fn, self._fn), result) + + def testGrepOnlyMatchingLineNo(self): + if debug: + return + appl.TextTool.main(['texttool', + 'grep', + r'text\sis', + self.tempFile('first*txt'), + '--line-no', + '--only-matching']) + result = base.BaseTool.result() + self.assertEquals("{}-3: text is".format(self._fn, self._fn, self._fn), result) + + def testGrepFormat(self): + if debug: + return + appl.TextTool.main(['texttool', + 'grep', + r'(\d+)', + self.tempFile('first*txt'), + r'--format=!full!-!no!: $1']) + self.assertEquals("{}-1: 1\n{}-2: 2\n{}-3: 3".format(self._fn, self._fn, self._fn), base.BaseTool.result()) + + def testGrepReverse(self): + if debug: + return + appl.TextTool.main(['texttool', + 'grep', + r'line\s+', + self.tempFile('*.txt', 'unittest'), + r'--reverse']) + self.assertEquals("{}: How are you".format(self._fn2), base.BaseTool.result()) + + def testGrepMissing(self): + if debug: + return + appl.TextTool.main(['texttool', + 'grep', + r'line\s+', + self.tempFile('*.txt', 'unittest'), + '--missing']) + self.assertEquals(self._fn2, base.BaseTool.result()) + + def testReplace(self): + if debug: + return + fnOut = self._tempDir + os.sep + 'first.out' + appl.TextTool.main(['texttool', + 'replace', + r'line (\d+)', + 'zeile $1$1', + self.tempFile('*.txt', 'unittest'), + fnOut]) + replaced = base.StringUtils.fromFile(fnOut) + self.assertEquals('zeile 11\nzeile 22\nThis text is in zeile 33\n', replaced) + + def testReplace2(self): + if debug: + return + fnSrc = self.tempFile('first.txt', 'unittest', 'texttool') + fnOut = self.tempFile('copy.txt', 'unittest') + shutil.copy2(fnSrc, fnOut) + appl.TextTool.main(['texttool', '-v4', + 'replace', + r'line (\d+)', + 'zeile $1$1', + fnOut]) + replaced = base.StringUtils.fromFile(fnOut) + self.assertEquals('zeile 11\nzeile 22\nThis text is in zeile 33\n', replaced) + self.ensureFileDoesNotExist(fnOut) + + def testReplaceCount(self): + if debug: + return + fnOut = self._tempDir + os.sep + 'first.out' + appl.TextTool.main(['texttool', + 'replace', + r'(\s+)([il])', + '$1*$2$2', + self.tempFile('*.txt', 'unittest'), + fnOut, + '--count=1']) + replaced = base.StringUtils.fromFile(fnOut) + self.assertEquals('line 1\nline 2\nThis text *iis in line 3\n', replaced) + + def testReplaceInPlace(self): + if debug: + return + source = self._fn + fn = self._tempDir + os.sep + 'third' + shutil.copy(source, fn) + appl.TextTool.main(['texttool', + 'replace', + r'(\s+)(\S+)(\s+)', + '$1*$2*$3', + fn, + '--count=2']) + replaced = base.StringUtils.fromFile(fn) + self.assertEquals('line 1\nline 2\nThis *text* *is* in line 3\n', replaced) + + def testReplaceByTable(self): + if debug: + return + table = self._tempDir + os.sep + 'table.data' + fnOut = self._tempDir + os.sep + 'first.out' + base.StringUtils.toFile(table, """e\t33 +xt\tXT +33 \tE- +""") + appl.TextTool.main(['texttool', + 'replace', + '@' + table, + self.tempFile('f*txt'), + fnOut]) + replaced = base.StringUtils.fromFile(fnOut) + self.assertEquals('linE-1\nlinE-2\nThis t33XT is in linE-3\n', replaced) + + def testReplaceByTableCount(self): + if debug: + return + table = self._tempDir + os.sep + 'table.data' + base.StringUtils.toFile(table, """e\t33 +xt\tXT +ei\tI +""") + fnOut = self._tempDir + os.sep + 'first.out' + appl.TextTool.main(['texttool', + 'replace', + '@' + table, + self.tempFile('f*txt'), + fnOut, + '--count=1']) + replaced = base.StringUtils.fromFile(fnOut) + self.assertEquals('lin33 1\nlin33 2\nThis t33XT is in line 3\n', replaced) + + def testBuildTargetname(self): + if debug: + return + tool = appl.TextTool.TextTool(self.getOptions()) + tool._target = '!full!->!path!+!node! | !path!#!fn!+!ext!' + self.assertEquals('/home/bin/abc.def->/home/bin/+abc.def | /home/bin/#abc+.def', tool.buildTargetname('/home/bin/abc.def')) + + def testReplaceDirectInput(self): + if debug: + return + appl.TextTool.main(['texttool', + '--quiet', + 'replace', + '/', + r'\\', + '=/data/by/device.com\n/data/by/other', + '--count=1']) + replaced = base.BaseTool.result() + self.assertEquals('\\\\data/by/device.com\n\\\\data/by/other\n', replaced) + + def testAdaptConfig(self): + if debug: + return + fnVars = self._tempDir + os.sep + 'texttool.conf' + base.StringUtils.toFile(fnVars, '''php.memory_limit = 2048M +php.upload_max_filesize = 512M + +''') + fnConfig = self._tempDir + os.sep + 'config.conf' + base.StringUtils.toFile(fnConfig, '''#PHP-Configuration: +memory_limit = 512M +upload_max_filesize = 2M +''') + appl.TextTool.main(['texttool', + '-v3', + '--configuration-directory=' + self._tempDir, + 'adapt-configuration', + fnConfig, + 'prefix=php.']) + current = base.StringUtils.fromFile(fnConfig) + self.assertEquals('''#PHP-Configuration: +memory_limit=2048M +upload_max_filesize=512M +''', current) + + def testAdaptConfigFile(self): + if debug: + return + fnVars = self._tempDir + os.sep + 'vars.txt' + base.StringUtils.toFile(fnVars, '''abc=123 +command.log = "/var/log/command.log" +''') + fnConfig = self._tempDir + os.sep + 'config.conf' + base.StringUtils.toFile(fnConfig, '''#configuration: +abc=4 +command.log='/var/log/dummy.log' +''') + appl.TextTool.main(['texttool', + '-v3', + 'adapt-configuration', + fnConfig, + fnVars]) + current = base.StringUtils.fromFile(fnConfig) + self.assertEquals('''#configuration: +abc=123 +command.log = "/var/log/command.log" +''', current) + + def testReplaceOrInsert(self): + if debug: + return + fnVars1 = self._tempDir + os.sep + 'vars1.txt' + fnVars2 = self._tempDir + os.sep + 'vars2.txt' + base.StringUtils.toFile(fnVars1, '''#configuration: +abc=4 +command.log='/var/log/dummy.log' +''') + base.StringUtils.toFile(fnVars2, '''# Intro +command.log='/var/log/dummy.log' +[files] +blabla=4 +''') + appl.TextTool.main(['texttool', + '-v3', + 'replace-or-insert', + r'^abc\s*=', + 'abc=432', + fnVars1.replace('vars1.', 'v*.'), + r'--below-anchor=^\[files\]']) + current = base.StringUtils.fromFile(fnVars1) + self.assertEquals('''#configuration: +abc=432 +command.log='/var/log/dummy.log' +''', current) + current = base.StringUtils.fromFile(fnVars2) + self.assertEquals('''# Intro +command.log='/var/log/dummy.log' +[files] +abc=432 +blabla=4 +''', current) + + def testReplaceOrInsertExclude(self): + if debug: + return + fnVars1 = self._tempDir + os.sep + 'vars1.txt' + fnVars2 = os.path.dirname(self._tempDir) + os.sep + 'vars2.txt' + content = '''#configuration: +abc=4 +command.log='/var/log/dummy.log' +''' + base.StringUtils.toFile(fnVars1, content) + base.StringUtils.toFile(fnVars2, '''# Intro +command.log='/var/log/dummy.log' +[files] +blabla=4 +''') + appl.TextTool.main(['texttool', + '-v3', + 'replace-or-insert', + r'^abc\s*=', + 'abc=432', + fnVars2.replace('vars2.', 'v*.'), + r'--excluded=;*1*']) + current = base.StringUtils.fromFile(fnVars1) + self.assertEquals(content, current) + current = base.StringUtils.fromFile(fnVars2) + self.assertEquals('''# Intro +command.log='/var/log/dummy.log' +[files] +blabla=4 +abc=432 +''', current) + + def testReplaceOrInsertMaxDepth(self): + if debug: + return + fnVars1 = self._tempDir + os.sep + 'vars1.txt' + fnVars2 = os.path.dirname(self._tempDir) + os.sep + 'vars2.txt' + content = '''#configuration: +abc=4 +command.log='/var/log/dummy.log' +''' + base.StringUtils.toFile(fnVars1, content) + base.StringUtils.toFile(fnVars2, '''# Intro +command.log='/var/log/dummy.log' +[files] +blabla=4 +''') + appl.TextTool.main(['texttool', + '-v3', + 'replace-or-insert', + r'^abc\s*=', + 'abc=432', + fnVars2.replace('vars2.', 'v*.'), + r'--max-depth=0', + '--above-anchor=^#']) + current = base.StringUtils.fromFile(fnVars1) + self.assertEquals(content, current) + current = base.StringUtils.fromFile(fnVars2) + self.assertEquals('''abc=432 +# Intro +command.log='/var/log/dummy.log' +[files] +blabla=4 +''', current) + + def testRandom0(self): + if debug: + return + appl.TextTool.main(['texttool', + '-v3', + 'random', + '--seed=hiWorld']) + current = base.BaseTool.result() + self.assertEquals('388449311', current) + + def testFindRegion(self): + if debug: + return + tool = appl.TextTool.TextTool(self.getOptions()) + tool._lines = '''auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address + +auto eth1 +iface eth1 inet dhcp +'''.split('\n') + start, end = tool.findRegion(r'^(auto|iface)\s+eth0', True, r'^(auto|iface)', False, None, False) + self.assertEquals(3, start) + self.assertEquals(7, end) + tool.removeRegion() + self.assertEquals('''auto lo +iface lo inet loopback + +auto eth1 +iface eth1 inet dhcp +''', '\n'.join(tool._lines)) + + def testReplaceRegion(self): + if debug: + return + tool = appl.TextTool.TextTool(self.getOptions()) + tool._lines = '''auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address + +auto eth1 +iface eth1 inet dhcp +'''.split('\n') + start, end = tool.findRegion(r'^(auto|iface)\s+eth0', True, r'^(auto|iface)', False, None, False) + self.assertEquals(3, start) + self.assertEquals(7, end) + tool.replaceRegion('auto wlan0\niface wlan0 dhcp\n') + self.assertEquals('''auto lo +iface lo inet loopback + +auto wlan0 +iface wlan0 dhcp + +auto eth1 +iface eth1 inet dhcp +''', '\n'.join(tool._lines)) + + def testCurrentFind(self): + if debug: + return + tool = appl.TextTool.TextTool(self.getOptions()) + tool._lines = '''auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address + +auto eth1 +iface eth1 inet dhcp +'''.split('\n') + self.assertEquals(3, tool.currentFind(re.compile(r'.*\d'))) + self.assertNone(tool.currentFind(re.compile(r'neverever'))) + + def testCurrentFindRange(self): + if debug: + return + tool = appl.TextTool.TextTool(self.getOptions()) + tool._lines = '''auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address 192.168.6.2 + +auto eth1 +iface eth1 inet dhcp +'''.split('\n') + for start in range(6): + self.assertEquals(5, tool.currentFind(re.compile(r'.*address'), start, 6)) + self.assertNone(tool.currentFind(re.compile(r'.*address'), 7, 9)) + + def testBuildExamples(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'build-examples']) + + def testExampleAdapt1(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'adapt', + '/usr/share/pyrshell/examples/data/php.ini', + '/usr/share/pyrshell/examples/config/php_minimal.conf']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleAdapt2(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'adapt', + '/usr/share/pyrshell/examples/data/php.ini', + 'prefix=php.']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleGrep1(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'grep', + r'\bopen\b.*rb', + '/usr/share/pyrshell/base/*.py', + '--excluded=:*file*:*text*']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleGrep2(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'grep', + r'total:\s+([0-9.]+)', + '/usr/share/pyrshell/examples/data/sum.txt', + '--line-no', '--format=Sum: $1']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleGrep3(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'grep', + 'StringUtils', + '/usr/share/pyrshell/net/*.py', + '--missing', + '--list']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleGrep4(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'grep', + r'^((\d: \w+)|(\s*inet\s+[0-9.]+))', + '/usr/share/pyrshell/examples/data/ip.out.txt', + '--only-matching']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleGrep5(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'grep', + r'total:', + '/usr/share/pyrshell/examples/data/sum.txt', + '--reverse']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testExampleGrep6(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'grep', '-i', '--no-name', '-v', + r'jonny', + '/usr/share/pyrshell/examples/data/first.addr']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + lines = base.BaseTool.result() + self.assertEquals('info@whow.com', lines) + + def testExamplePythonToC1(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'python-to-c', + '/usr/share/pyrshell/base/ThreadLogger.py', + '/tmp/threadlogger.ts']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileContains(r"logger.log('Hi');", '/tmp/threadlogger.ts') + + def testExampleReplace1(self): + if debug: + return + timestring = time.strftime('DATE=%d%m.%Y') + appl.TextTool.main(['texttool', '-v3', 'replace', + r'DATE=\S+', + timestring, + '/usr/share/pyrshell/examples/data/today.sh']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileContains(timestring, '/usr/share/pyrshell/examples/data/today.sh') + + def testExampleReplace2(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'replace', + 'jonny@gmx.de', + 'mr.universe@gmx.de', + '/usr/share/pyrshell/examples/data/*.addr', + '!path!universe_!node!.out', + '--excluded=:shop.addr:test*', + '--max-depth=0']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileContains('mr.universe@gmx.de', '/usr/share/pyrshell/examples/data/universe_first.addr.out') + self.assertFileContains('mr.universe@gmx.de', '/usr/share/pyrshell/examples/data/universe_second.addr.out') + self.assertFileNotExists('/usr/share/pyrshell/examples/data/universe_shop.addr.out') + self.assertFileNotExists('/usr/share/pyrshell/examples/data/universe_test.addr.out') + self.assertFileNotExists('/usr/share/pyrshell/examples/data/subdir/universe_third.addr.out') + + def testExampleReplace3(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'replace', + 'hot', + 'cool', + '=should be hot']) + self.assertEquals('should be cool', base.BaseTool.result()) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + + def testExampleReplace4(self): + if debug: + return + appl.TextTool.main(['texttool', '-v3', 'replace', + '@/usr/share/pyrshell/examples/config/german.txt', + '/usr/share/pyrshell/examples/data/addr.csv']) + self.assertFileContains('Nr;Vorname;Nachname', '/usr/share/pyrshell/examples/data/addr.csv') + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + + def testCurrentInsertAnchored(self): + if debug: + return + lines = '''# missing auto lo +iface lo inet loopback'''.split('\n') + self._tool._lines = lines[:] + self._tool.currentInsertAnchored('auto lo', r'iface\s+lo', False) + self.assertEquals('auto lo', self._tool._lines[1]) + self.assertEquals('iface lo inet loopback', self._tool._lines[2]) + self._tool._lines = lines[:] + self._tool.currentInsertAnchored('auto eth0', None) + self.assertEquals('iface lo inet loopback', self._tool._lines[1]) + self.assertEquals('auto eth0', self._tool._lines[2]) + + lines = '''# missing auto lo +auto lo +iface lo inet loopback +auto eth0 +iface eth0 inet dhcp +auto wlan0 +iface wlan0 inet dhcp'''.split('\n') + self._tool._lines = lines[:] + (start, end) = self._tool.findRegion(r'auto eth', True, 'iface eth', True) + self.assertEquals(3, start) + self.assertEquals(5, end) + self._tool.currentInsertAnchored('auto eth1\niface eth1 inet dhcp', r'wlan0', False, start, end) + self.assertEquals('''# missing auto lo +auto lo +iface lo inet loopback +auto eth0 +iface eth0 inet dhcp +auto eth1 +iface eth1 inet dhcp +auto wlan0 +iface wlan0 inet dhcp''', '\n'.join(self._tool._lines)) + + def testCurrentReplace(self): + if debug: + return + lines = '''[log] +file=test.txt +[db] +file=test.sql +[opts]'''.split('\n') + self._tool._lines = lines[:] + self._tool.currentReplace(r'file=', 'file=example.txt', r'\[db\]', False) + self._tool.currentReplace(r'level=', 'level=3', r'\[db\]', False) + self.assertEquals('file=example.txt', self._tool._lines[1]) + self.assertEquals('level=3', self._tool._lines[3]) + self._tool._lines = lines[:] + self._tool.currentReplace(r'file=', 'file=addr.sql', None, False, 1, 4) + self._tool.currentReplace(r'user=', 'user=bob', r'\[db\]', False, 1, 5) + self.assertEquals('file=addr.sql', self._tool._lines[1]) + self.assertEquals('user=bob', self._tool._lines[3]) + + def testCurrentSetLine(self): + if debug: + return + lines = '''[log] +file=test.txt +[db] +file=test.sql +[opts] +file=data.conf'''.split('\n') + self._tool._lines = lines[:] + self._tool.currentSetLine(r'file=', 'file=example') + self.assertEquals('file=example', self._tool._lines[1]) + self.assertEquals('file=test.sql', self._tool._lines[3]) + self.assertEquals('file=data.conf', self._tool._lines[5]) + self._tool._lines = lines[:] + self._tool.currentSetLine(r'file=', 'file=default', 2) + self.assertEquals('file=default', self._tool._lines[1]) + self.assertEquals('file=default', self._tool._lines[3]) + self.assertEquals('file=data.conf', self._tool._lines[5]) + self._tool._lines = lines[:] + self._tool.currentSetLine(r'file=', 'file=default', 2, 2, 4) + self.assertEquals('file=test.txt', self._tool._lines[1]) + self.assertEquals('file=default', self._tool._lines[3]) + self.assertEquals('file=data.conf', self._tool._lines[5]) + self._tool._lines = lines[:] + self._tool.currentSetLine(r'file=', 'file=default', 2, 2, 6) + self.assertEquals('file=test.txt', self._tool._lines[1]) + self.assertEquals('file=default', self._tool._lines[3]) + self.assertEquals('file=default', self._tool._lines[5]) + + def testExampleReplaceRegion(self): + if debug: + return + # texttool replace-region '<body>' '</body>' '<p>no access!</p>' /usr/share/pyrshell/examples/index.html + temp = self.tempDir('replace', 'unittest') + trg = temp + os.sep + 'index.html' + base.StringUtils.toFile(trg, '<html>\n<body>\n<h1>Remember</h1>\n<p>forget it!</p>\n</body>\n</html>') + appl.TextTool.main(['texttool', '-v3', + 'replace-region', + '<body>', + '</body>', + '<p>no access!</p>', + trg, + '--start-excluded', + '--max-depth=0']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileContent(trg, '''<html> +<body> +<p>no access!</p> +</body> +</html>''') + + def testExampleReplaceRegion2(self): + if debug: + return + # texttool replace-region '<body>' '</body>' '<p>no access!</p>' /usr/share/pyrshell/examples/index.html + temp = self.tempDir('replace', 'unittest') + trg = temp + os.sep + 'index.html' + base.StringUtils.toFile(trg, '<html>\n<body>\n<h1>Remember</h1>\n<p>forget it!</p>\n</body>\n</html>') + appl.TextTool.main(['texttool', '-v4', + 'replace-region', + r'<h[1-9]>', + '.*</p>', + '<p>no access!</p>', + trg, + '--end-included']) + tool = base.BaseTool.latestTool() + self.assertEquals(0, tool._logger._errors) + self.assertFileContent(trg, '''<html> +<body> +<p>no access!</p> +</body> +</html>''') + +if __name__ == '__main__': + # import sys;sys.argv = ['', 'Test.testName'] + tester = TextToolTest() + tester.run() diff --git a/unittest/appl/WikiToolTest.py b/unittest/appl/WikiToolTest.py new file mode 100644 index 0000000..8a9effa --- /dev/null +++ b/unittest/appl/WikiToolTest.py @@ -0,0 +1,101 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase + +import appl.WikiTool +import base.BaseTool + +debug = False + +def usage(msg=None): + return 'test usage' + +class WikiToolTest(UnitTestCase): + + def getOptions(self): + appInfo = base.BaseTool.ApplicationInfo('wiki', 'appl.WikiTool', usage) + rc = base.BaseTool.GlobalOptions(appInfo) + rc._exampleText='log.file=/var/log/local/wikitool.log' + rc._logFiles.append('-') + rc._verboseLevel = 0 + return rc + + def check(self, current): + fnInput = self.tempFile('inp', 'wikitest') + fnOutput = self.tempFile('out', 'wikitest') + base.StringUtils.toFile(fnInput, current) + tool = appl.WikiTool.WikiTool(self.getOptions()) + tool.gitlabToMediaWiki([fnInput, fnOutput]) + out = base.StringUtils.fromFile(fnOutput) + return out + + def testInlineCode(self): + if debug: + return + line = 'one __fat__ and **other fat** thing in _italics_ or _in another way_' + out = self.check(line) + self.assertEquals("one '''fat''' and '''other fat''' thing in ''italics'' or ''in another way''", out) + + def testLink(self): + if debug: + return + line = '[abc def](Link) [Noch ein Text](Bedienungsanleitung)' + out = self.check(line) + self.assertEquals('[[Link|abc def]] [[Bedienungsanleitung|Noch ein Text]]', out) + + def testHeadline(self): + if debug: + return + line = '#Haupt\n## Unter-1\n### Unter-2' + out = self.check(line) + self.assertEquals('= Haupt =\n== Unter-1 ==\n=== Unter-2 ===', out) + + def testUnorderedList(self): + if debug: + return + line = ' - erstens\n + 1.1\n - zweitens' + out = self.check(line) + self.assertEquals('* erstens\n** 1.1\n* zweitens', out) + + def testTable(self): + if debug: + return + line = """Tabelle +| Id | Name | +| 1 | Huber | +| 2 | Maier |""" + out = self.check(line) + self.assertEquals("""Tabelle +{| +! Id +! Name +|- +| 1 +| Huber +|- +| 2 +| Maier +|}""", out) + + def testScript(self): + if False and debug: + return + line = """Script +```bash +FN=abc +echo "nicht **fett**" +```""" + out = self.check(line) + self.assertEquals("""Script +<syntaxhighlight lang="bash" "line='line'> +FN=abc +echo "nicht **fett**" +</syntaxhighlight>""", out) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = WikiToolTest() + tester.run() diff --git a/unittest/appl/__init__.py b/unittest/appl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/unittest/base/BaseTester.py b/unittest/base/BaseTester.py new file mode 100644 index 0000000..87786f2 --- /dev/null +++ b/unittest/base/BaseTester.py @@ -0,0 +1,36 @@ +''' +Created on 22.04.2018 + +@author: hm +''' + +import unittest.base.MemoryLoggerTest +import unittest.base.ThreadLoggerTest +import unittest.base.LoggerTest +import unittest.base.StringUtilsTest +import unittest.base.LinuxUtilsTest + +class Runner: + def __init__(self): + self._modules = 0 + self._errors = 0 + self._asserts = 0 + + def run(self, clazz): + tester = clazz() + tester.run() + self._modules += 1 + self._errors += tester._errors + self._asserts += tester._asserts + +def main(): + runner = Runner() + runner.run(unittest.base.LoggerTest.LoggerTest) + runner.run(unittest.base.MemoryLoggerTest.MemoryLoggerTest) + runner.run(unittest.base.ThreadLoggerTest.ThreadLoggerTest) + runner.run(unittest.base.StringUtilsTest.StringUtilsTest) + runner.run(unittest.base.LinuxUtilsTest.LinuxUtilsTest) + + print('= BaseTester: {:d} modules with {:d} asserts and {:d} error(s)'.format(runner._modules, runner._asserts, runner._errors)) +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/unittest/base/CryptoEngineTest.py b/unittest/base/CryptoEngineTest.py new file mode 100644 index 0000000..9d43b9d --- /dev/null +++ b/unittest/base/CryptoEngineTest.py @@ -0,0 +1,174 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import base.CryptoEngine +import base.MemoryLogger +import os.path +import base64 + +debug = False + +class CryptoEngineTest(UnitTestCase): + + def testBasic(self): + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + #self.log('random: ' + engine.nextString(60, 'ascii95')) + + def testEncode(self): + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + text = 'Hi_World' + encoded = engine.encode(text, 'word') + self.log('=' + encoded) + decoded = engine.decode(encoded, 'word') + self.assertEquals(text, decoded) + self.assertEquals(0, len(logger.getMessages())) + + def testDecode(self): + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + for aSet in engine.getCharSetNames(): + text = engine.nextString(20, aSet) + encoded = engine.encode(text, aSet) + decoded = engine.decode(encoded, aSet) + self.assertEquals(text, decoded) + self.assertEquals(0, len(logger.getMessages())) + for aSet in engine.getCharSetNames(): + text = engine.getCharSet(aSet) + encoded = engine.encode(text, aSet) + decoded = engine.decode(encoded, aSet) + self.assertEquals(text, decoded) + self.assertEquals(0, len(logger.getMessages())) + + def buildBinary(self, length): + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + engine.setSeedRandomly() + rc = '' + for ix in range(length): + rc += chr(engine.nextInt(127, 1)) + return rc + + def testEncodeBinaryBase(self): + if debug: + return + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + text = '12' + encoded = engine.encodeBinary(text) + decoded = engine.decodeBinary(encoded) + self.assertEquals(text, decoded) + text = '123' + encoded = engine.encodeBinary(text) + decoded = engine.decodeBinary(encoded) + self.assertEquals(text, decoded) + text = '1235' + encoded = engine.encodeBinary(text) + decoded = engine.decodeBinary(encoded) + self.assertEquals(text, decoded) + + def testEncodeBinary(self): + if debug: + return + if self.assertTrue(False): + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + for length in range(20, 256): + text = self.buildBinary(length) + print(length) + encoded = engine.encodeBinary(text) + try: + decoded = engine.decodeBinary(encoded) + except Exception as exc: + self.assertEquals('', str(exc)) + break + self.assertEquals(text, decoded) + self.assertEquals(0, len(logger.getMessages())) + + def testTestCharSet(self): + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + for name in engine.getCharSetNames(): + aSet = engine.getCharSet(name) + self.assertEquals(-1, engine.testCharSet(aSet, name)) + aSet += "\t" + self.assertEquals(len(aSet) - 1, engine.testCharSet(aSet, name)) + + def testOneTimePad(self): + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + fn = '/tmp/otp_request.txt' + with open(fn, 'w') as fp: + for user in range(1, 100): + data = 'X{:04x}y'.format(user) + pad = engine.oneTimePad(user, data) + padData = engine.unpackOneTimePad(pad) + self.assertEquals(user, padData[1]) + self.assertEquals(data, padData[2]) + fp.write('{:d}\t{:s}\t{:s}'.format(user, data, pad)) + + def testExternOneTimePad(self): + if "x"+"y" == "xy": + return + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + fn = '/tmp/otp.txt' + if self.assertTrue(os.path.exists(fn)): + with open(fn, 'r') as fp: + for line in fp: + [user, data, pad] = line.rstrip().split("\t") + padData = engine.unpackOneTimePad(pad, 3600) + self.assertEquals(int(user), padData[1]) + self.assertEquals(data, padData[2]) + + def testSetSeedFromString(self): + if debug: + return + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + engine.setSeedFromString('') + self.assertEquals(231702727, engine.nextInt()) + engine.setSeedFromString('x') + self.assertEquals(1157398379, engine.nextInt()) + engine.setSeedFromString('blubber') + self.assertEquals(604275342, engine.nextInt()) + + def testSaveRestore(self): + if debug: + return + logger = base.MemoryLogger.MemoryLogger() + engine = base.CryptoEngine.CryptoEngine(logger) + engine.setSeedFromString('') + seed1 = engine.saveSeed() + value1 = engine.nextString(10, 'ascii94') + engine.restoreSeed(seed1) + value2 = engine.nextString(10, 'ascii94') + self.assertEquals(value1, value2) + + def testBase64(self): + if False and debug: + return + buffer = b'x' + ''' + for ix in range(256): + buffer = buffer[0:-1] + print("ix: " + str(ix)) + encoded = base64.encodebytes(buffer) + decoded = base64.decodebytes(encoded) + if decoded != buffer: + print("Different: {:02x}".format(ix)) + for ix2 in range(32, 128): + buffer += bytes(ix2) + encoded = base64.encodebytes(buffer) + decoded = base64.decodebytes(encoded) + if decoded != buffer: + print("Different: {:02x}, {:02x}".format(ix, ix2)) + ''' +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = CryptoEngineTest() + tester.run() diff --git a/unittest/base/FileHelperTest.py b/unittest/base/FileHelperTest.py new file mode 100644 index 0000000..4f759cb --- /dev/null +++ b/unittest/base/FileHelperTest.py @@ -0,0 +1,371 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase + +import shutil +import datetime +import time +import os.path + +import base.FileHelper +import base.StringUtils + +debug = False + +def usage(msg=None): + return 'test usage' + +class FileHelperTest(UnitTestCase): + def __init__(self): + UnitTestCase.__init__(self) + self.tempDir('filetool', 'unittest') + self._fn = self.tempFile('first.txt', 'unittest', 'filetool') + base.StringUtils.toFile(self._fn, "line 1\nline 2\nThis file is in line 3") + + def _finish(self): + shutil.rmtree(self.tempDir('unittest')) + + def getOptions(self): + rc = base.BaseTool.GlobalOptions() + rc._exampleFile = 'log.file=/var/log/local/filetool.log' + rc._logFiles.append('-') + rc._verboseLevel = 0 + return rc + + def checkPart(self, container, full, path, node, fn, ext): + self.assertEquals(path, container['path']) + self.assertEquals(full, container['full']) + self.assertEquals(node, container['node']) + self.assertEquals(fn, container['fn']) + self.assertEquals(ext, container['ext']) + + def testSplitFilename(self): + if debug: + return + container = base.FileHelper.splitFilename('/tmp/jonny.txt') + self.checkPart(container, '/tmp/jonny.txt', '/tmp/', 'jonny.txt', 'jonny', '.txt') + container = base.FileHelper.splitFilename('/home/authors/jonny.txt') + self.checkPart(container, '/home/authors/jonny.txt', '/home/authors/', 'jonny.txt', 'jonny', '.txt') + container = base.FileHelper.splitFilename('jonny.v2.txt') + self.checkPart(container, 'jonny.v2.txt', '', 'jonny.v2.txt', 'jonny.v2', '.txt') + container = base.FileHelper.splitFilename('.config') + self.checkPart(container, '.config', '', '.config', '.config', '') + + def testTail(self): + if debug: + return + tail = base.FileHelper.tail(self._fn) + self.assertEquals(1, len(tail)) + self.assertEquals('This file is in line 3', tail[0]) + + def testTailNumbers(self): + if debug: + return + tail = base.FileHelper.tail(self._fn, 2, True) + self.assertEquals(2, len(tail)) + asString = ''.join(tail) + self.assertEquals('2: line 2\n3: This file is in line 3', asString) + + def testDirectoryInfo(self): + if debug: + return + info = base.FileHelper.directoryInfo('/etc', r'.*\.conf') + self.assertTrue(info._fileCount > 0) + self.assertTrue(info._fileSizes > 0) + self.assertTrue(info._dirCount > 0) + self.assertTrue(info._ignoredDirs > 0) + # self.assertTrue(info._ignoredFiles > 0) + self.assertEquals(5, len(info._youngest)) + self.assertEquals(5, len(info._largest)) + + def testPathToNode(self): + if debug: + return + self.assertEquals('x__abc_def_x.txt', base.FileHelper.pathToNode('x:/abc/def/x.txt')) + + def testSetModified(self): + if debug: + return + fn = self.tempFile('test.txt', 'unittest') + base.StringUtils.toFile(fn, 'Hi') + yesterday = time.time() - 86400 + januar = datetime.datetime(2016, 1, 2, 10, 22, 55) + januar2 = time.mktime(januar.timetuple()) + base.FileHelper.setModified(fn, yesterday) + self.assertEquals(yesterday, os.path.getmtime(fn)) + base.FileHelper.setModified(fn, None, januar) + self.assertEquals(januar2, os.path.getmtime(fn)) + + def testDistinctPaths(self): + if debug: + return + tempDir = self.tempDir('disticts', 'unittest') + self.clearDirectory(tempDir) + dir1 = tempDir + os.sep + 'abc' + dir2 = tempDir + os.sep + 'def' + dirLink = tempDir + os.sep + 'link' + dirChild = dir1 + os.sep + 'child' + dirChildInLink = dirLink + os.sep + 'childInLink' + dirLinkLink = dir1 + os.sep + 'linkLink' + self.ensureDirectory(dir1) + self.ensureDirectory(dir2) + self.ensureDirectory(dirChild) + os.symlink(dir2, dirLink) + os.symlink(dirChildInLink, dirLinkLink) + # base/abc + # base/abc/child + # base/abc/linkInLink -> def + # base/def + # base/link -> def + # base/def/childInLink + # base/def/linkLink -> def/childInLink + self.assertTrue(base.FileHelper.distinctPaths(dir1, dir2)) + self.assertTrue(base.FileHelper.distinctPaths(dir2, dir1)) + self.assertTrue(base.FileHelper.distinctPaths(dirChild, dir2)) + self.assertTrue(base.FileHelper.distinctPaths(dir2, dirChild)) + self.assertTrue(base.FileHelper.distinctPaths(dir1, dirLink)) + self.assertTrue(base.FileHelper.distinctPaths(dirLink, dir1)) + + self.assertFalse(base.FileHelper.distinctPaths(dirChild, dir1)) + self.assertFalse(base.FileHelper.distinctPaths(dir1, dirChild)) + self.assertFalse(base.FileHelper.distinctPaths(dir2, dirLink)) + self.assertFalse(base.FileHelper.distinctPaths(dirLink, dir2)) + self.assertFalse(base.FileHelper.distinctPaths(dir2, dirChildInLink)) + self.assertFalse(base.FileHelper.distinctPaths(dirChildInLink, dir2)) + self.assertFalse(base.FileHelper.distinctPaths(dir2, dirLinkLink)) + self.assertFalse(base.FileHelper.distinctPaths(dirLinkLink, dir2)) + self.assertFalse(base.FileHelper.distinctPaths(dirChildInLink, dirLinkLink)) + self.assertFalse(base.FileHelper.distinctPaths(dirLinkLink, dirChildInLink)) + self.assertFalse(base.FileHelper.distinctPaths(dirLinkLink, dir2)) + self.assertFalse(base.FileHelper.distinctPaths(dir2, dirLinkLink)) + + def testFromBytes(self): + if False and debug: + return + self.assertEquals('ascii', base.FileHelper.fromBytes(b'ascii')) + self.assertEquals('äöüÃÃÃÃ', base.FileHelper.fromBytes('äöüÃÃÃÃ'.encode('utf_8'))) + line = 'äöüÃÃÃÃ'.encode('latin-1') + self.assertEquals('äöüÃÃÃÃ', base.FileHelper.fromBytes(line)) + line = 'äöüÃÃÃÃ'.encode('cp850') + self.assertFalse('äöüÃÃÃÃ' == base.FileHelper.fromBytes(line)) + line = b'' + hex = '' + for ix in range(1, 255): + hex += "{:02x}".format(ix) + line = bytes.fromhex(hex) + self.assertFalse('äöüÃÃÃÃ' == base.FileHelper.fromBytes(line)) + + def testJoinPaths(self): + self.assertEquals('family/sister', base.FileHelper.joinPaths('../sister', 'family/brother', self._logger)) + self.assertEquals('/a/family/sister', base.FileHelper.joinPaths('../../sister', '/a/family/brother/b', self._logger)) + self.assertEquals('sister', base.FileHelper.joinPaths('../sister', 'brother', self._logger)) + current = os.curdir + self.assertEquals(os.path.dirname(current), base.FileHelper.joinPaths('..', None, self._logger)) + + def testJoinPathsErrors(self): + self.log('= expecting error') + self.assertNone(base.FileHelper.joinPaths('./sister', 'brother', self._logger)) + self.assertNone(base.FileHelper.joinPaths('/sister', 'brother', self._logger)) + self.assertNone(base.FileHelper.joinPaths('../../sister', 'brother', self._logger)) + + def testEnsureDir(self): + temp = self.tempDir('dir1', 'unittest') + # already exists + base.FileHelper.ensureDirectory(temp, self._logger) + self.assertTrue(os.path.isdir(temp)) + # does not exist with logger + self.ensureFileDoesNotExist(temp) + base.FileHelper.ensureDirectory(temp, self._logger) + self.assertTrue(os.path.isdir(temp)) + # does not exist without logger + self.ensureFileDoesNotExist(temp) + base.FileHelper.ensureDirectory(temp) + self.assertTrue(os.path.isdir(temp)) + # file exists, with logger + self.ensureFileDoesNotExist(temp) + base.StringUtils.toFile(temp, 'anything') + base.FileHelper.ensureDirectory(temp) + self.assertTrue(os.path.isdir(temp)) + # file exists, with logger + self.ensureFileDoesNotExist(temp) + base.StringUtils.toFile(temp, 'anything') + base.FileHelper.ensureDirectory(temp, self._logger) + self.assertTrue(os.path.isdir(temp)) + # invalid link, with logger + self.ensureFileDoesNotExist(temp) + os.symlink('../does-not-exist', temp) + base.FileHelper.ensureDirectory(temp, self._logger) + self.assertTrue(os.path.isdir(temp)) + # invalid link, without logger + self.ensureFileDoesNotExist(temp) + os.symlink('../does-not-exist2', temp) + base.FileHelper.ensureDirectory(temp, self._logger) + self.assertTrue(os.path.isdir(temp)) + + def testEnsureFileDoesNotExist(self): + temp = self.tempDir('file', 'unittest') + # directory exists + base.FileHelper.ensureFileDoesNotExist(temp, self._logger) + self.assertFalse(os.path.exists(temp)) + # does not exists: + base.FileHelper.ensureFileDoesNotExist(temp, self._logger) + self.assertFalse(os.path.exists(temp)) + base.FileHelper.ensureFileDoesNotExist(temp) + self.assertFalse(os.path.exists(temp)) + # file exists + base.StringUtils.toFile(temp, 'x') + base.FileHelper.ensureFileDoesNotExist(temp, self._logger) + self.assertFalse(os.path.exists(temp)) + base.StringUtils.toFile(temp, 'x') + base.FileHelper.ensureFileDoesNotExist(temp) + self.assertFalse(os.path.exists(temp)) + # invalid link exists + os.symlink('../invalid-link-source', temp) + base.FileHelper.ensureFileDoesNotExist(temp, self._logger) + self.assertFalse(os.path.exists(temp)) + os.symlink('../invalid-link-source', temp) + base.FileHelper.ensureFileDoesNotExist(temp) + self.assertFalse(os.path.exists(temp)) + + def testEnsureSymbolicLink(self): + tempDir = self.tempDir('jail', 'unittest') + target = tempDir + os.sep + 'parent' + # creating base dir and target: + self.ensureFileDoesNotExist(tempDir) + self.tempDir('sibling', 'unittest') + base.FileHelper.ensureSymbolicLink('../../sibling', target) + self.assertTrue(os.path.islink(target)) + self.assertEquals('../../sibling', os.readlink(target)) + # changing link source: + self.tempDir('sibling2', 'unittest') + base.FileHelper.ensureSymbolicLink('../../sibling2', target, True, self._logger) + self.assertTrue(os.path.islink(target)) + self.assertEquals('../../sibling2', os.readlink(target)) + # removing existing target: + self.ensureFileDoesNotExist(target) + base.StringUtils.toFile(target, 'anything') + base.FileHelper.ensureSymbolicLink('../../sibling2', target, True, self._logger) + self.assertTrue(os.path.islink(target)) + self.assertEquals('../../sibling2', os.readlink(target)) + + def testEnsureSymbolicLinkErrors(self): + tempDir = self.tempDir('jail', 'unittest') + target = tempDir + os.sep + 'parent' + self.ensureDirectory(target) + # creating base dir and target: + self.ensureFileDoesNotExist(tempDir) + self.tempDir('sibling', 'unittest') + self._logger.log('= expecting error is directory') + base.FileHelper.ensureSymbolicLink('../../sibling', target, True, self._logger) + self.assertFalse(os.path.exists(target)) + # must not create parent: + self._logger.log('= expecting error missing parent') + self.ensureFileDoesNotExist(os.path.dirname(target)) + base.FileHelper.ensureSymbolicLink('../../sibling', target, False, self._logger) + self.assertFalse(os.path.exists(target)) + + def testFileClass(self): + baseDir = '/usr/share/pyrshell/unittest/data/' + aClass, subClass = base.FileHelper.fileClass(baseDir + 'example.zip') + self.assertEquals('container', aClass) + self.assertEquals('zip', subClass) + aClass, subClass = base.FileHelper.fileClass(baseDir + 'example.tar') + self.assertEquals('container', aClass) + self.assertEquals('tar', subClass) + aClass, subClass = base.FileHelper.fileClass(baseDir + 'example.tgz') + self.assertEquals('container', aClass) + self.assertEquals('tar', subClass) + aClass, subClass = base.FileHelper.fileClass(baseDir + 'example.tbz') + self.assertEquals('container', aClass) + self.assertEquals('tar', subClass) + aClass, subClass = base.FileHelper.fileClass(baseDir + 'example.html') + self.assertEquals('text', aClass) + self.assertEquals('xml', subClass) + aClass, subClass = base.FileHelper.fileClass(baseDir + 'example.sh') + self.assertEquals('text', aClass) + self.assertEquals('shell', subClass) + aClass, subClass = base.FileHelper.fileClass(baseDir + 'example.txt') + self.assertEquals('text', aClass) + self.assertEquals('text', subClass) + + def testEnsureFileExists(self): + fn = self.tempFile('should.exist.txt', 'unittest') + base.FileHelper.ensureFileDoesNotExist(fn, self._logger) + base.FileHelper.ensureFileExists(fn, 'Hi world', self._logger) + self.assertFileContains('Hi world', fn) + + def testEnsureFileExistsError(self): + fn = self.tempDir('blocking.dir', 'unittest') + self._logger.log('expectig error: blocking dir') + base.FileHelper.ensureFileExists(fn, 'Hi', self._logger) + self.assertDirExists(fn) + + def testCopyDirectoryClear(self): + source = self.tempDir('src', 'unittest') + target = self.tempDir('trg', 'unittest') + base.StringUtils.toFile(source + '/hi.txt', 'Hi') + os.symlink('hi.txt', source + os.sep + 'hi.link.txt') + source2 = self.tempDir('src/dir1', 'unittest') + base.StringUtils.toFile(source2 + '/wow.txt', 'Wow') + if not os.path.exists(source2 + '/wow.symlink.txt'): + os.symlink('wow.txt', source2 + '/wow.symlink.txt') + base.FileHelper.copyDirectory(source, target, 'clear', self._logger, 3) + self.assertFileContains('Hi', target + '/hi.txt') + self.assertDirExists(target + '/dir1') + self.assertFileContains('Wow', target + '/dir1/wow.txt') + trg2 = target + '/dir1/wow.symlink.txt' + self.assertFileContains('Wow', trg2) + self.assertTrue(os.path.islink(trg2)) + fn = target + os.sep + 'hi.link.txt' + self.assertFileExists(fn) + self.assertEquals('hi.txt', os.readlink(fn)) + + def testCopyDirectoryUpdate(self): + source = self.tempDir('src', 'unittest') + target = self.tempDir('trg', 'unittest') + base.StringUtils.toFile(source + '/hi.txt', 'Hi') + source2 = self.tempDir('src/dir1', 'unittest') + base.StringUtils.toFile(source2 + '/wow.txt', 'Wow') + base.FileHelper.copyDirectory(source, target, 'clear', self._logger, 3) + time.sleep(1) + base.StringUtils.toFile(source + '/hi.txt', 'hi!') + base.FileHelper.setModified(source + '/hi.txt', 365*24*3600) + base.StringUtils.toFile(source + '/hi2.txt', 'hi!') + base.StringUtils.toFile(source2 + '/wow2.txt', 'wow!') + base.FileHelper.setModified(source2 + '/wow2.txt', 365*24*3600) + base.FileHelper.copyDirectory(source, target, 'update', self._logger) + self.assertFileContains('Hi', target + '/hi.txt') + self.assertFileContains('hi!', target + '/hi2.txt') + self.assertDirExists(target + '/dir1') + self.assertFileContains('Wow', target + '/dir1/wow.txt') + self.assertFileContains('wow!', target + '/dir1/wow2.txt') + + def testUnpackTgz(self): + target = self.tempDir('unittest') + fn = target + os.sep + 'dummy' + base.StringUtils.toFile(fn, '') + base.FileHelper.unpack('/usr/share/pyrshell/unittest/data/etc.work.tgz', target, self._logger, True) + self.assertFileNotExists(fn) + self.assertFileExists(target + '/etc/passwd') + self.assertFileExists(target + '/etc/nginx/sites-available/default') + + def testUnpackZip(self): + target = self.tempDir('archive', 'unittest') + base.FileHelper.unpack('/usr/share/pyrshell/unittest/data/example.zip', target, self._logger, True) + self.assertFileExists(target + '/All.sh') + + def testTempFile(self): + fn = base.FileHelper.tempFile('test.txt', 'unittest.2', self._logger) + parent = os.path.dirname(fn) + self.assertEquals('test.txt', os.path.basename(fn)) + self.assertEquals('unittest.2', os.path.basename(parent)) + self.assertFileExists(parent) + os.rmdir(parent) + +if __name__ == '__main__': + # import sys;sys.argv = ['', 'Test.testName'] + tester = FileHelperTest() + tester.run() diff --git a/unittest/base/JavaConfigTest.py b/unittest/base/JavaConfigTest.py new file mode 100644 index 0000000..d021c29 --- /dev/null +++ b/unittest/base/JavaConfigTest.py @@ -0,0 +1,68 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +import os +from unittest.UnitTestCase import UnitTestCase +import base.JavaConfig +import base.StringUtils +import base.MemoryLogger + +class JavaConfigTest(UnitTestCase): + + def testBasic(self): + logger = base.MemoryLogger.MemoryLogger() + fn = self.tempFile('javaconf.conf') + base.StringUtils.toFile(fn, '# comment\nabc.def=/dev\n\t\n\tFile = /tmp/x') + config = base.JavaConfig.JavaConfig(fn, logger) + self.assertEquals('/dev', config.getString('abc.def')) + self.assertEquals('/tmp/x', config.getString('File')) + self.assertNone(config.getString('file')) + self.assertNone(config.getString('unknown')) + os.unlink(fn) + + def testSyntaxError(self): + fn = self.tempFile('error.conf') + base.StringUtils.toFile(fn, '# comment\nabc.def:=/dev\n\t\n\tFile') + logger = base.MemoryLogger.MemoryLogger() + base.JavaConfig.JavaConfig(fn, logger) + self.assertTrue(logger.contains('error.conf line 2: unexpected syntax [expected: <var>=<value>]: abc.def:=/dev')) + self.assertTrue(logger.contains('error.conf line 4: unexpected syntax [expected: <var>=<value>]: File')) + + def testIntVar(self): + fn = self.tempFile('javaconf.conf') + base.StringUtils.toFile(fn, '# comment\nnumber=123\nWrong = zwo') + logger = base.MemoryLogger.MemoryLogger() + config = base.JavaConfig.JavaConfig(fn, logger) + self.assertEquals(123, config.getInt('number')) + self.assertEquals(456, config.getInt('unknown', 456)) + self.assertEquals(111, config.getInt('Wrong', 111)) + self.assertTrue(logger.contains('avaconf.conf: variable Wrong is not an integer: zwo')) + os.unlink(fn) + + def testGetKeys(self): + fn = self.tempFile('javaconf.conf') + base.StringUtils.toFile(fn, '# comment\nnumber=123\nWrong = zwo') + logger = base.MemoryLogger.MemoryLogger() + config = base.JavaConfig.JavaConfig(fn, logger) + keys = config.getKeys() + self.assertEquals(2, len(keys)) + self.assertEquals('Wrong', keys[0]) + self.assertEquals('number', keys[1]) + os.unlink(fn) + + def testGetKeysRegExpr(self): + fn = self.tempFile('javaconf.conf') + base.StringUtils.toFile(fn, '# comment\nnumber=123\nWrong = zwo') + logger = base.MemoryLogger.MemoryLogger() + config = base.JavaConfig.JavaConfig(fn, logger) + keys = config.getKeys(r'number|int') + self.assertEquals(1, len(keys)) + self.assertEquals('number', keys[0]) + os.unlink(fn) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = JavaConfigTest() + tester.run() diff --git a/unittest/base/LinuxUtilsTest.py b/unittest/base/LinuxUtilsTest.py new file mode 100644 index 0000000..3ef95c2 --- /dev/null +++ b/unittest/base/LinuxUtilsTest.py @@ -0,0 +1,96 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import base.LinuxUtils + +class LinuxUtilsTest(UnitTestCase): + + def testDiskFree(self): + infos = base.LinuxUtils.diskFree() + x = self.__dict__ + self.assertTrue(len(infos) >= 1) + for info in infos: + if info[0] not in ['/', '/opt', '/work', '/home'] and not info[0].startswith('/media') and info[0].find('jail') < 0: + self.assertEquals('valid path', info[0]) + self.assertTrue(type(info[1]) == int) + self.assertTrue(type(info[2]) == int) + self.assertTrue(type(info[3]) == int) + self.assertTrue(info[1] >= info[2]) + self.assertTrue(info[1] >= info[3]) + + def testUsers(self): + infos = base.LinuxUtils.users() + self.assertTrue(len(infos) >= 1) + for info in infos: + self.assertMatches(r'[\w]+', info[0]) + self.assertMatches(r'(:?\d+)(\.\d+)?|([\d.]+)', info[1]) + self.assertMatches(r'[0-2]?\d+:[0-5]\d', info[2]) + + def testLoad(self): + info = base.LinuxUtils.load() + self.assertEquals(5, len(info)) + for ix in range(3): + self.assertTrue(type(info[ix]) == float) + self.assertTrue(type(info[3]) == int) + self.assertTrue(type(info[4]) == int) + self.assertTrue(int(info[3]) < int(info[4])) + + def testMemoryInfo(self): + info = base.LinuxUtils.memoryInfo() + self.assertEquals(5, len(info)) + # TOTAL_RAM, AVAILABLE_RAM, TOTAL_SWAP, FREE_SWAP, BUFFERS + for ix in range(len(info)): + self.assertTrue(type(info[ix]) == int) + self.assertTrue(info[0] >= info[1]) + self.assertTrue(info[0] >= info[2]) + self.assertTrue(info[2] >= info[3]) + + def checkMdadm(self, name, aType, members, blocks, status, info): + self.assertEquals(name, info[0]) + self.assertEquals(aType, info[1]) + self.assertEquals(members, info[2]) + self.assertEquals(blocks, info[3]) + self.assertEquals(status, info[4]) + + def testMdadmInfo(self): + fn = self.tempFile('mdadm.info') + with open(fn, "w") as fp: + fp.write('''Personalities : [raid1] +md2 : active raid1 sdc1[0] sdd1[1] + 1953378368 blocks super 1.2 [2/2] [UU] + bitmap: 0/15 pages [0KB], 65536KB chunk + +md1 : active raid1 sda2[0] sdb2[1] + 508523520 blocks super 1.2 [2/2] [UU] + bitmap: 2/4 pages [8KB], 65536KB chunk + +md0 : active raid1 sda1[0] sdb1[1] + 242496 blocks super 1.2 [2/2] [UU] +''') + infos = base.LinuxUtils.mdadmInfo(fn) + self.assertEquals(3, len(infos)) + self.checkMdadm('md2', 'raid1', 'sdc1[0] sdd1[1]', 1953378368, 'OK', infos[0]) + self.checkMdadm('md1', 'raid1', 'sda2[0] sdb2[1]', 508523520, 'OK', infos[1]) + self.checkMdadm('md0', 'raid1', 'sda1[0] sdb1[1]', 242496, 'OK', infos[2]) + + def testMdadmInfoBroken(self): + fn = self.tempFile('mdadm.info') + with open(fn, "w") as fp: + fp.write('''Personalities : [raid1] +md1 : active raid1 hda14[0] sda11[2](F) + 2803200 blocks [2/1] [U_]''') + infos = base.LinuxUtils.mdadmInfo(fn) + self.assertEquals(1, len(infos)) + self.checkMdadm('md1', 'raid1', 'hda14[0] sda11[2](F)', 2803200, 'broken', infos[0]) + + def testStress(self): + info = base.LinuxUtils.stress(r'^(sda|nvme0n1)$', r'^(enp2s0|wlp4s0)$') + self.assertEquals(7, len(info)) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = LinuxUtilsTest() + tester.run() diff --git a/unittest/base/LoggerTest.py b/unittest/base/LoggerTest.py new file mode 100644 index 0000000..6ad5c24 --- /dev/null +++ b/unittest/base/LoggerTest.py @@ -0,0 +1,92 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +from base.Logger import Logger +import os +import re +# import from base.Logger Logger + +class LoggerTest(UnitTestCase): + + def testLogger(self): + logFile = '/tmp/logger.log' + if os.path.isfile(logFile): + os.remove(logFile) + logger = Logger(logFile, True) + logger.log('Hi world') + logger.error('an expected error') + logger.debug('debug message') + self.assertFileContains('Hi world', logFile) + self.assertFileContains('+++ an expected error', logFile) + self.assertFileContains('debug message', logFile) + self.assertEquals('an expected error', logger._firstErrors[0]) + self.assertEquals(1, logger._errors) + + def testTextFilter(self): + logFile = '/tmp/logger.log' + if os.path.isfile(logFile): + os.remove(logFile) + logger = Logger(logFile, True) + logger.setErrorFilter('[second]') + logger.log('Hi world') + logger.error('an expected error') + logger.error('a [second] expected error') + logger.debug('debug message') + self.assertFileContains('Hi world', logFile) + self.assertFileContains('+++ an expected error', logFile) + self.assertFileNotContains('a [second] expected error', logFile) + self.assertFileContains('debug message', logFile) + self.assertEquals('an expected error', logger._firstErrors[0]) + self.assertEquals(1, logger._errors) + + def testRegExprFilter(self): + logFile = '/tmp/logger.log' + if os.path.isfile(logFile): + os.remove(logFile) + logger = Logger(logFile, True) + logger.setErrorFilter(re.compile('second|third')) + logger.log('Hi world') + logger.error('an expected error') + logger.error('a [second] expected error') + logger.debug('debug message') + self.assertFileContains('Hi world', logFile) + self.assertFileContains('+++ an expected error', logFile) + self.assertFileNotContains('a [second] expected error', logFile) + self.assertFileContains('debug message', logFile) + self.assertEquals('an expected error', logger._firstErrors[0]) + self.assertEquals(1, logger._errors) + + def testMirror(self): + logFile1 = '/tmp/logger1.log' + if os.path.isfile(logFile1): + os.remove(logFile1) + logger = Logger(logFile1, True) + + logFile2 = '/tmp/logger2.log' + if os.path.isfile(logFile2): + os.remove(logFile2) + loggerMirror = Logger(logFile2, True) + logger.setMirror(loggerMirror) + + logger.log('Hi world') + logger.error('an expected error') + logger.debug('debug message') + self.assertFileContains('Hi world', logFile1) + self.assertFileContains('+++ an expected error', logFile1) + self.assertFileContains('debug message', logFile1) + self.assertEquals('an expected error', logger._firstErrors[0]) + self.assertEquals(1, logger._errors) + + self.assertFileContains('Hi world', logFile2) + self.assertFileContains('+++ an expected error', logFile2) + self.assertFileContains('debug message', logFile2) + self.assertEquals('an expected error', loggerMirror._firstErrors[0]) + self.assertEquals(1, loggerMirror._errors) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = LoggerTest() + tester.run() diff --git a/unittest/base/MemoryLoggerTest.py b/unittest/base/MemoryLoggerTest.py new file mode 100644 index 0000000..e0b5646 --- /dev/null +++ b/unittest/base/MemoryLoggerTest.py @@ -0,0 +1,27 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import base.MemoryLogger + +class MemoryLoggerTest(UnitTestCase): + + def testBase(self): + logger = base.MemoryLogger.MemoryLogger() + logger.log('Hi world') + logger.error('an expected error') + logger.debug('debug message') + self.assertTrue(logger.contains('Hi world')) + self.assertTrue(logger.contains('an expected error')) + self.assertTrue(logger.contains('debug message')) + self.assertFalse(logger.contains('Hi world!')) + self.assertTrue(logger.matches(r'Hi\sworld')) + self.assertTrue(logger.matches(r'an [a-z]+ error')) + self.assertTrue(logger.matches(r'^de.*sage$')) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = MemoryLoggerTest() + tester.run() diff --git a/unittest/base/ProcessHelperTest.py b/unittest/base/ProcessHelperTest.py new file mode 100644 index 0000000..e981e66 --- /dev/null +++ b/unittest/base/ProcessHelperTest.py @@ -0,0 +1,70 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase + +import os + +import base.ProcessHelper +import base.StringUtils + +debug = False + +def usage(msg=None): + return 'test usage' + +class ProcessHelperTest(UnitTestCase): + def __init__(self): + UnitTestCase.__init__(self) + self._dir = self.tempDir('processtool', 'unittest') + self._testFile = self._dir + os.sep + 'simple.file.txt' + base.StringUtils.toFile(self._testFile, 'line 1\nline 2\nline 3\n') + self._tool = base.ProcessHelper.ProcessHelper(3, base.Logger.Logger('/tmp/processtooltest.log', True)) + + def getOptions(self): + appInfo = base.BaseTool.ApplicationInfo('processHelper', 'base.WikiTool', usage) + rc = base.BaseTool.GlobalOptions(appInfo) + rc._verboseLevel = 0 + return rc + + def testExecute(self): + self._tool.execute(['tail', '-n2', self._testFile], True) + if self.assertTrue(len(self._tool._output) == 2): + self.assertEquals('line 2', self._tool._output[0]) + self.assertEquals('line 3', self._tool._output[1]) + + def testExecuteError(self): + self._tool._logger.log('expecting an error:') + self._tool.execute(['tail', '-n2', '/etc/shadow'], True) + self.assertEquals(0, len(self._tool._output)) + self.assertTrue(self._tool._error[0].startswith("tail: '/etc/shadow'")) + + def testExecuteInput(self): + self._tool.executeInput(['grep', '-o', '[0-9][0-9]*'], True, 'line1\n\line222') + self.assertEquals('1', self._tool._output[0]) + self.assertEquals('222', self._tool._output[1]) + + def testExecuteInputError(self): + self._tool._logger.log('expecting an error:') + self._tool.executeInput(['veryUnknownCommand!', '[0-9]+'], True, 'line1\n\line222') + self.assertEquals(0, len(self._tool._output)) + self.assertEquals("[Errno 2] No such file or directory: 'veryUnknownCommand!': 'veryUnknownCommand!'", self._tool._error[0]) + + def testExecuteScript(self): + rc = self._tool.executeScript('#! /bin/bash\n/bin/echo $1', 'getArg1', True, ['Hi world', 'Bye world']) + #if self.assertEquals(1, len(rc)): + # self.assertEquals('Hi world', rc[0]) + + def testExecuteInChain(self): + fn = self.tempFile('gzip.input', 'unittest') + base.StringUtils.toFile(fn, 'Hi') + rc = self._tool.executeInChain(['gzip', '-c', fn], None, ['zcat'], '!shell') + #if self.assertEquals(1, len(rc)): + # self.assertEquals('Hi world', rc[0]) + +if __name__ == '__main__': + # import sys;sys.argv = ['', 'Test.testName'] + tester = ProcessHelperTest() + tester.run() diff --git a/unittest/base/PythonToTypeScriptTest.py b/unittest/base/PythonToTypeScriptTest.py new file mode 100644 index 0000000..3588aac --- /dev/null +++ b/unittest/base/PythonToTypeScriptTest.py @@ -0,0 +1,230 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import base.PythonToTypeScript +import appl.TextTool + +debug = False + +def usage(msg=None): + return 'test usage' + +class PythonToTypeScriptTest(UnitTestCase): + + def __init__(self): + UnitTestCase.__init__(self) + + self._python = appl.TextTool.TextTool(self.getOptions()) + self._typeScript = appl.TextTool.TextTool(self.getOptions()) + self._tool = base.PythonToTypeScript.PythonToTypeScript(self._python, self._typeScript) + + def fill(self, pythonAsString): + self._python._lines = pythonAsString.split('\n') + self._typeScript._lines.clear() + + def getOptions(self): + appInfo = base.BaseTool.ApplicationInfo('wiki', 'appl.WikiTool', usage) + rc = base.BaseTool.GlobalOptions(appInfo) + rc._exampleText = 'log.file=/var/log/local/pythontotypescript.log' + rc._logFiles.append('-') + return rc + + def check(self, expected): + self._tool.convert() + current = '\n'.join(self._typeScript._lines) + base.StringUtils.toFile('/tmp/ucur.txt', current) + base.StringUtils.toFile('/tmp/uexp.txt', expected) + self.assertEquals(expected, current) + + def testForIfWhile(self): + if debug: + return + self.fill(""" +def fibu(n): + # first fun + while True: + rc = 0 + # if: + if n < 3: + #then + rc = 1 + # then end + else: + # while + while False: + # pass: + pass + # for: + for x in range(n): + # add + rc += fibu(n-1) + fibu(n-2) + # result + break + return rc + # end fun +""") + self.check("""fibu(n: string): number { + // first fun + while (true) { + let rc = 0; + // if: + if (n < 3) { + //then + let rc = 1; + // then end + } else { + // while + while (false) { + // pass: + pass; + // for: + } // while + for (let x in range(n)) { + // add + rc += fibu(n-1) + fibu(n-2); + // result + } // for + } // else + break; + } // while + return rc; + // end fun +}""") + + def testFor(self): + if debug: + return + self.fill("""# first +import os.path +# 2nd +def fibu(n): +# first fun + rc = 0 + # for: + for x in range(n): + # add + rc += fibu(n-1) + fibu(n-2) + # result + return rc + # end fun +""") + self.check("""// first +// 2nd +fibu(n: string): number { +// first fun + let rc = 0; + // for: + for (let x in range(n)) { + // add + rc += fibu(n-1) + fibu(n-2); + // result + } // for + return rc; + // end fun +} +""") + + def testDef(self): + if debug: + return + self.fill(""" +import os.path +def testIt(name): + '''Tests whether a given file exists. + @param name: the filename + @return: True: the file exists + ''' + if os.path.exists(name): + return True + else: + return False + return False +""") + self.check("""/** + * Tests whether a given file exists. + * @param name: the filename + * @return: true: the file exists + */ +testIt(name: string): boolean { + if (os.path.exists(name)) { + return true; + } else { + return false; + } // else + return false; +} +""") + + def testClass(self): + if False and debug: + return + self.fill(""" +class Derived (base.BaseClass): +""\"A derivation of the BaseClass. +Not realy meaningful. +""\" + def __init__(self): + '''Constructor. + ''' + base.BaseClass.__init__(self, 'default') + self._count = 1 + self._stop = False + self._dict = dict() + + def fac(n): + '''Iterative calculation of faculty + @param n: the filename + @return: the faculty of n + ''' + self._log.append("fac") + rc = 1 + while n > 1: + self._count += 1 + rc *= n + n -= 1 + return rc +""") + self.check("""/** + * A derivation of the BaseClass. + * Not realy meaningful. + */ +export class Derived extends BaseClass { + private _count: number; + private _stop: boolean; + private _dict: object; + private _log: object; + /** + * Constructor. + */ + constructor() { + super('default'); + this._count = 1; + this._stop = false; + this._dict = dict(); + } + + /** + * Iterative calculation of faculty + * @param n: the filename + * @return: the faculty of n + */ + fac(n: string): number { + this._log.append("fac"); + let rc = 1; + while (n > 1) { + this._count += 1; + rc *= n; + n -= 1; + } // while + return rc; + } +} +""") + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = PythonToTypeScriptTest() + tester.run() diff --git a/unittest/base/StringUtilsTest.py b/unittest/base/StringUtilsTest.py new file mode 100644 index 0000000..cac6a7e --- /dev/null +++ b/unittest/base/StringUtilsTest.py @@ -0,0 +1,301 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import base.StringUtils +import os +import re +from base import StringUtils +# import from base.Logger Logger + +class StringUtilsTest(UnitTestCase): + + def testJoin(self): + self.assertEquals('1 2 3', base.StringUtils.join(' ', [1,2,3])) + self.assertEquals('1,B,[]', base.StringUtils.join(',', [1, 'B', []])) + self.assertEquals('A.B.C', base.StringUtils.join('.', ['A', 'B', 'C'])) + self.assertEquals('', base.StringUtils.join('.', None)) + + def testToFile(self): + fn = '/tmp/stringutils.tmp' + if os.path.exists(fn): + os.unlink(fn) + content = 'line1\nline2' + base.StringUtils.toFile(fn, content) + self.assertTrue(os.path.exists(fn)) + self.assertFileContains('line1', fn) + self.assertFileContains('line2', fn) + + def testFromFile(self): + fn = '/tmp/stringutils.tmp' + content = 'xline1\nxline2' + base.StringUtils.toFile(fn, content) + current = base.StringUtils.fromFile(fn) + self.assertEquals(content, current) + + def testFromFileSep(self): + fn = '/tmp/stringutils.tmp' + content = 'xline1\nxline2' + base.StringUtils.toFile(fn, content) + current = base.StringUtils.fromFile(fn, '\n') + self.assertEquals(content.split('\n'), current) + + def testTailOfWord(self): + self.assertEquals('x', base.StringUtils.tailOfWord('-ax', '-a')) + self.assertEquals('x', base.StringUtils.tailOfWord('-b -ax', '-a')) + self.assertEquals('x', base.StringUtils.tailOfWord('-ax -b', '-a')) + self.assertEquals('x', base.StringUtils.tailOfWord('-c -ax -b', '-a')) + self.assertEquals('x', base.StringUtils.tailOfWord('-ax\t -b', '-a')) + self.assertEquals('x', base.StringUtils.tailOfWord('y \t-ax\t -b', '-a')) + + self.assertNone(base.StringUtils.tailOfWord('--find-a-ax', '-a')) + self.assertNone(base.StringUtils.tailOfWord('-b\t-c -d', '-a')) + + def testFormatSize(self): + self.assertEquals('120 Byte', base.StringUtils.formatSize(120)) + self.assertEquals('123.456 KB', base.StringUtils.formatSize(123456)) + self.assertEquals('123.456 MB', base.StringUtils.formatSize(123456*1000)) + self.assertEquals('12.346 MB', base.StringUtils.formatSize(123456*100)) + self.assertEquals('1.235 MB', base.StringUtils.formatSize(123456*10)) + self.assertEquals('123.456 GB', base.StringUtils.formatSize(123456*1000*1000)) + self.assertEquals('123.456 TB', base.StringUtils.formatSize(123456*1000*1000*1000)) + + def testHasContent(self): + filename = self.tempFile('example.txt', 'stringutiltest') + base.StringUtils.toFile(filename, '') + self.assertFalse(base.StringUtils.hasContent(filename)) + base.StringUtils.toFile(filename, '# comment') + self.assertFalse(base.StringUtils.hasContent(filename)) + base.StringUtils.toFile(filename, '# comment\n\t \n\n#comment2') + self.assertFalse(base.StringUtils.hasContent(filename)) + self.assertFalse(base.StringUtils.hasContent(filename + '.not.existing')) + base.StringUtils.toFile(filename, '\t// comment\n\t \n\n//comment2') + self.assertFalse(base.StringUtils.hasContent(filename, '//')) + + base.StringUtils.toFile(filename, '\t// comment\n\t \n\//comment2') + self.assertTrue(base.StringUtils.hasContent(filename, '#')) + base.StringUtils.toFile(filename, '# has content!\n\na=3') + self.assertTrue(base.StringUtils.hasContent(filename, '#')) + + def testFirstMatch(self): + aList = ['# a=2', '#', 'b=3', '\t name = Jonny Cash '] + regExpr = re.compile(r'^\s*(\w+)\s*=\s*(.*?)\s*$') + matcher = base.StringUtils.firstMatch(aList, regExpr) + self.assertNotNone(matcher) + self.assertEquals('b', matcher.group(1)) + self.assertEquals('3', matcher.group(2)) + + matcher = base.StringUtils.firstMatch(aList, regExpr, 3) + self.assertNotNone(matcher) + self.assertEquals('name', matcher.group(1)) + self.assertEquals('Jonny Cash', matcher.group(2)) + + def testGrepInFile(self): + filename = self.tempFile('grep.txt', 'stringutiltest') + base.StringUtils.toFile(filename, """# Test +a = 1 +# öäü b = 2 +c=333 +""") + regExpr = re.compile(r'^\s*(\w+)\s*=\s*(.*?)\s*$') + found = base.StringUtils.grepInFile(filename, regExpr) + self.assertEquals(2, len(found)) + self.assertEquals('a = 1\n', found[0]) + self.assertEquals('c=333\n', found[1]) + + found = base.StringUtils.grepInFile(filename, regExpr, 1) + self.assertEquals(1, len(found)) + self.assertEquals("a = 1\n", found[0]) + + def testGrepInFileGroup(self): + filename = self.tempFile('grep.txt', 'stringutiltest') + base.StringUtils.toFile(filename, """# Test +a = 1 +# öäü b = 2 +c=333 +""") + regExpr = re.compile(r'^\s*\w+\s*=\s*(.*?)\s*$') + found = base.StringUtils.grepInFile(filename, regExpr, None, 1) + self.assertEquals(2, len(found)) + self.assertEquals('1', found[0]) + self.assertEquals('333', found[1]) + + found = base.StringUtils.grepInFile(filename, regExpr, 1) + self.assertEquals(1, len(found)) + self.assertEquals("a = 1\n", found[0]) + + def testLimitItemLength_WithoutElipsis(self): + source = ['1', '22', '333', '4444'] + result = base.StringUtils.limitItemLength(source, 2) + self.assertEquals(source[0], '1') + self.assertEquals(source[3], '4444') + self.assertEquals(len(source), len(result)) + for ix in range(len(source)): + self.assertEquals(source[ix][0:2], result[ix]) + result = base.StringUtils.limitItemLength(source, 0) + self.assertEquals('', ''.join(result)) + + def testLimitItemLength(self): + source = ['abcd1', 'abcd22', 'abcd333', 'abcd4444'] + result = base.StringUtils.limitItemLength(source, 5) + self.assertEquals(source[0], 'abcd1') + self.assertEquals(source[3], 'abcd4444') + self.assertEquals(len(source), len(result)) + for ix in range(len(source)): + if ix == 0: + self.assertEquals(source[ix], result[ix]) + else: + self.assertEquals(source[ix][0:2] + '...', result[ix]) + result = base.StringUtils.limitItemLength(source, 0) + self.assertEquals('', ''.join(result)) + + def testToFloatAndTypeDate(self): + [value, dataType] = base.StringUtils.toFloatAndType('2019.10.23') + self.assertEquals(1571781600.0, value) + self.assertEquals('date', dataType) + [value, dataType] = base.StringUtils.toFloatAndType('1970-01-01') + self.assertEquals(-3600.0, value) + self.assertEquals('date', dataType) + + def testToFloatAndTypeTime(self): + [value, dataType] = base.StringUtils.toFloatAndType('01:02:03') + self.assertEquals(1*3600+2*60+3, value) + self.assertEquals('time', dataType) + [value, dataType] = base.StringUtils.toFloatAndType('2:17') + self.assertEquals(2*3600+17*60, value) + self.assertEquals('time', dataType) + + def testToFloatAndTypeDateTime(self): + [value, dataType] = base.StringUtils.toFloatAndType('2019.10.23T01:02:03') + self.assertEquals(1571785323.0, value) + self.assertEquals('datetime', dataType) + [value, dataType] = base.StringUtils.toFloatAndType('1970-01-02 5:17') + self.assertEquals(101820.0, value) + self.assertEquals('datetime', dataType) + + def testToFloatAndTypeHex(self): + [value, dataType] = base.StringUtils.toFloatAndType('0x1234') + self.assertEquals(float(0x1234), value) + self.assertEquals('int', dataType) + [value, dataType] = base.StringUtils.toFloatAndType('0XABCDEF0123456') + self.assertEquals(float(0xABCDEF0123456), value) + self.assertEquals('int', dataType) + [value, dataType] = base.StringUtils.toFloatAndType('0Xabcdef0') + self.assertEquals(float(0xABCDEF0), value) + self.assertEquals('int', dataType) + + def testToFloatAndTypeOct(self): + [value, dataType] = base.StringUtils.toFloatAndType('01234') + self.assertEquals(float(0o1234), value) + self.assertEquals('int', dataType) + [value, dataType] = base.StringUtils.toFloatAndType('012345670') + self.assertEquals(float(0o12345670), value) + self.assertEquals('int', dataType) + + def testToFloatAndTypeInt(self): + [value, dataType] = base.StringUtils.toFloatAndType('1234') + self.assertEquals(1234.0, value) + self.assertEquals('int', dataType) + [value, dataType] = base.StringUtils.toFloatAndType('987654321') + self.assertEquals(987654321.0, value) + self.assertEquals('int', dataType) + + def testToFloatAndTypeFloat(self): + [value, dataType] = base.StringUtils.toFloatAndType('1234.0') + self.assertEquals(1234.0, value) + self.assertEquals('float', dataType) + [value, dataType] = base.StringUtils.toFloatAndType('987654321.0') + self.assertEquals(987654321.0, value) + self.assertEquals('float', dataType) + [value, dataType] = base.StringUtils.toFloatAndType('1.23E+44') + self.assertEquals(1.23E+44, value) + self.assertEquals('float', dataType) + + def testToFloatAndTypeError(self): + [value, dataType] = base.StringUtils.toFloatAndType('host3') + self.assertEquals('float (or int or date(time)) expected, found: host3', value) + self.assertEquals('undef', dataType) + + def testToFloatDate(self): + value = base.StringUtils.toFloat('2019.10.23') + self.assertEquals(1571781600.0, value) + value = base.StringUtils.toFloat('1970-01-01') + self.assertEquals(-3600.0, value) + + def testToFloatTime(self): + value = base.StringUtils.toFloat('01:02:03') + self.assertEquals(1*3600+2*60+3, value) + value = base.StringUtils.toFloat('2:17') + self.assertEquals(2*3600+17*60, value) + + def testToFloatDateTime(self): + value = base.StringUtils.toFloat('2019.10.23T01:02:03') + self.assertEquals(1571785323.0, value) + value = base.StringUtils.toFloat('1970-01-02 5:17') + self.assertEquals(101820.0, value) + + def testToFloatHex(self): + value = base.StringUtils.toFloat('0x1234') + self.assertEquals(float(0x1234), value) + value = base.StringUtils.toFloat('0XABCDEF0123456') + self.assertEquals(float(0xABCDEF0123456), value) + value = base.StringUtils.toFloat('0Xabcdef0') + self.assertEquals(float(0xABCDEF0), value) + + def testToFloatOct(self): + value = base.StringUtils.toFloat('01234') + self.assertEquals(float(0o1234), value) + value = base.StringUtils.toFloat('012345670') + self.assertEquals(float(0o12345670), value) + + def testToFloatInt(self): + value = base.StringUtils.toFloat('1234') + self.assertEquals(1234.0, value) + value = base.StringUtils.toFloat('987654321') + self.assertEquals(987654321.0, value) + + def testToFloatFloat(self): + value = base.StringUtils.toFloat('1234.0') + self.assertEquals(1234.0, value) + value = base.StringUtils.toFloat('987654321.0') + self.assertEquals(987654321.0, value) + value = base.StringUtils.toFloat('1.23E+44') + self.assertEquals(1.23E+44, value) + + def testToFloatError(self): + value = base.StringUtils.toFloat('host3') + self.assertEquals('float (or int or date(time)) expected, found: host3', value) + + def testAsInt(self): + self.assertEquals(321, base.StringUtils.asInt('321')) + self.assertEquals(0x321, base.StringUtils.asInt('0x321')) + self.assertEquals(0o321, base.StringUtils.asInt('0321')) + self.assertEquals(77, base.StringUtils.asInt('99x', 77)) + self.assertEquals(777, base.StringUtils.asInt('x2', 777)) + + def testRegExprCompile(self): + rexpr = base.StringUtils.regExprCompile('\d', None, None, True) + self.assertNotNone(rexpr.match('7')) + rexpr = base.StringUtils.regExprCompile('Hi', None, None, False) + self.assertNotNone(rexpr.match('hi')) + + def testRegExprCompileError(self): + rexpr = base.StringUtils.regExprCompile('*.txt', 'test of wrong pattern', self._logger) + self.assertNone(rexpr) + self._logger.contains('error in regular expression in test of wrong pattern: nothing to repeat at position 0') + rexpr = base.StringUtils.regExprCompile('(*.txt', 'test of wrong pattern') + self.assertNone(rexpr) + + def testMinimizeArrayUtfError(self): + list = [b'\xffabcdefghijklmnopqrstuvwxyz01234567890', b'abcdefghijklmnopqrstuvwxyz01234567890\xff'] + rc = base.StringUtils.minimizeArrayUtfError(list, self._logger) + self.assertEquals(2, len(rc)) + self.assertEquals(1, rc[0].find('abcdefghijklmnopqrstuvwxyz01234567890')) + self.assertEquals(0, rc[1].find('abcdefghijklmnopqrstuvwxyz01234567890')) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = StringUtilsTest() + tester.run() diff --git a/unittest/base/TextProcessorTest.py b/unittest/base/TextProcessorTest.py new file mode 100644 index 0000000..fe5e599 --- /dev/null +++ b/unittest/base/TextProcessorTest.py @@ -0,0 +1,481 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +from base.TextProcessor import TextProcessor, IntegerVariable, TextVariable,\ + DictVariable, ListVariable +import base.Logger +from base import StringUtils +import appl.TextTool + +debug = True +class TextProcessorTest(UnitTestCase): + def __init__(self): + UnitTestCase.__init__(self) + self._logger = base.Logger.Logger('/tmp/unittest.log', True) + + def checkPos(self, processor, line, col): + self.assertEquals(line, processor._currentPos._line) + self.assertEquals(col, processor._currentPos._col) + + def checkMark(self, processor, ixMark, line, col): + self.assertEquals(line, processor._marks[ixMark]._line) + self.assertEquals(col, processor._marks[ixMark]._col) + + def testContentString(self): + if debug: + return + processor = TextProcessor(None, 'line1\nline2', self._logger) + self.assertEquals(2, len(processor._lines)); + self.assertEquals('line1', processor._lines[0]); + self.assertEquals('line2', processor._lines[1]); + + def testContentList(self): + if debug: + return + processor = TextProcessor(None, ['xA', 'yB'], self._logger) + self.assertEquals(2, len(processor._lines)); + self.assertEquals('xA', processor._lines[0]); + self.assertEquals('yB', processor._lines[1]); + + def testIndexOf(self): + if debug: + return + processor = TextProcessor(None, 'line1\nline2', self._logger) + aTuple = processor.indexOf('2') + self.assertNotNone(aTuple) + (ixLine, col, length, matcher) = aTuple + self.assertEquals(1, ixLine); + self.assertEquals(4, col); + self.assertEquals(1, length); + self.assertEquals('2', matcher.group()); + + def testIndexOfMultiple(self): + if debug: + return + processor = TextProcessor(None, 'line1\nline2', self._logger) + aTuple = processor.indexOfMultiple(['1', '2']) + self.assertNotNone(aTuple) + (ixLine, col, length, matcher) = aTuple + self.assertEquals(1, ixLine); + self.assertEquals(4, col); + self.assertEquals(1, length); + self.assertEquals('2', matcher.group()); + + def testIndexOf2AncorsSameLine(self): + if debug: + return + processor = TextProcessor(None, 'abc<a href="abc">', self._logger) + aTuple = processor.indexOfMultiple(['"', 'abc']) + self.assertNotNone(aTuple) + (ixLine, col, length, matcher) = aTuple + self.assertEquals(0, ixLine); + self.assertEquals(12, col); + self.assertEquals(3, length); + self.assertEquals('abc', matcher.group()); + processor = TextProcessor(None, 'abc\nabc<a href="abc">', self._logger) + aTuple = processor.indexOfMultiple(['href', 'abc']) + self.assertNotNone(aTuple) + (ixLine, col, length, matcher) = aTuple + self.assertEquals(1, ixLine); + self.assertEquals(12, col); + self.assertEquals(3, length); + self.assertEquals('abc', matcher.group()); + self.assertTrue(processor._errors == 0) + + def checkTuple(self, expectedline, expectedCol, expectedString, aTuple): + self.assertNotNone(aTuple) + (ixLine, col, length, matcher) = aTuple + self.assertEquals(expectedline, ixLine) + self.assertEquals(expectedCol, col) + self.assertEquals(len(expectedString), length) + self.assertEquals(expectedString, matcher.group()) + + def testRIndex(self): + if debug: + return + # ...............................0123456789 123456789 + processor = TextProcessor(None, 'abc<a href="abc">', self._logger) + aTuple = processor.rindexOf('abc') + self.checkTuple(0, 12, 'abc', aTuple) + aTuple = processor.rindexOf('abc', 0, 15) + self.checkTuple(0, 12, 'abc', aTuple) + aTuple = processor.rindexOf('abc', 0, 14) + self.checkTuple(0, 0, 'abc', aTuple) + self.assertTrue(processor._errors == 0) + + def testReplaceOneHit(self): + if debug: + return + processor = TextProcessor(None, '\tabcd\nabc', self._logger) + processor.replace('abc', 'xy', 1) + self.assertEquals('\txyd', processor._lines[0]); + self.assertEquals('abc', processor._lines[1]); + self.assertTrue(processor._errors == 0) + + def testReplaceManyHits(self): + if debug: + return + processor = TextProcessor(None, 'abc.def\nabcabc', self._logger) + processor.replace('abc', 'xy') + self.assertEquals('xy.def', processor._lines[0]); + self.assertEquals('xyxy', processor._lines[1]); + self.assertTrue(processor._errors == 0) + + def testDeleteChar(self): + if debug: + return + processor = TextProcessor(None, 'abcdefg\n0123456\nABCD', self._logger) + processor.parseAndInterpret('g1;d2;g0 2;delete 3 chars;g2 3;d10c') + self.assertEquals('abfg\n23456\nABC', '\n'.join(processor._lines)); + self.assertTrue(processor._errors == 0) + + def testDeleteToMarkBackward(self): + if debug: + return + processor = TextProcessor(None, 'abcdefg\n0123456\nABCD', self._logger) + processor.parseAndInterpret('g2 1;m#0;g0 2; delete#0; ') + self.assertEquals('abBCD', '\n'.join(processor._lines)); + self.checkPos(processor, 0, 2) + self.checkMark(processor, 0, 0, 2) + self.assertTrue(processor._errors == 0) + + def testDeleteToMarkforward(self): + if debug: + return + processor = TextProcessor(None, 'abcdefg\n0123456\nABCD', self._logger) + processor.parseAndInterpret('g 0 2;m#0;g 2 1;delete #0') + self.assertEquals('abBCD', '\n'.join(processor._lines)); + self.checkPos(processor, 0, 2) + self.checkMark(processor, 0, 0, 2) + self.assertTrue(processor._errors == 0) + + def testDeleteLines(self): + if debug: + return + processor = TextProcessor(None, '0\n1\n2\n3\n4\n5\n\6-abc\n7\n8', self._logger) + processor.parseAndInterpret('g6 2;d1l;goto 4;delete 1 line ; g1; d 2 lines;g+0') + self.assertEquals('0,3,5,7,8', ','.join(processor._lines)); + self.checkPos(processor, 1, 0) + self.assertTrue(processor._errors == 0) + + def testGotoRelative(self): + if debug: + return + processor = TextProcessor(None, 'A12345678\nB12345678\nC12345678\nD12345678\nE12345678\nF12345678', self._logger) + # ...........................(3,5) (5,0)(2,0) (2,8) (2,4) + processor.parseAndInterpret('g3 5;goto +2;g -3;g +0 +8 g-0-4') + self.checkPos(processor, 2, 4) + self.assertTrue(processor._errors == 0) + + def testFind(self): + if debug: + return + # ...............................(0,0).................(2,0) + processor = TextProcessor(None, 'A12345678\nB12345678\nC12345678\nD12345678\nE12345678\nF12345678', self._logger) + processor.parseAndInterpret('f/^c/i') + self.checkPos(processor, 2, 0) + processor.parseAndInterpret('find backwards <78<c;') + self.checkPos(processor, 1, 7) + processor.parseAndInterpret('find !123!') + self.checkPos(processor, 2, 1) + processor.parseAndInterpret('fb/a/i;') + self.checkPos(processor, 0, 0) + processor.parseAndInterpret('f/F/;fbackwards /[567]/i;findbackwards =[234]{3}=') + self.checkPos(processor, 4, 2) + self.assertTrue(processor._errors == 0) + + def testFindExpand(self): + if debug: + return + # ...............................(0,0).................(2,0) + processor = TextProcessor(None, 'A12345678\nB12345678\nC12345678\nD12345678\nE12345678\nF12345678', self._logger) + processor.parseAndInterpret('f/^!x43/i -e!') + self.checkPos(processor, 2, 0) + processor.parseAndInterpret('var int eight;$eight=8;find backwards <^x37!eight<c --meta-esc=^ --var-prefix=!') + self.checkPos(processor, 1, 7) + processor.parseAndInterpret('var int one;$one=1;find !${one}23! -p$ --meta-esc==') + self.checkPos(processor, 2, 1) + self.assertTrue(processor._errors == 0) + + def testInsertOneLine(self): + if debug: + return + # ...............................(0,0).................(2,0) + processor = TextProcessor(None, 'A123\nB123', self._logger) + processor.parseAndInterpret('g0 2;i "xy" ;g1 1;insert \'lub\'') + self.assertEquals('A1xy23\nBlub123', '\n'.join(processor._lines)); + + def testInsertThreeLine(self): + if debug: + return + processor = TextProcessor(None, 'A123\nB123', self._logger) + processor.parseAndInterpret('g1 0;i <x\\ny\\nz< -e\\ -p!') + self.assertEquals('A123|x|y|zB123', '|'.join(processor._lines)); + self.assertTrue(processor._errors == 0) + + def testInsertThreeLineLongOpts(self): + if debug: + return + processor = TextProcessor(None, 'A123\nB123', self._logger) + processor.parseAndInterpret('g0 2;i <x\\ny\\nz< --var-prefix=% --meta-esc=\\') + self.assertEquals('A1x|y|z23|B123', '|'.join(processor._lines)); + self.assertTrue(processor._errors == 0) + + def testInsertWrongOpts(self): + if debug: + return + logger = base.MemoryLogger.MemoryLogger() + processor = TextProcessor(None, '', logger) + processor.parseAndInterpret('g0 2;i "1" --var-prefix=A -e.. -x --dummy-var') + self.assertTrue(logger.contains('+++ invalid meta-esc (length=1, non alfanum): ..')); + self.assertTrue(logger.contains('+++ invalid var-prefix (length=1, non alfanum): A')); + self.assertTrue(logger.contains('+++ unknown short option: x allowed: ep')); + self.assertTrue(logger.contains('+++ unknown long option: dummy-var allowed: var-prefix meta-esc')); + + def testExpandMeta(self): + if debug: + return + processor = TextProcessor(None, '', self._logger) + self.assertEquals('\t\n\r. tab:\t!', processor._expandMeta('!t!n!r!x2e tab:!t!x21', '!')); + self.assertEquals('x\ty17', processor._expandMeta('x\\ty\x317', '\\')); + self.assertTrue(processor._errors == 0) + + def testExpandVar(self): + if debug: + return + # ...............................(0,0).................(2,0) + processor = TextProcessor(None, '', self._logger) + intVar = IntegerVariable('number') + intVar._value = 2 + strVar = TextVariable('txt') + strVar._value = 'red' + dictVar = DictVariable('color') + dictVar._dict = { 'red': 'f00', 'green': '0f0', 'blue': '00f'} + listVar = ListVariable('arr') + listVar._list = ['Adam', 'Berta', 'Charly'] + processor._vars['number'] = intVar + processor._vars['txt'] = strVar + processor._vars['color'] = dictVar + processor._vars['arr'] = listVar + self.assertEquals('n: 2 t: red blue: 00f l1: Berta', processor._expandVar('n: %number t: %{txt} blue: %color[blue] l1: %arr[1]', '%')); + self.assertEquals('2red_and_greenf00Charly', processor._expandVar('$number${txt}_and_green$color[$txt]$arr[$number]', '$')); + self.assertTrue(processor._errors == 0) + + def testSubstituteExpandLimit(self): + if debug: + return + processor = TextProcessor(None, 'A123\nB123', self._logger) + processor._vars['x'] = TextVariable('x', '1') + processor.parseAndInterpret('s/%x/./c -p% ; substitute =2=:\\t= c --meta-esc=\\ --limit=1') + self.assertEquals('A.:\t3|B.23', '|'.join(processor._lines)); + self.assertTrue(processor._errors == 0) + + def testSubstituteIgnore(self): + if debug: + return + processor = TextProcessor(None, 'A123\nB123', self._logger) + processor.parseAndInterpret('s/a/Alfa/i;substitute =b=:\t= i --var-prefix=% --meta-esc=^ --limit=99') + self.assertEquals('Alfa123|:\t123', '|'.join(processor._lines)); + self.assertTrue(processor._errors == 0) + + def testAssignmentScalar(self): + if debug: + return + processor = TextProcessor(None, None, self._logger) + processor.parseAndInterpret('var int num;var text name;$num=3;$name="Joe"') + self.assertEquals(3, processor._vars['num']._value); + self.assertEquals('Joe', processor._vars['name']._value); + self.assertTrue(processor._errors == 0) + + def testAssignmentDict(self): + if debug: + return + processor = TextProcessor(None, None, self._logger) + processor.parseAndInterpret('var dict color;$color[red]="f00";$color[blue]=15') + self.assertEquals('f00', processor._vars['color']._dict['red']); + self.assertEquals('15', processor._vars['color']._dict['blue']); + self.assertTrue(processor._errors == 0) + + def testAssignmentList(self): + if debug: + return + processor = TextProcessor(None, None, self._logger) + processor.parseAndInterpret('var list x;$x[]=100;$x[]=200;$x[0]=101') + self.assertEquals('101', processor._vars['x']._list[0]); + self.assertEquals('200', processor._vars['x']._list[1]); + self.assertTrue(processor._errors == 0) + + def testLoadMark(self): + if debug: + return + processor = TextProcessor(None, 'abcdefgh\n012345\n_.:-', self._logger) + processor.parseAndInterpret('var text x;v t y; g4 0;mark #1; g 0 2; load x #1;g 1 3; m #2; g 2 2;l y #2') + self.assertEquals('cdefgh\n012345\n_.:-', processor._vars['x']._value); + self.assertEquals('345\n_.', processor._vars['y']._value); + self.assertTrue(processor._errors == 0) + + def testLoadChars(self): + if debug: + return + processor = TextProcessor(None, 'abcdefgh\n012345\n_.:-', self._logger) + processor.parseAndInterpret('var text x;v t y; g1;load x 3; load y 20chars') + self.assertEquals('012', processor._vars['x']._value); + self.assertEquals('012345', processor._vars['y']._value); + self.assertTrue(processor._errors == 0) + + def testLoadLines(self): + if debug: + return + processor = TextProcessor(None, 'abcdefgh\n012345\n_.:-', self._logger) + processor.parseAndInterpret('var text x;v t y; g0 1; load x 3l; load y 1 line') + self.assertEquals('bcdefgh\n012345\n_.:-', processor._vars['x']._value); + self.assertEquals('bcdefgh\n', processor._vars['y']._value); + self.assertTrue(processor._errors == 0) + + def testPrintMark(self): + if debug: + return + logger = base.MemoryLogger.MemoryLogger() + processor = TextProcessor(None, 'abcdefgh\n012345\n_.:-', logger) + processor.parseAndInterpret('g4 0;mark #1; g 0 2; print #1 -l; g 1 3; m #2; g 2 2;p #2 --to-log') + if self.assertEquals(2, len(logger._lines)): + self.assertEquals('cdefgh\n012345\n_.:-', logger._lines[0]) + self.assertEquals('345\n_.', logger._lines[1]) + self.assertTrue(processor._errors == 0) + + def testPrintChars(self): + if debug: + return + logger = base.MemoryLogger.MemoryLogger() + processor = TextProcessor(None, 'abcdefgh\n012345\n_.:-', logger) + processor.parseAndInterpret('g1;print 3 -l; print 20chars -l') + self.assertEquals('012', logger._lines[0]) + self.assertEquals('012345', logger._lines[1]) + self.assertTrue(processor._errors == 0) + + def testPrintLines(self): + if debug: + return + logger = base.MemoryLogger.MemoryLogger() + processor = TextProcessor(None, 'abcdefgh\n012345\n_.:-', logger) + processor.parseAndInterpret('g0 1; print 3l -l;p 1 line -l') + self.assertEquals('bcdefgh\n012345\n_.:-', logger._lines[0]) + self.assertEquals('bcdefgh', logger._lines[1]) + self.assertTrue(processor._errors == 0) + + def testRead(self): + if debug: + return + processor = TextProcessor(None, None, self._logger) + fn = self.tempFile('unittest.txt') + StringUtils.toFile(fn, 'abc') + processor.parseAndInterpret('v t n;$n="{:s}";read "$n" -e^ -p$'.format(fn)) + self.assertEquals('abc', '|'.join(processor._lines)); + self.assertTrue(processor._errors == 0) + + def testWrite(self): + if debug: + return + processor = TextProcessor(None, 'write successful', self._logger) + fn = self.tempFile('unittest.txt') + processor.parseAndInterpret('v t n;$n=!{:s}!;write !$n! --meta-esc=^ --var-prefix=$'.format(fn)) + content = StringUtils.fromFile(fn) + self.assertEquals('write successful\n', content); + self.assertTrue(processor._errors == 0) + + def testVariables(self): + if debug: + return + processor = TextProcessor(None, "x", self._logger) + processor.parseAndInterpret('var int a b;$a=1;$b=2;insert "$a$b" -p$') + self.assertEquals('12x', '|'.join(processor._lines)); + processor.parseAndInterpret('var text a2 b2 c2;$a2=5;$b2=6;$c2=7;insert "$a2$b2$c2" -p$') + self.assertEquals('56712x', '|'.join(processor._lines)); + + def testIfFind(self): + if debug: + return + processor = TextProcessor(None, "x", self._logger) + processor.parseAndInterpret('if find /x/ then i " OK " ; else i " wrong " ; fi') + self.assertEquals(' OK x', '|'.join(processor._lines)); + processor.parseAndInterpret('if find /y/ then i " wrong " ; else i " OK " ; fi') + self.assertEquals(' OK OK x', '|'.join(processor._lines)); + + def testIfExpr(self): + if debug: + return + processor = TextProcessor(None, "x", self._logger) + processor.parseAndInterpret('if "x" then i " OK " ; else i " wrong " ; fi') + self.assertEquals(' OK x', '|'.join(processor._lines)); + processor.parseAndInterpret('if "" then i " wrong " ; else i " OK " ; fi') + self.assertEquals(' OK OK x', '|'.join(processor._lines)); + + def testIfIf(self): + if debug: + return + processor = TextProcessor(None, "x", self._logger) + processor.parseAndInterpret('if find /x/ then if "T" then i "TT" ; else i "FTwrong" ; fi ; else i " wrong " ; fi') + self.assertEquals('TTx', '|'.join(processor._lines)); + processor.parseAndInterpret('if find /x/ then if "" then i "TTwrong" ; else i "TF" ; fi ; else i " wrong " ; fi') + self.assertEquals('TTTFx', '|'.join(processor._lines)); + processor.parseAndInterpret('if find /y/ then if "" then i "wrong" ; else i "wrong" ; fi ; else if "1" then i "ft" ; else i "wrong" ; fi ; fi') + self.assertEquals('TTftTFx', '|'.join(processor._lines)); + processor.parseAndInterpret('if find /y/ then if "" then i "wrong" ; else i "wrong" ; fi ; else if "" then i "ftwrong" ; else i "FF" ; fi ; fi') + self.assertEquals('TTFFftTFx', '|'.join(processor._lines)); + + def testNumAssign(self): + if debug: + return + processor = TextProcessor(None, ";", self._logger) + processor.parseAndInterpret('var int a b c;$a:=1+2;$b:=$a*2;$c:=$b/$a;i "$a,$b,$c" -p$') + self.assertEquals('3,6,2;', '|'.join(processor._lines)); + processor.parseAndInterpret('$a+=5;$b-=-1;$c*=3;d20;i "$a,$b,$c" -p$') + self.assertEquals('8,7,6', '|'.join(processor._lines)); + processor.parseAndInterpret('$a/=-1+3;$b%=3*1;$c+=-1;d20;i "$a,$b,$c" -p$') + self.assertEquals('4,1,5', '|'.join(processor._lines)); + + def testTextToolPrint(self): + if debug: + return + appl.TextTool.main(['any', 'execute', 'print "hi"']) + + def testCallMethod(self): + if debug: + return + processor = TextProcessor(None, ";", self._logger) + processor.parseAndInterpret('var t line; $line="1,2";var list args; $line.split(args, ","); var int c; $c=$line.length()'); + #processor.parseAndInterpret('var text hi; $hi="wow"; var int len; $len=$hi.length();i "$len" -p$') + self.assertEquals('1|2', '|'.join(processor._vars['args']._list)); + self.assertEquals(3, processor._vars['c']._value); + + def testListSize(self): + if debug: + return + processor = TextProcessor(None, "adam\nberta\ncharly", self._logger) + processor.parseAndInterpret('var list names; load $names; var int size; $size=$names.size()'); + self.assertEquals(3, processor._vars['size']._value); + + def testListJoin(self): + if debug: + return + processor = TextProcessor(None, "adam\nberta\ncharly", self._logger) + processor.parseAndInterpret('var list names; load $names; var text csv; $csv=$names.join(",")'); + self.assertEquals('adam,berta,charly', processor._vars['csv']._value); + processor.parseAndInterpret('var list names2; load $names2; var text lines; $lines=$names.join()'); + self.assertEquals('adam\nberta\ncharly', processor._vars['lines']._value); + + def testTextLength(self): + if debug: + return + processor = TextProcessor(None, None, self._logger) + processor.parseAndInterpret('var text t1 t2; $t1="'"; $t2=/012345/;var int l1 l1; $l1=t1.length(); $l2=$t2.length()"); + self.assertEquals(0, processor._vars['l1']._value); + self.assertEquals(6, processor._vars['l2']._value); + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = TextProcessorTest() + tester.run() diff --git a/unittest/base/ThreadLoggerTest.py b/unittest/base/ThreadLoggerTest.py new file mode 100644 index 0000000..3b12e2c --- /dev/null +++ b/unittest/base/ThreadLoggerTest.py @@ -0,0 +1,30 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +from base.ThreadLogger import ThreadLogger +import os +# import from base.Logger Logger + +class ThreadLoggerTest(UnitTestCase): + + def testBase(self): + logFile = '/tmp/logger.log' + if os.path.isfile(logFile): + os.remove(logFile) + logger = ThreadLogger(logFile, True) + logger.log('Hi world') + logger.info('Hi readers') + logger.error('an expected error') + logger.debug('debug message') + self.assertFileContains('Hi world', logFile) + self.assertFileContains('Hi readers', logFile) + self.assertFileContains('+++ an expected error', logFile) + self.assertFileContains('debug message', logFile) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = ThreadLoggerTest() + tester.run() diff --git a/unittest/base/ZipperTest.py b/unittest/base/ZipperTest.py new file mode 100644 index 0000000..c708ebc --- /dev/null +++ b/unittest/base/ZipperTest.py @@ -0,0 +1,274 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import base.StringUtils +import base.Zipper +import appl.ZipTool +import os +import re +import shutil +import time +# import from base.Logger Logger + +debug=True +class ZipperTest(UnitTestCase): + + def makeTree(self): + text = 'Dies ist ein langer Text, der hoffentlich ein wenig komprimiert werden kann. Wenn nicht, kann man nichts machen' * 1000 + dirName = self.tempFile('srcdir', 'zippertest') + os.makedirs(dirName, 0o777, True) + # /tmp/zippertest/srcdir: file[1-5].txt dir[123] .git linkConfig + # /tmp/zippertest/srcdir/dirX: levelX.txt subdir1 + # /tmp/zippertest/srcdir/dirX/subdir1: a.jpg b.txt c.odt + # /tmp/zippertest/srcdir/.git: config aa bb + # /tmp/zippertest/srcdir/.git/xx: xx1234 xx2345 xx3456 + for ix in range(1, 5+1): + fn = dirName + os.sep + 'file{:d}.txt'.format(ix) + base.StringUtils.toFile(fn, 'no{:d}: {:s}'.format(ix, text)) + for ix in range(1, 3+1): + subdir = dirName + os.sep + 'dir' + str(ix) + os.makedirs(subdir, 0o777, True) + fn = subdir + os.sep + 'level{:d}'.format(ix) + base.StringUtils.toFile(fn, 'level{:d}: {:s}'.format(ix, text)) + dir2 = subdir + os.sep + 'subdir1' + os.makedirs(dir2, 0o777, True) + for node in ['a.jpg', 'b.txt', 'c.odt']: + fn = dir2 + os.sep + node + base.StringUtils.toFile(fn, node + ': ' + text) + + gitdir = dirName + os.sep + '.git' + os.makedirs(gitdir, 0o777, True) + fn = gitdir + os.sep + 'config' + base.StringUtils.toFile(fn, '.git: {:s}'.format(text)) + for node in ['aa', 'bb']: + dir2 = gitdir + os.sep + node + os.makedirs(dir2, 0o777, True) + for node2 in ['123', '234', '567']: + fn = dir2 + os.sep + node + node2 + base.StringUtils.toFile(fn, node + ': ' + text) + if not os.path.exists(dirName + os.sep + 'linkConfig'): + os.symlink('.git/config', dirName + os.sep + 'linkConfig') + return dirName + + def _finish(self): + shutil.rmtree(self.tempFile('zippertest')) + + def checkInfo(self, name, size, compressedSize, info): + self.assertEquals(name, info.filename) + self.assertEquals(size, info.file_size) + self.assertEquals(compressedSize, info.compress_size) + + def testAppendDir(self): + if debug: + return + archive = self.tempFile('test.zip', 'zippertest') + dirName = self.makeTree() + os.makedirs(dirName, 0o777, True) + if os.path.exists(archive): + os.unlink(archive) + zipper = base.Zipper.Zipper(archive, 'w') + start = len(dirName) + zipper.appendDir(dirName, start, re.compile('file3'), True, re.compile('^dir2$'), re.compile('^[.]git$')) + zipper.close() + self.assertTrue(os.path.exists(archive)) + zipper2 = base.Zipper.Zipper(archive, 'r') + infos = zipper2.infosOf(re.compile(r'file[1-5]')) + self.assertEquals(4, len(infos)) + self.checkInfo('file2.txt', 111005, 489, infos[0]); + self.checkInfo('file5.txt', 111005, 489, infos[1]); + self.checkInfo('file4.txt', 111005, 489, infos[2]); + self.checkInfo('file1.txt', 111005, 488, infos[3]); + + infos = zipper2.infosOf(None, 'dir1/subdir1/') + self.assertEquals(3, len(infos)) + self.checkInfo('dir1/subdir1/a.jpg', 111007, 111007, infos[0]); + self.checkInfo('dir1/subdir1/b.txt', 111007, 489, infos[1]); + self.checkInfo('dir1/subdir1/c.odt', 111007, 111007, infos[2]); + + infos = zipper2.infosOf('config') + self.assertEquals(1, len(infos)) + self.checkInfo('.git/config', 111006, 111006, infos[0]); + + infos = zipper2.infosOf(re.compile(r'[ab]\d3'), re.compile(r'[.]\w')) + self.assertEquals(2, len(infos)) + self.checkInfo('.git/aa/aa234', 111004, 111004, infos[0]); + self.checkInfo('.git/bb/bb234', 111004, 111004, infos[1]); + zipper2.close() + + def testCreate(self): + if debug: + return + dirName = self.makeTree() + logFile = self.tempFile('zt.log', 'zippertest') + archive = self.tempFile('createtest.zip', 'zippertest') + if os.path.exists(archive): + os.unlink(archive) + src = dirName + os.sep + # /tmp/zippertest/srcdir/dir1: levelX.txt subdir1 + appl.ZipTool.main(['ziptool', '-l' + logFile, 'create', archive, + '--statistic', + '--ignored-files=[.]jpg', '--ignored-dirs=aa', + '--shortest-path', src + 'dir1', src + 'file1.txt', src + 'linkConfig', + '--store-node-only', src + 'dir3', src + 'file3.txt', + '--already-compressed-dirs=^[.]git$', src + '.git']) + + zipper = base.Zipper.Zipper(archive, 'r'); + infos = zipper.infosOf(re.compile('file[13].txt')) + self.assertEquals(2, len(infos)) + # --shortest-path', src + 'file1.txt' + self.checkInfo('file1.txt', 111005, 488, infos[0]); + # --store-node-only', src + 'file3.txt', + self.checkInfo('srcdir/file3.txt', 111005, 489, infos[1]); + infos = zipper.infosOf('level1') + # --shortest-path', src + 'dir1' + self.assertEquals(1, len(infos)) + self.checkInfo('level1', 111008, 490, infos[0]); + infos = zipper.infosOf('level3') + self.assertEquals(1, len(infos)) + # --store-node-only', src + 'dir3' + self.checkInfo('dir3/level3', 111008, 491, infos[0]); + # --ignored-files: + infos = zipper.infosOf(re.compile('[.]jpg')) + self.assertEquals(0, len(infos)) + # --ignored-dirs=/aa/ + infos = zipper.infosOf(None, re.compile('aa')) + self.assertEquals(0, len(infos)) + # --already-compressed-dirs=^[.]git$ + infos = zipper.infosOf('config') + self.assertEquals(1, len(infos)) + self.checkInfo('.git/config', 111006, 111006, infos[0]); + + def testInfo(self): + if debug: + return + dirName = self.makeTree() + logFile = self.tempFile('info.log', 'zippertest') + archive = self.tempFile('infotest.zip', 'zippertest') + if os.path.exists(archive): + os.unlink(archive) + if os.path.exists(logFile): + os.unlink(logFile) + appl.ZipTool.main(['ziptool', '-l', 'create', archive, + '--shortest-path', '--already-compressed-dirs=^[.]git$', dirName]) + + appl.ZipTool.main(['ziptool', '-l' + logFile, 'info', archive, + 'file2', + '-1', 'config', + '-r', '--reg-expr', '--reg-expression', r'file[4]\W\w{3}', + '-w', '--wildcard', '--wildcards', '*1?[tT][Xx][Tt]', + '-s', '--substring', 'a23', + ]) + + lines = base.StringUtils.fromFile(logFile).split('\n') + self.assertEquals(6, len(lines)) + self.assertMatches(r'111005\s+0% \d+\.\d+\.\d+-\d+:\d+:\d+\s+file2.txt', lines[0]) + self.assertMatches(r'.git/config', lines[1]) + self.assertMatches(r'^\d+\.\d+\.\d+ \d+:\d+:\d+\s+file4.txt$', lines[2]) + self.assertMatches(r'^\d+\.\d+\.\d+ \d+:\d+:\d+\s+file1.txt$', lines[3]) + self.assertMatches(r'^\d+\.\d+\.\d+ \d+:\d+:\d+\s+.git/aa/aa234$', lines[4]) + + def testCreateError(self): + if debug: + return + dirName = self.makeTree() + logFile = self.tempFile('info.log', 'zippertest') + archive = self.tempFile('infotest.zip', 'zippertest') + if os.path.exists(archive): + os.unlink(archive) + if os.path.exists(logFile): + os.unlink(logFile) + rc = appl.ZipTool.main(['ziptool', '-l', '--quiet', 'create', + '--shortest-path', '--already-compressed-dirs=^[.]git$', dirName]) + self.assertEquals('missing <archive>', rc) + + def testArgFile(self): + if debug: + return + dirName = self.makeTree() + logFile = self.tempFile('zt.log', 'zippertest') + archive = self.tempFile('createtest.zip', 'zippertest') + argFile = self.tempFile('argfile.txt', 'zippertest') + if os.path.exists(archive): + os.unlink(archive) + src = dirName + os.sep + base.StringUtils.toFile(argFile, '''# argument file test +create +{:s} +--ignored-files=[.]jpg +--ignored-dirs=aa +--shortest-path +{:s}dir1 +{:s}file1.txt + +--store-node-only +{:s}dir3 +{:s}file3.txt +--already-compressed-dirs=^[.]git$ +{:s}.git'''.format(archive, src, src, src, src, src)) + # /tmp/zippertest/srcdir/dir1: levelX.txt subdir1 + appl.ZipTool.main(['ziptool', '-l' + logFile, '--args={:s}'.format(argFile)]) + + zipper = base.Zipper.Zipper(archive, 'r'); + infos = zipper.infosOf(re.compile('file[13].txt')) + self.assertEquals(2, len(infos)) + # --shortest-path', src + 'file1.txt' + self.checkInfo('file1.txt', 111005, 488, infos[0]); + # --store-node-only', src + 'file3.txt', + self.checkInfo('srcdir/file3.txt', 111005, 489, infos[1]); + infos = zipper.infosOf('level1') + # --shortest-path', src + 'dir1' + self.assertEquals(1, len(infos)) + self.checkInfo('level1', 111008, 490, infos[0]); + infos = zipper.infosOf('level3') + self.assertEquals(1, len(infos)) + # --store-node-only', src + 'dir3' + self.checkInfo('dir3/level3', 111008, 491, infos[0]); + # --ignored-files: + infos = zipper.infosOf(re.compile('[.]jpg')) + self.assertEquals(0, len(infos)) + # --ignored-dirs=/aa/ + infos = zipper.infosOf(None, re.compile('aa')) + self.assertEquals(0, len(infos)) + # --already-compressed-dirs=^[.]git$ + infos = zipper.infosOf('config') + self.assertEquals(1, len(infos)) + self.checkInfo('.git/config', 111006, 111006, infos[0]); + + def testExtract(self): + if False and debug: + return + dirName = self.makeTree() + logFile = self.tempFile('zt.log', 'zippertest') + archive = self.tempFile('extracttest.zip', 'zippertest') + destination = self.tempFile('trgdir', 'zippertest') + if os.path.exists(archive): + os.unlink(archive) + src = dirName + os.sep + # /tmp/zippertest/srcdir/dir1: levelX.txt subdir1 + appl.ZipTool.main(['ziptool', '-l' + logFile, 'create', archive, + '--shortest-path', src]) + shutil.rmtree(destination, True) + os.mkdir(destination) + appl.ZipTool.main(['ziptool', '-l' + logFile, 'extract', archive, + '-d' + destination, '--destination=' + destination, + '-w', + '-u', '--update', '*.txt']) + + oldTime = time.mktime((2000, 1, 1, 2, 33, 44, 0, 0, 0)) + os.utime(dirName + os.sep + 'file2.txt', (oldTime, oldTime)) + + appl.ZipTool.main(['ziptool', '-l' + logFile, 'extract', archive, + '-d' + destination, '--destination=' + destination, + '-w', + '-u', '--update', '*.txt']) + zipper = base.Zipper.Zipper(archive, 'r'); + infos = zipper.infosOf(re.compile('file[13].txt')) + self.assertEquals(2, len(infos)) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = ZipperTest() + tester.run() diff --git a/unittest/base/__init__.py b/unittest/base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/unittest/base/sample.env b/unittest/base/sample.env new file mode 100644 index 0000000..7f9b82f --- /dev/null +++ b/unittest/base/sample.env @@ -0,0 +1,6 @@ +PYTHONPATH=/home/ws/py/pyrshell/unittest/base:/opt/ecl-php-2018.12/eclipse/plugins/org.python.pydev.core_7.0.3.201811082356/pysrc:/home/ws/py/pyrshell:/usr/lib/python3.6:/usr/lib/python3.6/lib-dynload:/usr/local/lib/python3.6/dist-packages:/usr/lib/python3/dist-packages:/usr/share/pyrshell:/usr/lib/python36.zip:/home/ws/py/pyrshell/unittest +SAMPLE__CONFIG=/abc/def +SAMPLE__APPL=sample +SAMPLE__LOGFILE=/var/log/local/test.log +A=B +C=D \ No newline at end of file diff --git a/unittest/configuration/default.conf b/unittest/configuration/default.conf new file mode 100644 index 0000000..b6806ed --- /dev/null +++ b/unittest/configuration/default.conf @@ -0,0 +1,33 @@ +receivergroup "" { + warning: default_warn@hamatoma.de second_warn@hamatoma.de + error: default-error@hamatoma.de defaultroot@hamatoma.de +} + +receivergroup "globalGroup" { + warning: global_warn@hamatoma.de global_monitor@hamatoma.de + error: global_warn@hamatoma.de global_pit@hamatoma.de +} + +smtphost "" { + host: smtp.gmx.de + port: 587 + sender: hm.neutral@gmx.de + user: hm.neutral@gmx.de + code: 1G.e.h.t.H.e.i.m + tls: True +} +host "" { + address: localhost + disk "" { + used: 81% 91% + } + disk "/" { + receivers: globalGroup + used: 70% 80% + } + load1: 4.0 5.0 + load5: 3.1 4.1 + load10: 2.2 3.2 + processes: 500 1000 + swap: 50% 100% +} \ No newline at end of file diff --git a/unittest/configuration/error/default.conf b/unittest/configuration/error/default.conf new file mode 100644 index 0000000..cd3a963 --- /dev/null +++ b/unittest/configuration/error/default.conf @@ -0,0 +1,32 @@ +} +receivergroup "" { + warning: warn@hamatoma.de + error: error@hamatoma.de root@hamatoma.de +} + +receivergroup "globalgroup" { + warning: joe@hamatoma.de monitor@hamatoma.de + error: monitor@hamatoma.de pit@hamatoma.de + disk "wrongPlace" { +} +disk "outside" { +host "" { + receivers: unknownGroup2 + address: localhost + disk "" { + used: 81% 91% + } + disk "/" { + used: 70% 80% + } + load1: 4.0 5.0 + load5: 3.1 4.1 + load10: 2.2 3.2 + processes: 500 1000 unknownGroup1 + dummyProp: 1 2 + receivergroup "" { + host "dummy" { +} +process: 100 50 +host "x" { +) diff --git a/unittest/configuration/localhost.conf b/unittest/configuration/localhost.conf new file mode 100644 index 0000000..913d212 --- /dev/null +++ b/unittest/configuration/localhost.conf @@ -0,0 +1,23 @@ +receivergroup "group1" { + error: critical1@hamatoma.de + warning: maybe1@hamatoma.de +} + +host "localhost" { + address: 127.0.0.1 + disk "/" { + } + disk "/home" { + used: 83% 93% + receivers: group1 + } + disk "/opt" { + used: 84% 94% + } + receivers: group1 + load1: 3.3 5.3 + load5: 2.4 3.4 + load10: 1.5 2.5 globalGroup + processes: 502 1002 + swap: 10% 90% +} \ No newline at end of file diff --git a/unittest/configuration/observed/default.conf b/unittest/configuration/observed/default.conf new file mode 100644 index 0000000..318ebb4 --- /dev/null +++ b/unittest/configuration/observed/default.conf @@ -0,0 +1,23 @@ +receivergroup "" { + warning: warn@hamatoma.de + error: error@hamatoma.de root@hamatoma.de +} + +receivergroup "globalgroup" { + warning: joe@hamatoma.de monitor@hamatoma.de + error: monitor@hamatoma.de pit@hamatoma.de +} + +host "" { + address: localhost + disk "" { + used: 81% 91% + } + disk "/" { + used: 70% 80% + } + load1: 4.0 5.0 + load5: 3.1 4.1 + load10: 2.2 3.2 + processes: 500 1000 +} \ No newline at end of file diff --git a/unittest/configuration/observed/localhost.conf b/unittest/configuration/observed/localhost.conf new file mode 100644 index 0000000..083bffe --- /dev/null +++ b/unittest/configuration/observed/localhost.conf @@ -0,0 +1,29 @@ +receivergroup "localgroup" { + error: critical@hamatoma.de + warning: one@hamatoma.de two@hamatoma.de +} + +site "wiki" { + url: https://wiki.hamatoma.de +} +host "localhost" { + address: 127.0.0.1 + disk "/" { + used: 10% 100% + } + disk "/home" { + used: 10% 11% globalgroup + } + receivers: localgroup + raid "md0" { + raidtype: raid1 + } + raid "md1" { + raidtype: raid0 + } + load1: 0.01 10.0 + load5: 0.01 0.02 + load10: 0.01 10.0 + processes: 50 100 + swap: 0% 1% +} \ No newline at end of file diff --git a/unittest/configuration/sites/otherport b/unittest/configuration/sites/otherport new file mode 100644 index 0000000..ffcfcfc --- /dev/null +++ b/unittest/configuration/sites/otherport @@ -0,0 +1,5 @@ +server { + listen 10116; + server_name monitor.infeos.eu; + server_name_in_redirect off; +} diff --git a/unittest/configuration/sites/wiki.hamatoma.de b/unittest/configuration/sites/wiki.hamatoma.de new file mode 100644 index 0000000..ad5a9d8 --- /dev/null +++ b/unittest/configuration/sites/wiki.hamatoma.de @@ -0,0 +1,20 @@ +server { + #listen 81; + listen 80; + server_name wiki.hamatoma.de; + #server_name wrong + root /home/www/freiwiki; + location /.well-known { + alias /home/www/dromedar/.well-known; + allow all; + } + location / { + return 301 https://$server_name$request_uri; # enforce https + } +} + +server { + listen 443 ssl; + server_name wiki.hamatoma.de; +# server_name wrong +} diff --git a/unittest/data/etc.tgz b/unittest/data/etc.tgz new file mode 100644 index 0000000000000000000000000000000000000000..f4945c0d1643447cf5727623a6fb64f9bd2a0aed GIT binary patch literal 25579 zcmYIuWmFtZur?B0g9LXC9wa~lSzLp=dywF6VUb|L-6gnda9J$46M_ZT;0_DBJ73=W zoqO(|J~K5vRZmY<J=J|`=wdLD6jFLjkWf!ePTY4o)6XGi<(Dg!r`e+7Z+fXJEF7eR zzZxqJoExjEc#X9pVQlW;e4X5I5t!kFOhe$7Ok)R>c5F1O9n@dau;r3%tZH3qOIegd zlNerofE!<8W*2R#NoqYpXRtnCB^TKx(~ovPVP(7;QmeU6xrm|1XCiDkE$_X7yBk0+ zNC?{?ApC(3Kh6r6G1Z*FvKq2Fa%a4l$-@ixGpI3V7ru3*e#o}Y+C=%wh-B8w&?1Ej zIEzM_-Sc6H*O|)zPt;kIfLPn}{O2unT^m2A=W0r4u%f1)td8~{#ONz}#7)j)h`}S7 z3z#(+{YM1f5t`{~GXyq(AQ)tfmDn<LuAjW8nGSa&L3(b>_n65_v>v3t4Asd3^XQSw z9Za0y=8awiSO|83V6~Gwp3ZgHn>gxKn!iTd2s>FnI>aU!%k#w^)RRSSjtPnpryD4C zZqeJiB0te3-Hzf<II=}unQ(8s9Nudy=0EaLOHtvZ=6W?LVu9ir^C^p)W6d#oGyeb) zUDs22Z#Dv^xk5pw`VM9VWM!G}%}8t%)2t`K2CvwG8wtii#m`pAXk`?Agwe%k6n(=T zvHL~c3=I=C;{pZ520U9$1t7NT%G6ea+8<Mah!m(}Kp%;^qN+Js5^>6l(#db^lKFvT zkNguJvbtN^2hDW8=$(u*OEN)poY(B!)W~#;6CPq!29kg<`xVKJ{s<WLMf&;Aj%Dz# zD<(ui6-RQ-w`jQDe@F&^A1!RJ9I9ThYVuJg-hNb3-y>EI7n00gf1nmIlaZGFHX!wK zCgGJ*Dy`0r!~zN$4SCq|OuMP3=N?*E_`4)5?kY}fZ&Xfy{R&MP9c-NOkLsd$9VoeC zi=RR#U)SjGJZ631wER-W%_$guXn+OJsSvctA@xPzz5d|^BjDb9=`;+py)h$9e*%B2 zs7M9;klF;EW1PGg%8hzPsXWrx!cC)K<^$hEBpff~lgxNXbkjL9^HI4nV$PXPD`u*2 zFtWV3NeqmVDCkKoLxWFHF=Q%cSD$p#jE%TGCEBG0vvC+FJVJU}H_~LuID@DhgFZB! zg1Rx{T?WZam|}pahfJV|RWW<~a}-cjG4>C@RuUXY!bfVVaa8nA{;Au@kluU25S8F~ z*Az)#Oth*s733x>NsONqq2JEcG%+Mn&rXBZ!v_Czk*aKhg$Kn8<-ejL$yFc1I-X3L zbzO?~Ogo^#GT1V(CfToKlI)Jkcu#C_1rd9!=UC+jSn^i^<4ay5z{st`{?SF&ln9PP zH7t4>INGpDa`JeQz&65rfogq4*8x`q%!svBG^62%_An5cV`H4mU~_w3E$!MJiudZH z&G5Z{+5q7TG*5MUp8j0B!XFRE51P#SdysR#xd;p}NZb#m_nTRA*9Q5@OxqP!5Dy>% zYaGGi*qsc*l&(kMW$={{|C*Uj%6$fliiQpm@O>8oH9v82lx#EprG@jXuoB59rAl>9 z%V2W>EgIeF)nm8~LB@%e(8t$#qQi)R3D?pG3c-1?ztJ<|N!)_mQ7aKM$ee4Njm8tm zZzGuma<LdMn^YxQfrMv(=oq+R<lT9Me3M4e1`dNo0Z=e~C~I}X3bP}bU;Y$*bTIn$ z-3TdBIp=L2$}3Ft4JvO(spe~prDjG4!VON{Yf$V*{5z;C;7lwXYyguV{<kV!Yg3{* zFh8PR9pEJs3j^#)%;mm}KRqO*!7|<k#C=2%0|aZ}60{f80GUomFOY%LTMbClRU%De zie<oJf)UaO^v!S`w>f~!FFYRD3uA$+8g>U$btSg?Dwy;n#GWF|)~SZ@J((T{>&!m^ zYKi4=P)ID`K*_6#I1HgX0#wjK9U}g-cQNd(gfKn;UuM6%17tK5c7Rl%!#sLZITaBm z5Ih$NK1~L6Hru$vx{=3{NyZ<25?T;oa$Motrw8IkUak?a?=f%~X1fe#ekpi=GuMy} zSS@XhPG<g7iB*+53A|!4P&fqRc*5AozM4{#H#Fs`=L_u6OVq*NhS4e!$fZFIG{aHJ zg`5cQecmCvN~3m`6`n78`!al)>1xo%iF{7_5cRYci{(V#uSC4ea7Vd8^!pODr#^hL z0ybD+9tXrOai0^ILsx->h!z;zI(XgP!xwn}7vOyavj17$4?P?P5@pm`E<+M&0J<Ju zO5giuvmIsxuN{#oAer_TmJoPpj7T{0-HIGAe5Q+hg!7`>hmgHTTDdnfY67b0H4ulP zwwH8@GW@8q?~v#)IH7cjNQH-#ITT2fdyubZpyQhfF~2LS5ugG-gJBPwd0`bGhV`@^ z0dB*|Ru{vGfnCTU4$1+sFK{yh9htOl;PnlE9*}~e+XC)97%3ZosQCgO9dMf`gq~_& zjDhw+`h?{r5hbaH`Q<fvn$e8c2(Bq!u{&~7m_k4R9bOD90rh$D0Bm{9^u)=L1ypLl z3vhH>!JVvhu|Cw~xH}nj$SI|OtZDua5HH&mNT*~M++qH?cQYY-^GMnX2^HNQVBkkm zd=?nfj7+OE=Buzqt4}0P#|fXa8&dJWj>Y*Nos02({um?C3Bz;=+hD?m7x7e1VGv>~ zV1(TLfU3PiXMVU1fvD$Pf72fW|0yydKxf7643+^WqPqs^Q&?Y{>6XKmWJ&;8NzKZC z!JGdRSaiQqpb+EXFDK0g!|A3yFVk^a@lgqXIwzbV?q-4=qV)Qd5E!%Qjh}BjK44xN znnlnC6V}f?TXE3G0Lg;8kkDsvQK7vj!j{MP6|gk^lZmv!xG)%T`M8b$l>0X{8!xWc z8MuQSIlRXc{u=Kr{r0Qw5O!%Y-)nZ=M7FtF7=K6K(^{ebZioIDaH+b57SHt@lPcob z7wm|(d)huLfBe)z+vjl#Q91y27W@9IH|RhLdch?)&zNf#Z1P=n!<MvEY#NUf3Ya@5 z!qrv)Mg3SHDpcVg_iP!!%(%&LPR&fL<DZ3R<3g7e#SrNy14vCHvp;CLGU7~9YK{76 zBY&kHeocc3JMzBmnB_-pC=MfkkKpMBwx{R?puCf(;3h;t#Nf|VK$JcOc$oVNPbjqv zfhIIxIcTK6FaB@|r<th1^z}1)3As~LL>%i=0nOWeJ0D$$nUD-%WTOa|uYdyk=o1}E z9*{$jPXj3L?`R}|R0^7J<>REQgT~Jr<k@lZB*zfwjibQJUrEUuw}h9(n?Mqpq&iv% z+cP-E6Sy#LsDsZXVGRU*xPg#10c@t2N?ZHZ2w}BdNJrnVMx*pr@9TNBC26?8FouQ5 zHHrW<yCmG_?NW=Q2zY!KixCZSp|)pxuH#m9vB~)Rnl9^hB_McO?H(9<4WPxbAqo_r z1vrENT_LFjh`{fQ+CGr%87SZ-;GKKLVdF~m>VQXLv7c)29YQ_{M_%*{$f*HB55ZBR zh>0--<R3L5U4fdY$<g}A^_?%qD1%2w+QPM#d{HX)V(&eNkEjL>5rcgEcX5sY^u;rz z8v)_CyoJvgCyD~Ze=!)bqd)bHJerqLt9Bo8;tv144bBQuP|aSz@};Yk>5X#zu2zL< zO1HTS!Jo=~0u1>9c^#qXN8p>(12BdWwON+`_C9v!aUylwK2&T>e@jH^@olwv^m&T) zJ2TjnB{y&=)>Y7pnCJ%TckT*w!q0(Bf}lS@Tfs46jp-PG7}V?n#-4{imqzy>nUg@8 zY&coUof%8oJ>Vf(2Hw1a{c0;<^4)v1Cvb~iO<lXdJ$a<->n+CU6B51oiGytY4E&=l z#ZdNWq7^{v)!%2je_sQ<`YRec^Gn`wuzj-Fgz+zQfe~|75%%ce=goJ&ufUhrHgx6x z%yIGb01VZh@Wa5wSfKcKP}5<;c#yK1!R|G|0a<iOH+Vtc5Ku`NjZdIr4EpByTu<~o z^SOw=N(8|>2dDEGc$M=*6o+jPsp72GlseHVpS&AY&;oA#uxX7p8eO<`798?dpGW=& zhUiD)(IQDOC`6jv&TwH6XF`K+>uh`ODFGyjY`q3p17rpZKdj;Mip9^6y)bTXpZrTM z{(DCbG~f-0IkU_nyg`$$Bs7dPbo;0C*UmT-!Zi)qz1Jt3e6OH%eS{qzu7mIX5kiyF zDF}u3f$apSOa-9kF?pYFY~LZ@%!fu7oFfF6I2F7Tl;*EuUryi3)tw=Gyz?4L@LJ<< zhmvAa{EFP7J}RW_T}ii+bqpu<>+t=^Z%J1GOd&sPUW$kRMwNknqi0=7xph(OAl%0i zQ)>$3jUDx?o(_4Kjk>C538Q4!Q<)jaN<SHy3mpF3wtCPo3b-x@;62x~k3B<*h9^Q5 zPC<9TP5-N$R;WM+D2Zh9Imt#2j$K4CbNhYqM4t3lI&OQL6h1vcsPau3flLlTm@Gp6 z^22I3%IFSch!~;lyNFSO+W<e*83&9TeyXm*varoLIP_6#-b|9}r0V7DW9P%+qwIQn z(P?1EWX{qw^o}AWx-sg*cv%EUAMttlACqwIK!Q&p>9@dNSV3UJ&oU>($0s#rv}T|M zjQTUdDhWt|2YAE*%9{LuUi$1yeAvySjf4Z>(Xac|1W-P)y&rJ(B|a?OfW7;f_>J&4 zPJ5IUS_m0ezOvFxnsnz^Du3@&iFjXRlufJFWg|6$l~F5|`pLJo0l%@`h#m^!Z3h5L z0FN%k<g-ZpvkyA>rt_sI=K>RTq=M?3r;Sk-d21`CPmK!uON6)<>(~yvR1+YS#~xN) zbMTtnuVvGXmsb~GsrK!oip3}nt;-&y&*`Zn)7S2nZ!-f`j|tPAv0d5*lW*h+@q4u- zg<GB6&5x-S2o!}r8akYpdnT&$B3^V?u06QWEdg*2#v2XN$pJMZC|?VkcC#;A%G1!r zl5l_tL$S$iN&c>zuWdcZfOl7;=GywUON`?B)2A7+%5sM_FG6IRuYA@AhB*82gP8BQ zJtsfz*Isi%&85i~Q3+lmzwqK>KfNwHLHoprpEiaZ+5Uz!xumo7Q-RH-$stM1uM&T% zuQy)19AtDS9<;VnA7OO`c%D2RQu|ALpSkC77oMwO*_a))&lY@p$qDih%0*xN5c>}C z-ytvg=E7{cjzk;rooo4}KO|rM)V^bW?a`I&dKMJC$~e`BT_m;Y6yp?zRA^`sT%e>N zCrm`s@P~X1O}zkpeSde-(~2wAHn?xT=TTT-ikDbcb6NFO*G5pFL$XE^1*=oWo>Iu6 z>_X~~r?HfN_ZAcSE>{($k`~NIs4%tpisN<e>_v!ve-rW$Ra@%VpkSep3}pGFNrSEo zijONj8)lK^)t9-*fTO>TevTwgMkmfBHg07UWpu=>aEwt#QRtBv962kLbee#q$){po zd>AVv{+1x^*1kl#cW+2(bK96)9=QR0BKjawNcAphCMoXI=v*Mjij&-HhcV$mmRrZ) zza;Y4vc(v22WT-mK9IRRA1&<UGIKT!DzQZ|cxmZb%mFtezT!5__EcWT&Cg(|cK}XT z=rWLU^_l(`FoEoG1B%|2*B;l#9bb+#y1)D+CG{SBL3{E@u)aN9m-G{yHUxh8AcRFj zj97q!J*1vHo5B9`b>8<WctL|7Ft@bn$D1GCjR?G!RAn6Zv;139==7JJ<v5_O#1i9H zgQL7oVPMmH@(slso|$_(Qx0oJh|P$69r2Y*`e%b_Q@5DOtG|lX9{va?;I>HM&Ym^8 ztDJ4>w`pl`CV#*S;_S=?%SG?!3jX~WLFkcZ{mMIsZUdUmQScv~tzbI0#srx1sA#`A z>eA>AsGtUr-vAQF5H8qiyCA5!4a|k9I%qPH6p#ih!qkW_iOgky1t=IAJPeqU0MCsL zn_YPKwej;kSrwZB*!Mkc3ahuaEcKWvn_p9&kln2ZWQ-iSD&%wt6sI{$vclOgR_(qz zI(_AGBtLAB-nAql=7NG7W0ssBw&n8jZ~-;Ro}eZKBsLN(dkQEf%lxZ@Xz$5ehD}et zIu*3_>b}RjZ*Wu5%-Yqhek1mIvFh$vu~xYKgrKHpKB``4hH}c`&a%zhF4n&%X3e2q zqt?IT&q|#;_?@nNi{XeUxfy@C*~KJAiI`gP_f`A$hjP=4raIcyXD_WcEH19U@0Fc6 z(^ZukzVw#gKgplz-l$fXc;;GFzxB>MDf+yABHcoAbJ%(~+SNK=d%noay*p7sZ%QZZ zoE`8}{f{`7SE=aH*~)wzSWj$)?9XG}5LsO&y!grnH6O#&v+PMPywQtXFK5oC$JxZx zr&g-RDbOn?-`?y)Y_C{heR$+V<oPzmQN}br&d*`mV!6nXsf{n&a!HYFTY<*!YQj!V zigVJ{*C5MXcBFKV0ym!_t2#bLyH9DHaa-zpyE}`D^n=a!SRTL~%={>5g-T{=V`F>n zOBd<i@R&iyO~(Q=p(>2hUbn>*Wq3?;n2C*g_+s;Mzk*neyy@I*Rns`qq6pw$ol(`Z zYuK{7eD=Nk%bZgSevk`Zr!cp?*Qs2y@2`#b?^@#Fk*jfDmZk=g1Cn3N#L3T0`NQ>* zbkF=BxX6v=FDJ76P?e^X`ysld;QBASborZqM7o!bssDIIKnBa^V=2o+--&eo+0?7& znaONMcARi$yWJ(WSc*T1gvSg{H_UE`Yl=}(pZ-3d>?#cK`t=6gNl_HyK$l=7|FdlE z^jK#G0S~2kb&CU8yGmWq_4Nt(!xf6$UbSiJga17LGoydys*Wnw8iK>PL{?Krrb}w2 zeH&qGyVbq7o}Ss4#O2|;+v@zUj7mcefA@tMRO5l^>>Mi0XnjhXL$!LuI-cBP_hg|Q zqR%aZa$ODY1>=8-zx0yjS%2yyPE8poJg4!8pK$7(hi>P&d9X>@=)L^&Y*cRq0=<7< zb2oiFtPW{$A^21GL4@Pn<esGr>?;sUgQO&0YO)D(*{p_eVr)b|%kr*M5`+ddoP_VH zGIhJ;=}iH)7h@12y{3G#DYK-Dm5YxLmtRkjpKr4Ud9XP`gQs(@zA{VDQyJ*zo<r9l zRoKcBHX5<w3trEjZLDn^X`NmfReyfId`uS3)*O6x``c=jxri+N=l;6qcWUopwN&fs zf#92*N#o|9-7I@N>?~Hs2RyV?jvdJz-5#q8A3s~0Qc9ms_mIj)>`c67{QaqA=+2Kn zt?HFr*6=UgANrPJT)3eOZF{R$36;O)eH_6RS!a7k14{uxk!NFwC|%Mdg_B{5jF&gx zHXKT6Qr_wP)t)Qbdme6C=gN-zyb;4f3BC2on)?!<-bG^q`dvRv+O<;}gHS0@XhZMW z5n8t;b`V<XrEI$Pi!Hc(kZWDfHH1g_Pi60RG34!8h6M;6taZ%o>in)WT+~*g%OYkp zBFCc)=i2XMAk*<5(hG>j7pLAR&sijA+IZkFR5>=oYZm@ypD+{Cl>XkMdQsEScmHE8 zC!^V`chR|X^#MT(ZbSM_^YhNi-Cm{+R4W<<7v=78x$YL#gWo-O^B1`rSz0wSN-e5$ zUcYXnOzL(W>ChkkCN$2%;S<E8XmD;wqM48<v*th`w=nsoa_X=D6ouHQ1)*x6yz?WU zX=W$Vg?*3r((R?5RZHxPVjePiCq-YrC%8*5aCe|>*VRWM8o){23f9M-+^oN~R$|IN z(DF^UTd7z{L-tiL)#hXx6NQqcol>V_F&^%=v+_C)iJQzWjN_bn=EDM?i$HT$&}FAu zUp-3x*zK0vi5r}tKgVg&JF=lOQHTEC`aSU~o#AV8yPah3t2b1MgTTMMJjB=ezUQ`2 zlzW-ETKbrF{nc_>xb|zn$NuW{s5{_rSC{!+BZVVS+Qs#-^tpQ|!2vZUD}U4GN%)#R zjY79vYW(~FuH3TdS;rHxA<Lj~5QUvMYP;V!RocPuGXY~<N1Ql-#2I}hr#fJGG|uSF zrW1a-=o4dB9?FWk{g0TQu?BX*y2v}ugo?xKKT{%lC!`$0O-t4by~gCwYhjeXc`%#v zN*?ULm5zuqQ;yW8ErgVs&5Hlqob;UaTk@mt&VR*BbfJoCMJmzU+Vk3PjHpoT*FSkI z2C}~IRng-QQ`~;DKI+L!CBte`xre*zoTCG5h_$5&Oh5!Qjadd;-0#Hn#>N(u>ICb= zHpER|ak<vMs8`{k3;X<{U4-ZNO|8;FYi60t&EKoXs`VG|*SK6pBUCFX>swZMX4Czg zz6kz3cFI=pDE;W18-YnSQ#tRHow{76)Ml?~nZvPOrWoy1<(b3E()zy-^n~mi!AZ(g zX}3t4AIB1^4T_Rg6Ji#-jCUhC$E=lh11oR2C_EOsZg5U|e^HHP^pSJeR{f<eG>%Xx zXk5`2tzGwpm6%QA8lpBEyw3HhYU$U}5Z+U^C(=4;85tvI-(8bwkrdX_<idJky@27K z%tlKme%@;CCfDo{<Xuj-Xqa4VztOVraU_kmUQ7$p&YNLW^qsuT^y<S*TFb<*5>n2M z&E^EZj6`VQ#EGIMn{GxG+#!q-MfS^-?eUr0b?Zux(1d5B)j~&fqA>MlotfMCx~R_~ z$WTYzfJU{ST0KP!yjVa&+V#cMxtv!Vq2c4Xmd!`GmiIQLHNdQMBD+tM_pzyZ?e*Ph zYzpyW`oW-BaEEr;hK~#91JOGF?XzMTU6&xQmGD^T`^G)vW<id9mX%5Tk|4GM)v}48 zHcfJr6(2ozvEQp&MS5HdrP)Y=4{;o(OD?B>(=?Cwr02JzUp(4zK0(NIAHdBE-=g4v zEaex#LMHu{IX8U+Kr8}-vk?VHs_wZ6bOLgSoXk9jyyPPhx(!x!@uj4af@Bm~>et!t z>A}hpe~Otlb<(u>U&B<(mN%n|>0;KlZxDP{eSj*fp2jxW3g5R;7=8e5v3_e?Xg@OU zRn}n~w{1fxW<h!&x`u{)#9pmfdRfKh97wHEyu54S#JMl5puwK6&^NgR+D>S^wT*^s zYW|6wB$FX(K#F9o(b}^Q6Mcj=g?Uhb+HEhMg(~hXZ7dl~9_HKfA}}8yct&>hLD<@* z%UuCm!jpgo(gC0V1yR^47)E*EAjozY_q=%~0YOCpb;hsIf?~Tq!1($L6tq+amO=1x zQ~*Dtso7rK(HWcqFY&s?fhBneaztBSGZZMg1HRkk=j;Q!_DTDRfbcXxjMwdgfY@BS z67|VLI3vm%h5^a>)~E+SN*%DZ0eH=U3x#0UPt$xRBiByX!gdfTQyS&?%B*#*>&o|7 z-EqdB^1Gl`J<>~B8yjq%`dHo^H@#af<<`qa=%%u?ADC~`Lm+6={Ji4$9pD~U0&L#d zfGuuRUf*|o14(U@1eRX(ga`Gwu6-vwJQx9Cd^_pr9+^&beFrRPvCq&&Iv2hhv#*Cx z&?+2A1~Q(()Bq`-Ibs9`*#qL+CIJr-h6W)-Sl&;tJIV|uV-+yC3j>n+=KCUrz8u_Q z=z;vr$r(cqp+Lj!<{L!JHhFp(8tpz{>jqO}Zg>Sy<%-DX0)DDs-^an?rx|Kq%O`mh z-eEAED@FqxR*=62aOoo_B8r0Vz|6rnAlPcdG+@<&cQ0%Hndm#LJPZcZu)A`}&s+mm zh=dpMV+XlA0YNDGbtI|C<Pnu`Nb*WBe<pvC7U~h1bR+#inJ}{55*|LRhRS;bZ1eXy ziUU@aXn;Tn;?UO-kVRR$hMfAJWsDLpOxij?bL0@v629+i&DMO%A#L4vmQtBIew3te z;Y*<38)@wh)b<NN83$0jQ8Izz;G<j6Zv=>`W)X0^dz8qdeWWmYEU@u)cm?2{2BALz zZ$kV2K}2zd3C9D&SDPXTsuA$+<3)ie;<@;458x1M0q}EShrC8-pa{$GaJc;UC$(7m zoyU3wq(hxPKT=myv|K&pL!C#VZRpd?+!J`_7<AL!x6p?vu?g1y=M}82ry0mw$2|ZQ z8hknm#3BWMLIqp`5mf)&1+-Z7Ai7TaWeyX6L<S7&0%ojR;0OzYBq=DJ%*zq5EXpW^ zI3kF?3H5q+Il&BD7rH4dM8F*yh92y)56+gogS>df8YFcGafKH|nSbBB`UGTr6+-xU zc9f5Yg>`~m9)J>2PsEHC*A!+2f_P#Z4rX@!7{A!c*wLd04G#Z}^9Qr<gP;K(N7f`~ zf#d6u>mT0Z?@8=dTAr|FB(C<#j^UJO-^~7De}%L@48Z;XG3Bo%B@V&3VK)#Nly*dU z9q~$Y;QbCu{W(-+Z=uIJ@#h^Gb?1`IuKWU&MSma2;z{(vd%B<<f3aKz$okU)crr-u zfMStuAmK$K;DbLa7Gu|2jxe+D;zRCBE~r$AC_2<j3@W8~qZ$GdR4O78W|1~o#>z&U z_~>6{74*DhtC!T9!z5k$@o9o>09Q$U`n<Ais{PkPqq7GO+WC0rKBt>+(098VrIaBz zS1(hu%N*O~@`m5E4Lo8~>WX?85!jVkeZENbRY=7sl#~s`M9!gy>5*(5s7-8H-|(xC zr--*ZUpOFC3MG76_d5-Z2wWvWuKA(k!8{$KTi=?d+L<X+mFXv47>0<Se%%$;q6A^O zs@fIp@<h^&yHG_nNMog{z`*Rp=A1zBQ++OdQ}?Jk?l++968bMw?!biYY068}P5YXN zCd8$ss~$Oq0V?NdQ~Xt^NyE4$oipWsf$|0I&p6Occmy^i41?99sQu@*ON}SMw;6eh z14#VW_*;Ng{hhvhs%ll6yglxmR}0h$9Lz+NyelX;35*a4l3ObNUJwDc*pSo~n*cDT zks8ARd!?_?P0=LOKLj6<?0^N55C1%{;f-kPG#!u+{)}So#?hXN^xfeX&BRrl>l+78 zhCm&AFskmzsar5|#z$&43olP)J#F7IcBA;6<N^jZ<wOuFzScVLo<6FbQ<}E6$(?pM zUx^~M{);Qo;kJ@K4U#J@?F5AAH(B6oc^WxBUd|p^`x#`4xdRGj7}b(Q2)`@<WKb`9 z0UQfJ&k;q~1<(}^n%JZ5@0l{{B^W(#q<MP{mU#xf+y}y8yhX-+u7LdRO@YEzc>{3Z zZ8D02J03AqAArA{-(&v5Qkpxn+93a0FUywq8dS0z@;12ZA1~p|fc+x+pFl)?ummW0 z44Cp6MRb@Uc)v*^=me($YP>C&8RTvE5P;zctRmo%eggnW%o*@6w}HSmSi#hKGx#lt z`GM^#85A@Q0~p{lvr=0?$PHKyMIQiG8(M)S^wp3U2+St(kvAUa80^6~^^F}6X4$%c zB=yDo5jVpUp}qxE>Ovv4+3puxDF4C&<F3FQg<Nhc5fSNJm<cc;$$*=yvr>k-#8LCE zu3@@Q%?&7GL&dK%(WCY@on#pLZVMLZNa5NW78Iun;E3IR{NRvHfP$PT{x2Q!*CQAG zZ}rvZ(Spf>H2!Kd4zEwu!0{Fo9|86)il=H;W&|=?f6%p=1k{Wje*uKOECJ4chn%76 zpKn|6(Rlqm^7D%ep@IlPxP|Ga$=@0G8l=YptUQ`t=AsQTsgHok5iivAVkkXU?+oX@ z9Hfl(EB_XJhWQ5W^Bt_$BKX{rGXVdapc#504=lwKE`Z6Ns@-S&@3Q(3x8<TCYbl6& z8iq1}=hg?_N>z!lz2E=eX`J&%GI`znEJK`oppRJ1ba1ox|IBgU%8Mp#cv5!@!zq#u z?;C(k!T&%!nhBKDJsF~1VnZQh@c)r<-$o)Q*Z+B(G6z3t1+0KwDF3qqyIAXIF7RY_ zazSnY!Tk)Emk<H}p*_6y5xNjp1$;gL`*zu{e3AEoSAFjNhme<j=ubvZA-%|_a^OBV zE~pw9c0-B#ONZuI2r2SrHhb7W;3{n4A?T?0{3uT!?UC&eg$Rs6D<DB(>oHS=^@6Lw zGa$X5v&7i|Rr6ZCPkUtE>oac3)}M`3;CP^fDjoI4(f8Gg7xNM;%Y2Kc3d0w2$P3na zZS&G5D{?U}tfgO7oc4*Vwxv1o^(R`h#(FGmk+cp*yY$vLUW6EVl6U9%tcAq()V11$ z>v3O;JJT;lt7427i4wn=T7n#BnuJzVe0yz0vrD?7&3_QI9(Xa+z-Z6$1!FiJj8Z1s z_dSX2Jfm|DD){ZhYyIMR1A2>zTt6tY+UwWJk0)1mf)m-5KB68(zffPNCyp+r{H=iI zQqy5)I2vz7Vci$%fg1bb2FoAS1C>uGA=P-crubb|^gMY-cF{nz;jWSUA5?ZiI&sYI zl=7@zY}+op6wc??(4s%9T7sK)@uPhgG|8)BWrF@OPnM1qOjsXNkdN4v6Rl?#5eCPr z@j@u5(r}|KWc(v6bd+FkLY$A0Cllh2Jfb6QEHpJ}O<2Ff@WM`aw!T+Y*9q!dJHKH@ ziw8rC>=v`C_Cn74sDgpvYz)1Z`Vay6Afiv1h9CI97uvFhk#$E@tFoBrut=wuS&oHB zh=qWr0*lsPV80RM!uXVU)rZ+fHGL7X5pS`RX-v;Imi7Lwit6r)>*Xt`%MK<wc@=9` z$!J#W7tHf7CV!qr{{9826?CXk)1<T*`qf?(Tls$Zow}<(?{!yh!po<A$XhHqsXU>j zHs~%nXDoQQE<Yuwg)EP(>sekr<|fENDwye^ymrxP;T#g3r~m2qLmYN`yi1DtSTM5K zX!d%TUsK<cwhCDTVJb%zE?D1hf+nBw($XD~nq4LR`zSkDz3JGOXGKS45sUZB<t?%A z=U#pD=3E=ZVpvueeFU;i+KY@IDD6~rSLW#mmwVhPnb%zkOTQeA?*4rJtc1FlohK*h zAW7!yzKybZgDYNWVW=VTSvGIzEFaH|CLZHwo&$|Lb{S2L3?KGr;*#ys(940mmf#*P za4uZXsIlLAvVb`D9{Cyxio`3iX?jbG)sEXQf=x-d#{9A2(=shJh6Bpb<gA_vvm3o0 zfE;OCgueBSpUl7ChG<-=39pQ>72{;+cj;^Db{TgPj0*2P(S=9}k-%#jJa`H{BqYbY z1un5*p<IV!mg;FE-z@%v96R~MCZ=>H6+@tx8~~4kwkN-Y*1HAXe#NJm)jfA$L$Z;> z{zWZTiN*4wJ#fAe?^-~AL_w|}=0aRJv4?C2-zO=W?(~(A4r^0;NkOyuG5FxJp1j<g z*KPUq#PFifrou<9RoJr^Gx~!H@@LF@l8@htvb8jmE%-%6=z^=7&1Q`e$a+eYHGlH4 zwQs}@LcikRzeRV!(;vlSW;CQmn&4k#yUBbc_N4x3btz#&5P^h_Y;7p*Ax5Y0SD<7@ zcfY3vlOp6@dZrOk(~CtqO=luQEh2qVnv%9|i30lmp1QWgPNwzB@1GBlTEhLFYrOKJ zhF%cKW=9S^O4ZQv?jJAh^9m?FkrvhQ$=DtBPwAMN^uWI&hp}?PJiFaW>B36!Kiw6d z^>{6t18vAZf1qXpMmIns?l^#S0?f@`;p_v0P=F}}?IBSB^J*5RnfW<Xwy~RxZ;DZ$ zNMYCu*G?Gyk5!hLo)CqE*-p6iJmkZO6<8hxF>xpZb1_38UJLe#e)WxGg(O9d<<n0I z4Qg(^OWqFp658foP-Ekwv~-6-Ug?)rw>PHdarJ~`_CVZID~?<Kh27*;-Kg5++ds|z z@Xm;`1Doz}bi=t19~GV_*I@dp{crqi2BE39S`v~3lf1KI(BEEvS<RRN`q0DHxXkB# zXUj1BX|~#D2zOofAUlnq-GXQDhk#ApuFWpP!+^C0$mgcJ^|xJ{HcRjgA4u2kbjIOm ziK?SHZ#%Q{Yc;bsV#9sc^R><jhqZAI_{amOLoM&}F4urSlzX;Fylc8;sz@yODk~r= z`SlQtBCSQuUHAyfAAy1yL%!^Tj7f4zzyyw6ed!#rKdMnM-Gn-#<Z#o=ZjlObtX^gu zvFE7;_r%xn%qlLR%&Yt~ME^`tW0N0-cfEOYdnK9QD2r(3q8kFoq>g&915|7c%l}S1 zZ^r;zQY2&291^6-9~Yv@VzZOZ6)|xGH1{1Ge-eaa9{j@CcD}UfT8+JtwhAapk{TSH znwB}RwGQdSl#y4{LKO7ao;k0{c|%qqp>ZDPtzC;T{swnv>uENx$JSMHTk_5nK{fcY zQm?G8G?JV4esn<K)3^#R3ni4CwOO2^h~<fF(d&`X1-5X_8?86ruUMM8ECj}wX;krW zKl;0g_r|+EJf37kKvi_ux9MV`?1_)(&yGD%IhK%H{H(Xaj9V|y+2)vdTFU4z)AgHW zt!K_gUmf#>zphckeaQJi-iK!%<YCYH)^r0~GwQde9y@7p&achSEtPHr^Q*N3;BStF zTW?%Hi`~7dW5M}#hN5R+x~0<1UL#knpphF=U$^IQQOYV0b53ovsu?Dp@h6YG4=5AL ze=QiBU1Uh1!myD$Y02`Le_h;yA4Iv(j-Q6}82uwunNo+ftEhIe%ra}UWK+^fF%hcD zGZs7W&MLjpzp1Pg4^gYFAb1Yx@C)+0+ZjPj#Z2g}YL!r1u_ux(Uqs_!dvT9_YU(o? zQ~&I_WhU7rN_*(3rAI%UqWr9OR>_Jb@fV};s`e%n*SiGtPAP^&qn#Fq;y3K4wR29( zEzKY`B@^DBXRhJKeCY%7K#Ay6;jeiK*}-+SQv6opJC5A?pb$VkOF9qV>%HP%rtlff z&wgyt*%-}gZ^W=!-#XcXk6Z3NJ{sMEzR=k?t^-a#mJlGkYha*jO9_5K__y&M(7jU& z%?N;mx0mXe8txZ}kd6YSXR255yY-RuMmXF|I0?Wr8G)PWV~pRJ5hT-c0!v{<b}!ZG z(>Ns=oI8k8Is4fZ(XK)F88@K`+*T(D7t{xE&RN-pgZF!tWWc`z2uQp=f^Xhcup1)1 zC|UOoUV4$Rik@CU+eXHfl1??SA91<iaNV`~pi@bZbGzSrasOqrMMe>F*1l`>@%8y# z-F5*W=h6Xr?ovQ|G7IE=9?58w1*%Wt(AR<4$`gMg0E4a+;S<dK7fi8crv1FrUnu17 z-!dm!d%Wl!%hEM9;!B;{#r)~j?!4u*b)`9i_rPPJ+K>*6F_{kCw4Z~|O!*f_pBH!n zR?WFUq@v%w88t0?rsW#$pmn<)xnB-!lrlP>E-6!COX!UNF1od(wc4B86noDOBC#5c z$BpcgV}V1i1iQcyT1_cX!^O6R685YKx29@y*GOe>fs1^da=N!PAo6?Pc{q_@iDS_m zxc`OeEPZyd{cCb%zWYAQp337xnYyD!iI>g|s&|6qC$&8y2els%2Q_wp%`NvX@9bF- zBa>X3ZSR9((K(@{dQFGa2eA5b{)#R-IfQccA(qP3y!}s;3MWsveXTzgmwv~XSh3y9 zc1)gDQGizHyM_7NP)jboeVG7hJjJfBJNJB-D><JSpSTWud45*z^5QZ9Z&Q|}sW)vW zrPW$_Q#QW8LG20uMdTK7Vn6bsW$=xA?s9Kw)~qDeS-=#EU2?Hx%c!fjoAuBE@$Z9! zvy%x{Ib}+|ww*3>w^`KCv@z_?!6v_7qW7BJoT_hP1eFA1TFYse_-&2!v-p{<qVw$l z!-g%%u6UHan2Ai%>#aaMTUgjv+!jQ(T4a6i5(0MULei`#B*PCMiGd?ig)APv>6O~` z{j9;xS8ds~3T=2xX-Viw==^*5w)=ffoQcPzM<1^5p#)yJBenSB2D$x+*kI$?ho%ib zS=FzT53_XAv`)R&-{wN$+XKM?Z@VkS>F#-dYq~ePzp*}TusbkV)qC8*9u1P@Z*T_{ zeLKMY>Gp!i$KO9)Aj4sqNyV7cVGcVpd0X-9?+inr$SG>zV09Eo3{LTBGt!W9tW)Ft z+0v(~(ZA2N$;wfC(?kW`o!t~l<%E-$%&~0kA#Ypsti-O7Hw6d!std&X^}obEw3<9W zG{Jw@d0)3}s(P>VR?~StpX;4%d~we48JBE#ny9)gEpu<Ha&12Ljr)<0_ItpM*1cTn z)72iszCnp~Y6@cFZr;hIdtWkLJet1V1#<KfI<i{B-T+*rZwHMpKOx#yrg8y{-?kKh z-@o*GfbemA+kg@I3gb|z)7`ZS4}<>%Y<#T<95m|P=9Uf`Pb}cyE60Vt*2?|{gRBhk z53KylG8JH9tpeE2f}H90tQS5&YOmollJmuP5MOPQPB0sL*WZw+XLeaW2H#xP_A(Bg zeoayse;!;Z&c%$<jB6)(hUTt2kVB&@=6eCeQt#LU0%-dps6MK%rXxoATl0WTyB#P{ z`*!1b!{Iw%R&II@(6Bn)0U8`3TJPPj+V5Qtpz?c{Odnp)<5`KtyGubo{R!fmqly)c zkvhok52IT=xEHVf&PXCB)Gg4LOVOI+k5b96WD;`CFZDn#q13#x6O#JF?UM+4emJiv z)7f5B$#_U$a>YwFIxPhH9n<HAoI3oy1kFV&*ogfxmPTbo#<!=8f1`%zv*s@=zl^~1 zii$(6=x_eG@U(B8FDXikOv}`Av2Ovs)tRfAwF|&OcnpB7IS6;bV28TZ8plAS!Z+pU zCgG7j5fNqQCQ0DSW1RD|eE2s;dN<jt3iU;!;@9ynzMuF8G4-4u3F{ukzrAf?y7EsW zklLNM;@Ki|!8KX6wEOY3fz*{{DAzHgs^jwNoO_#+cJdg5x4k%Lao}f3Q^XdF#mfzA zIlbz{nYV=^F@Dx5OWhMiEQGX8!ib{9Uc{P@8&%NvgCNusTmKxjvYfZNKQzPAep>`7 zQ^5qp30xEf@VdLk_dh(oZNhBt=bFn~XKuT#s2b(ch;l;`+jb*YOMFg3mmazMqZDYA zdC=VV=CX*k+dnG&Naq7qrdwpjOvlwv9Z*tEfP(WmmCYFms-{2wVGd>-sIy>S?rrJe z#kLqY!MhVUB><u8yh_up=2;0e?eKtBr#nV}LqlFhbY#n*J%Jtn{f|Pi1_vF%%JIcc zTGu=gX3ZK;U}|RZRX}>RZ$!Ic#O(ZaM!@(gSy>Xe72B$#-1vX)ZIZP@5W;QcHJu64 zg{}EOi7*tZ=8KUUe+cosr0mlK-b<|B!PwmR9K~NK$sAl>Ggr(AI!yw5MQ7saAs*c^ z-X_zq$kb@S@$Fh1$o@?HH-qq(?Hf=AaHV?jVVoY})3x@vIjk3K4V<Vn0bUmGIlDy8 zYW-P&McC%rF+>1ZxHEHUJ?7p5w>+)AW>MsT)*k>H-hlW1zxIcn&xy(`+KGQLDxDJI zqR~(+4W@SkcG;kNe`Dr?3{cTOp1fPeF%beD^ttnu5Y<pf+tG{(bY>Ss<&Sy@HgfqV z8F9GMzBZczdh!rWH&1F$;O4af=w`*?Bp|LEQ=SYsATBgURt*5<X&?f+Ic@+vR6c3^ ze-zHZiXYrX*sEuurWjE^Xjymw&{qG{8Uu3gz-DcLxF)b53cMGEZCbVX{`UsRbg#3V z-+K?V0W3QS&LQLrT>lRmC?@ucfnnDw@Xma3O$0cZ68k@FSU8zI8E2a&XP};i5QbU$ zXEm3-BOsvWl^P4M1oLs)wAE;z(F3JM0c)GUB^>Mz6t}qnZGphe8XGBKRiBCdG*tXN zKaw7T%i|CHtHjXy3c$<I9|sOvTfc@yhLsR=o3}ZJn}&Y}wK1LB;6t>XO-bZ#1@Gp* zeH!`9#at$Dg<Ughx&gaAeSDOmn#}Yomla@Q^ch*2G5ahQ5LfGX8<5-Z+Fz-Nj_OHf zb*^y>sH=MU5E)rBL)Y%Jhih_i$qweQR-xwWme#KmTO?7C9`0S3tv~HD#7oernGdY+ zm)!Y4xpF`g<Dk;x7JGOFQpxGa+FWTo9c2B3!o5XZjvq{cq1^&(xeS9BVFV_-@qYJ# zyeWX%+99}kWpo;lljl7I80MqBrw+8Zf4G=@vLCo&uy9}C70VZ!?ZzJTLwjTHpY1cu z9acG5e=?jXAmPfFI6e#OYktM*_v}hgR3D$vmg|_jXKX+q{%*;-oLJ?p{a>*Zwe?0w zN=Y^_;3pcX4@j;p2t1i}RcBn$VJMDx@)D|U0Wdnq6_&KP{gpB65>3_*huu1(z7{k> zd^0VH%9%_ZgU3znH^lm0WW#}o!k$2nFJXFkS&sNCIG$qVmup_R9HTfcFNfG+JXIoP zp*upN8V#8Orc5y%<R^&{A91$l)5*S9NR`sp&=%Ct-~&t;j*GYp7Nj2Nq&DZ=(mpcp z#5nHR25=%W@0l&DD-LGx*1ot8Z|`lf`-+rl?KuXZ)5U86BU4+AC`#uH_EZ+CV-y7A z1nOyrZ-<e`YlkW>1=*?7+VAjWxTic`kFsEihx9I(%8<XK_QyA}Y<pjk9PV36an-+A zi2t(6wqD)2#7`*6`^XRz`_Q+KA>v;fnoK5w>I)0s{&SvhvM$l{rPfR;=o8T%q{8jI zC6@cztlnpp0$#sj=dA<<N2NCEzO@9?pAjKa#`euzBJ^BFmW1_qO~qqqdJ5*<kgS5J z`_TIr2^H#GhIu1}XLxx<LIN9C(Z{x~7pE>4ryl?9fh;uT0)HS!&l{uItu*Hx@*`vC z+p#X&xce}hND6j&O3df~)A-v{F`%z+qYu(f-R&2hri?Z3%*))~+1))xL|^chy)IG@ zpf!y%Gc(yR#^(VPNg5+TcBe_U{x^R<8586a?9%#<*9Wah>?50M7U^^ABbSjZN@q|x z-(NN(rY<89Mtn*Cz!xKslqkdkG(Kxn`Ej@RQm?~iAw|coV|S?^hG$a~N5Y&0clY<F zT>n7Y+lkt@ea$t=(@EL0RUYek>z$h(+cWodK8}OV4Q&uuoI@~TU6MWCI4K5^uzQdz z?@y0Ig3Y>%)}a%utG`6QBF6^u<&8C}hKyjOvs)kay<N_fh`zP15RmcKMWAj0D!>7r zo8HI*@`sjLj)jf+gD1KI^2QPV))nx&w>QwXa-RAO!BE^EH>vxxKMvoo`x6R-+?<Xx zuGasRw~v9jwB6Rg%Y|2`G!b-+G$U-IqUbba7z-u>f?^gXVcV{TSU&&+T_>3GW$(gi zRD1xFAW2`unG`y)i%uhoRrK?R_KvSq@?-fMxWmO1{kL@vtD-ngG28Du+xtXf5;yw$ zbNO~O$x^4Fs5t@sjJRjM6Ig2Q`qPeLN6hBh7;ACWie0)|o%yu9+G?v3Ezw(GriO+g z7yEMJ7`#(Or|s1;+czeZh{Tt7;Z$;y4OJr}SKNQB0<%4_y%qPhpR)?Y+zgj~d=IL6 zz*wcAHEqTe!#`@lQolHPktva;PZIV-&wUp}gXBvz3GPVY6a7*a8LADsV5J{v<uWG4 zTy`12o<9<@G(XzfSPvVq;0|qJUe4)4>Ll;F4Ebr`iCLQxHaydJ;)7M0laWDlh{lNs z&#zSd_R4=jVK)-Ei%i{CA4t`wo|Xu8z{$Vz%X??B4tmuU7qCSgP#Ba=Vo%vi)8FS8 zF!r50w3UDMhmuRA+glo`(>=!CEjtsblY_$t#_4+bZf6Zhz8z1^VTk<2qNT`1rN6V> zFTEKNp1Z}rd|}l`wZd)(Oq41;nOi6Q!Z2tdj}1veI$ESqgUt!;0bf{{C(4*{o-1J- zM9c3|h9Od8vtVL(W~cCN$f~<6dE}h0SlU<_wM-8IHJN0f7@lgh()5+{u!T6|`!w@F zV}hp_$BKnM)Sjw^y}3*nc$l6%AqUi+l7%5J+(qmZ3URRZ+-IXC_^m7}-89&i6+)rZ zzY66h3NfaC;N+R}|7#cRd`y9k`FTt7?`mg2$0VIl#`0!HrqNXLJ^V<z5A(qNeyoSS zGm^Qs?q+lC<$hA^;!;TgPa$C<C)P=~`C-%1N}qhmmuw~YvTy?TOpFODG8hD`q(0#6 zUlT1ynP2N?ZF8hdKQdf37UcbO*>q)Bf#CVg9!g{f_teC`Ka?N}hS%(Mb{wc@f9<J3 z>fVybj_C1jRg%VMsk?73VZvu_z0angC(|F)Dw;faPB^4iT4?mi4sl0*&s=x!Pq78K z)2ob`f^v8<=1WfVY=eP4iR|<qN%Y5LRC~*N9{xT(Y6Y$444X9;+vBLrRVtgpdeKDR z)ftpc_p<n?N;ua@y-1>f(1hFc!po~4^|qqZAwvs6>2Cs4<DyD?PW7Ql0-uKbvZo$4 zsYm^?&m{di<D**OTm&2L^tMjkXRsvH=oa{CVSD^%x$ZMt(V>yQQNC+4_*o~1VVRlB zYcuIN644O&X{Hj}1(_F$$}j?zCJ3-`Yy=FSh7ok=Na^>R49TnAAr|I-6AkmTYj|_x z-$~zo-kx=DvR_XuT>If@2iG+aIuyIWi#h1#71J{kD4~C4z+du_)aXq~t8<IFrLr^G z$-bB~#IfVnkCRka8Nw_1>(xX@epsp@CtSnyLujcKwrhOayvw0=VxziU<LctOzc(l> z6V`$xqWOj_gPoUREea8M+inhVvF`?~`iUsM;i}XoS{QK&wn)e^`a68V{rNr3ke2BU zS^qKlw=j_wkqGgBvz`8l^9(5mJ^o$W;9?By{u&@R#Z1sTv1IqIGdIBfS(#`^%5>Fe zt`RF^TJn2HVNcdUF?Q&0lHp6HMq(`-OLlkF2HFTqp2~ebqBV%UQ0$PYGIxZ99+$Nf zcdU#YR{DU#>C`VJ@;heFekt;3D9-PGg}vHwe)929`W9?-FRWz47h#iMm1{%dUl4MI zNGgYx==MqE?-ilZr<AM$g|CrcOqoq9&pu%^{ZJM{NUEWftRf`RLo3TbA{j`P=^%22 zVC_LV<9}lp-KYt#-94&-H@0C9Zla(+7Ngw_f7=Tc)Aa4oSG+#1Rs96k2<+8oiGTY3 z?3-%Xow<X{D4z>C$<iH8_yuHDOwR2kl`m`*WbjOQ<bO@?zqt%|IEYjXMO-Z7SMNM8 zi5E09sQ?$P!Z6@gwe1#oukemFVkCcbFCcbA>a%P#;TA|AV|&d&$v(n@;rl1n2Xw+G z2W8P!j<I#`SR>u8mh$i;B(k|HnPW!JP{_@000!=6!P=kq9rx4i!aEH=xK!Gs<CcyP zIxDZfI__XtRLCub-sNHa0>!_a!tNSP@-GM+j-jNkWkiN=v{Y>m)b=h|{Fa7uyxMiT z4{fiFNRu?94D!m-D=yW}()oEyz3GwFH5Vy1#VFv+d6<6jieLF=-6*LeNt~M+7ik1! zFHw`j=N4##v(*s=soKgzhuR=%%YpTlBYfO(bV=E2_I|z4D&yaVJ55(?V{JV?n3fnc zDLUV064DAtQWt8?#BnIA&Gk>*Wu314)qI2E@oBU9V~S)G&vpVuNGR9jiSfTxypJ%* zT0W1hHT0qY-!CYI(DI1SQjv^ZPpWusj#H^2X%q%nl)QQwRPp9f!13o~!u2@PDC@g; zJ=jzgepHogBLAj6kLuzhU)l1wb(ULGHs_vhZ2X^l){D%n)~|jVJe?QZ23rBGeDz;I z7~<(<WyTbNMZ=K0?lsZsKD+2oi|=ixF<>T)`mKeK!<E4dm0O+C=X_?(zJbK(;-c!I zx8e%|B!T=XuX&4O1PU!ni?D_vHq~VxKVDDgJ(I>-C@J^Ja|DjGTQSHve3GA*dZVLD zT7afxupKs4ub&d!mmk2PafZP<865qMnGeOL0KICZS4yum>!r@lvW8tE4qe0V7suG) zAAcf=rEaJf&4}GN+e-jyudnUmdbPd5<lJRi0nPp2Ysn|7m8`2+ct0eZYm-!a_vpRB zqvhpf#>qYvkQYF;k>mvRHVJ4Eis=;=BT9`)bt^xyzWj>N?)gS<P5kc!)fR*#)9CK| zX^8P?E%-Y@cBX{uXUCs{pX(}gKV<zRVX8-3*Rb<|9=+X&d=er0e*sD(wcIDP;m?OZ zN6~(iN}kVL=;Emtfc_R%V8ygemiTv8aZU0A591eY_tu$L69`q`GZhQj_Dn;Vf`%2& z7z3_X=~LkBJp5Ie;<ScOLWf+%pCs~EN&6R;?SbuR**Qan_nU2LgmbqR4xDa_lJ59T z5#75j))E0OqJ!9MsijDqd&`i~p&#e;U5OqTahvaLi+Q!?tzW2K5gh(I6KEV?&ryhZ zib5H?RfG(^(~{A*btb@_#Ny#JKP_zYE&|IxkH)(f(LrT&axf_&T*P02IKD5y2dIqh zw2IIHnn5pAxgGa5gC16lFxPM}!WH~YH%OLFZk+&>MW=6hCEVL*pVuyyu>ZqPJ=uHo z=l1&_o813bJpZZFEZ_egBrQTvNS4cG1@8_!JI3q4c~}81Q5b8#h(P7_L44C~a(s-y z&-9=g;gSx=vlesV47tC=r`H^_dssVgm+2@j0G_*y&Ik#$ZU>B+qZ58F`51%y!Ky+F z@Wb&j#v$Mf=%bSpL^(G?+tjXk_<2ZA@veH;A|)DWjv~_^yPm%Q9Z>jfvAOZUAKWqK z8*|Xq$0+u^MfCsdU0ai?N|v5i_!Y+46EPEGoGtfDcl1MY6%`c43%=|?M8zA3h?j`@ z?~@5yRZ`{Ib*B5&>F72e_O4pFGD+5#nJb}K->Psd(nIgTek|bG@yEtH6{+ATf+>b} z{bWClRskGv{uKO1oO2EoO(zG$6jZe4n7@{keuUc}w=ec@y?kWGmlv_HgFAMBM~9TZ zzF4T&YrV72e*F?BvadL;UPc!Ge!u-sq1Ww762&q4gd28|(I^;U_LoYh-Bw!at&bp2 zl2_c1AI6K|ny~P8@zyn!WoZC!QYw~{f95qs_D}3Je`h&+C#oovZdcOL(X@1#Ig(w6 zQ$fG{fZbd{zY3rkP9gd{e0zQsqGy;NYdA&M74B8~CUu~&dKKsC2e08Ol|0ntR#Uu4 zee){r<(=rew;m5(Hom+Sy*snN|Gk>*uNQi4{A!v(m>?dGN<G@A6*=eBxLPQ8RQ@_d zfktSKPon9$n5ryEnS~$whVq`KXny9I+>6+taF4#5FaCy}`&BQ#MH7`$I^JdXx+RM* z<=HG0-zK^&k_>*k4lN%R%D9@7YE^1`_{bQb9G1DTXmz_NI{3b<W11s~eYw7f+$zvS zbMx`CLwnWt*G=mqm3xZJ|H-!~{M*J0*NC$MEx#bB?7dA4cO#Ue65h@gw;Ruym;NPJ zywZ&w!&Uf(Zp?s|1+VRbVVEuey!}PcS;W1(2mD(l?*Mh5+dR*$wq8KAYe-hGh#`$Q zis>J|7kiUilGtCIcITJMFa4g{{iV<e;RQ=Nh6y@M5@aOOG?Y%O-^)o*p#Dp;xNjJC z%@v2r%K-jDk_74(yKOhH5{=>}JDm%>5%%A(|NU&I3|;ON4f|oI|5WFzfA;d`{FmGE zA0PeuiTsZoQ;z?o$!Gq@FDV7+=mWq7+)Cj(%-u$jWf!eJc=ju6jDGFH$K+p)PAL@g z14S)Lj$Seirs${2e-;{0`^V|W>{f+-`bpysTKTW>tADo5{*}sG{<l9Q`2RnC9{*4B z@76!@<bV8{g71H4EdNLP4`b!`zl&L5|GPK>?$m!Ch5w=U;}HD)9hHFoa(~xCmNY&S zq5qNPJ_^s@|9@ih_rE7-{sZO55%VA2=fm*#50oFrzW>>C|5@(#|HE<r$2fqW@_*7q zdH<)L{4ZZq@HeP`W{<v8|BS!5Yw+`3{X_NN<bC;YL=QdEo%Sm?_v`L@?=|k!KjAj+ z^grblJ{<l>ZsAVx(|)1U{ZAajN7|1)!-qHf&^3Id{qzlgc6qb^AH(gR!~cSE!<4@N zr3m7=|9?e!NdIyd0{Dmo2LE>BaH;fyWP&2U5ao*Lp$`eRZ+Kh2V;``yAVL<{!(&R8 z>x_I%$b!8cCH6wff(aCccvnP_hu}cRwGJxl==iHx*&U+mM+_S;Ss_)XP*suYG&)Qk zsbT_~YXUA0fn2Y{cpvVem@C9Y-hnrt2MHANg^+yH>r0)06>Kptp_G^)=IT=3M;s)T zO}JhMaC9PBj7~rm>mU!}$nT~+IPrSNB2rAk`QouAU1-WLU0%ocPVs6P<+aO4npewE zZ{k2Pv9OCs=eN17ptQYES{O!Ta_Od!LUC>YrsyF}ylIekeT|vI0D&R0oI9=@43HS& zHu(fc3I-?)anJmmdIkdwx)O8k^pGUM);|}+zyOOO2A7`@xL|-oU2uik1pU2pTbXze zc)+6y{yK`_nj)d-TEGye%{xjP42Y<W_{!^o0f1DQN2736s3M_8QlWw%{x$z%6aD$) zilF!5)BrMSCl~F|UT__pg2c*o+~_VB@T`U^R8(;_>wkCA;50Ni{<p|-FbRPv-N4Ml zV=;+Dvg`fF>%;!nO}IVmuLF=M)GPOxsRxs21TgoA(Fl_m1ReRCEJc{aqCv2Q(kAq8 zk+GPB5S6aI$Ke{*9B)+!Q;9E8HDQu~#&dVQGhF9VG0jD;9DIvO05xD9GB;ro!c@9q zny<#sfeKq{sB8W~*CEu92@Fw@l$l5{nMCr^tdyV};UQ9G)(A|dk(@3wgkUm*<fSRA zKsk%#>^m|NOy*EU3Gi_pR`95zG{44Bj@A#_CGQDFFj+)$vCJ)k$!HNF_i#@ug2@t+ zOJzn8OqP*cE|ZC1vV!DFnMVYZRU|L%O>jBOXnlxUnLGrCQcNL034$(^BM3kVf*zD3 z2tWw}2g(rypacQj0Tl!RC_$h>If4L`AZS8MD+mHmg1|vnMi77!1Tt!YFaRYCdZ+?{ z0K{@d-Ui_SN;td*{9y{h1C;P^p&a1>N_e~mIbsUJ1(a}s6H7t3fD$e)vJV0Vlz?#o zL`Fr(+_tM*Ogg3_T%@}NK7mq%ibUNrA7LuOM56CGtS}WJBHgX{DU>2SZhO#6kUFL! zG^D%z?E<~2im;IGmi7fo5fGAaPkx1|2n9*J=e@#I1b_tYDX%aUnf<oe+~UwN6&YQU z?}@H36`5L6-bx_=Dl)Wm=W*fwry?s$+C8xqj%ft)<aZ{9?o!MB`<;p5aZ5w4T(~na zJZxxa{}=8|e1K90N$*TNgHjer?@YXcw$wNzzB6%(uE!(sor!&<jW&4U&c^6?qlri? z-ivd6w7&~?c78L>H3`+;+4~HoGLqie`^^;B6jXa>cQ{cr_06H^g;@d9kPi^<41e~} zmx%fEA4;e62R>sIL7qR)|45Mxo9F)}37X)a`5(Wcyx?CX;`&3b@mc(N0!{#w|C=Nk znttN{6{Up#hl$w!-fR3c{|m_#>z`t14(0zPS%xQ{?|;6c{ECXC>V{R3)vjY`h724v zm*S>j$b=%xU=q&3(U8F0=<AnbL!#xnJ8Z6IackAO2%)ObK7AmLitRR<(b!mIz6ETx zg-bR#s*c(*0igr3uY%)IcejJfR3Ds8i5V%5%9;*1wG}{`ig~mK5tJFZ%yIyUUxkl5 zF$~FA0!x~&c8gW0i${2tu~iL#whU|mpj(jSc@zGk0-BsHnwH6>%EUU7Dj-kMvO#6y ziRyG!dqoVDt{D?MFkElDbE1`X)ilIoFltBJm^hKNw;PN8X|eQ<+5SvswZ3*7WpPY& zsI(St+_^wCo8mT<c<ZdvddpvL>*o}g%#+<$#B|Y1&g153)9MOA)DXS)u)58f4kxrC zr<q(-QTOPD#8Js$IJYO0quxGLM-kZJ_LQixU61JT)tb2H5<=sdv=K3><qB4P8bnUC z1humx@U%Hw8KL2(jwU)@|0u^xfwk~RH|6YXoh6Cr4hT8hR|h^IwkcJg#KeK>IpO~3 zEGBdNtSEj;_AGxH1i>LXoF$K2Xe*pDA}h$0=79C5(nd~e(ay6s+qOq7iL=fD+cp$n z11UO#V|@u4(%c%<yQT`30GM!Y;Sjc|YTHc{gtD}0kUrgA`|n`eUtv?h;Sxng+iV&$ zmjy{r#t&&T+#dkwkEL_d2u@U@nlnzX1^TJS8g_Q7wgDR+r<-5|TGKXFlV?(3XVvyp zuh|P*!;MojJ*4AN=FroGb7>(P#|J|_u52!AFkMTe#Qty|*AE5_4iUG|3E#50w0)Qf zc3pO2oZ$uiB(>M0+G(Cmh=sc6`pii0ah-NN9&{bb=FQ74b2byLUA-LJCuyJ$<*-KW zH8<G;3746;r3s_X*0Drq$mq>6Seuf!TV_0=8mhNb_*RQacN<S9M*X$tIA$_l%Aq>a z=WLJ|IB=Ve*xC=y<9K*%OhkBl4F)9RfH&$^9Ja^SG>bKLwcs@)oG^`xyXx*w)@j!? z!|{2L;<KI~a`V6a4bNA%YANqm{GYuWJ<C^@lK;=YM_zavL|x&1p9cR3_y1}CkIsJ# zo9F+ep8StrQ=b0+>HnYp|LOmq{{P>zy!iiHI`u!`8u|MllK9;FpZxhBR+G>7KVMPa z;s5>(&?-3peb*??e-H7eDV8X`|KW&d{`apbZ_a<6t}%d~pwd*`m-C;R+3?(_e00L< z$x_ol&gZ=)n5zIpP|sFogQ^-4mmOj4X$9}w%195*tyiP#uBdipr|VkkdE1L5Z#dNz z`yi@4s=1PUE)2F#eKg(0Tk@joRw7p~ts!0$?U58zH(R^r(beWSR4#6mP=Qh3(}v@_ z>3QhSnuL1MB)daVS~F{`65UPfCvLtaBNm@<fe?3+<6&X$GSeZ(NubsfFCBQ2lxD!? zcs7{Ju|zwywU%^S8DFEI8@ESUNNkoNfs@N4F(tW0J2aJAqjSmpQ`HWQMM#<>(uy6> z+;lE0YVIDJHG?+PdRH6S&FY069F<-`yZH35?eXJAyLpk?+Cfqb(>l`wgFIU-%W)xZ zXvt0;I&L^8Il+ua>l2~7O}ZM!vF!P5f_ojum_!mheVY;hsyv6*_I2Rel;)qD@wm!~ zm$ood_%(kdvq7uZcSi%#U36(d<oJ0D-_7{raYt-7WBM>2+7zg|K|hj}EV}r-+Z{-K zPmKB<VkFmBXOftUBH219%U)}&?48wUfgAJbo?h-HjmXYwmoxbBp6z!}4Y%Fft%ply zIcd}W<?KX>VGS+Wb*ytL2z_CE?seN1d%}IrpQs9#WR&RGf+)mBz1bTwVx8-G=BZ8h z=8a)uH|t@lt~9C>4@vC^Z0j)bvV;-i8Lp5m##N}Uy<Le!a2a=F)@bi!tusA!*x}Hs z6FrU$gU0r3Y`ktwHcV2q*}yrpM@gUFw()jS^;@AktHoALsqxd<SzkH2L52-83WoJf zx}qmXx=RywQ<~e#c_Xd&cIL9p*>IkcH4hI+HfhGdc8SYuq93YiBJ}xHHRXf`Lvoqk zY7@zs@2{G0Ry1v{p@iL&-<h7d(C;feW#OXJou=}vbz$jkGCVJq?0N}y@Z8+2Z;VkY zDBaF(S68i-THmhchmlE)G6_C!31^|#tq&ZWn0JD)SDUG#sJ6mm%*tTn6WOU%3^_^Y zdi&UKFnfLJUewlk*qZTL+wL9K)zBK6O`IM0tBF*nG8Z0ATdJ|^sV0!PbO!2&(O@D5 zS-8pE@rd&%LCJvGJ~q^iwIz8iHI8jk!6)vi7Rx$V&KP;1@E4O4vxCeGSEr%GE39@} z)qBL)ZHOE@)h=$jc2|+s-89mpC3?(;#`|?vJ5alfMMIW)(_Yw-s_m1z9ad|n!D$x^ z#x-Fxmvwn{Z0YT_AFlh&R7lp*=-8cz0zMFZi=p&Y4}!ntnhv#Sr75#kobA#S#?)lt zo>w%jj5|A}ZKvDiA+zh8Y_|2R)9*zIk1MW8Sw~_4=HZ^+475v^tNZ&n9{Zc&#v5{{ zAYE!n$I@xOz4t(4#|{}<XHpR9px+c*xO<SR3u{+9FG6LWWUEL$^)|96X1lR%O+9kI zWqq<%U8Kzb^u^uz;1WP!)BY^j9{ici<AcjKqySt_THPllvoqtntKL-ZZ`rE3Szmf9 zqsCX|-Kq{BcV?aK{5XoEmKkf=ZmCgLpTk3TDC`7h*{oYr(&`^q#7Sd=wid3(-te4e z&UxN$F5o`ls$eCo;h@*_MDxQPf27jTw(zd6$i3Ru7<2ygWF2d3-C~?*mo)pWeSfkW zu?ZOn4T?5Rb)V8Jb;0O{xzlWJovICelHw<xlRC3n$op+--LmE_ao5=#m=SYyN454w z$wrF>GoV(q=Gy{iQ+?kUI02yy?1*dC>vdcc-4USn0@!s<Ku^xJKAG2d(VHiecfRP) z_LKjr<=y%3-5bEa`}p_i^WW?FpXJEs_upSqp69>k`R{rDd!GNE=fCIq?|J_F_bV^w zKg0$L|Nrj)!|5M?^7mhgg^n;k{}Bwu^E~?gk7CJZ{`W5_-|%mhHiRDW1_q~sxy;W= z;|@s?MS`dmzu(ftqfo(UpRUi9Nq{0x5G`C~7A2K=unn#aqFDvKKn`a)g<0hqe!Yi% z;IJDjg0ghsrFl4@kHT}VJZnY9cyzgaWtK!6sD+}y!tMQlU>`Z#;{nCD<vQPQJrvA} zVTQ*0B1Fn4rWn87!!($M+pq|}N~3t6_c4ps>nP7)zYe!>0RAFcq2Iz;rQSAGc;nmu zLg9{i<Ns^#+I!o^k@(-yr{J{((kxv0l}9VUxO3fPH*K2a@@TtP6ozauu{N^gmE<_y z!+rOg-wgF~)MgLYb{BUhb{mW2ki+4;NJ>NP*%op)Ha2?7M#KAeY3!<#m#_B+7wYiU z{xQzg_RnZ+SCL~qj(l$jJW&71{_#7<0eI7<d<$jfV=xqldNHJ|1g)MglN8^`NgO@P zh$w<*Ua4M|&5t~|h(D!FF~<8-s1|-Y;h{cyrteZ@JyOvun=vY&CLWnC6vCH3Qu7p1 z;g$^r`YD<n%?)Iqu9;6i%`_1B2b;idyTnK-HN(L@Q$~tNP|BoX((}`Pkm@*eF<$}o z0o>g>#F<$pE<y$d%lcl3NBlGqX8D*?=2M2Ly=7S+bIPhupw@fM%;$O@3e?HrD{h7d z35Etbqovv$yPFNQNld)SpSEtM$tI+s@8L7n#AksaU}Bpap*_Z$62nth?OxmN*j;-Y zLKbHqy3+V$V99)xq<%W*q1M57dQbkDHT5rh$KD0vr%~XK5vh-5k5I&vT%7ka@g+e^ z6L&a6wPl<5#Nc4JOZlBrKH4oxx|NbW>hUWiqW;(=y5}tG5~mOPL-V|`0nyZXz(}fV zX6~zXUv!4)Cq==c+w}A_8uZcsd!BiHtK%ij0wh0CH+q&@{9+UfE<rWTQ&u0e_mgB! zGr=XwrAz^fgQ4k00OOz@EDj1F1mPXIiOzuh(9?HzdY5kC(_$%8oSCWRdFYdR9TF?y z3J>UyXk&u|QKpc+%`n=;Ul-bq`qG(-wl+4nUV;(3fx(`2L;_(q4cZo@h6{N#1~1CY z36YU?7Pw;(9mo?G<74f@66BO{k%>3Ul$rNIzictAM901x<dl;~l`)BYl$$D(9nPmC z9fRERt3+>;@tDb-fq~uz#*ko2E&!P`63r<hdC;67m?#k~(C#gC(z~`AoE=#_gGK8y zfEA=r&+McPZ}c*&o^7s;UCIQYK$&cs`=w<;<qk_qvPS|QGwHKVs%jOQi~jJ~s1j7e zKt>?=SBzS^h>T&w1hEysSJrF=-QW?_F|^ynzn;Lh#|U)1rQ~V=;sL}iCR?Ehsj*v@ z+H+wObqt;nYm)Gs*H<5u{DLcA2VM>%HdKa3Gcj9otX3{`SaC9Ij$KxkHpd;9ElcDg zt%aOQJ7qGtm>QeC4@VvAG#R@ynR0|2aB=GuOumMdQSUmaomlwVS?npB)G~XJ-pv^Q z9;YyR8T$!)xdFBGFpn9?DjS##E7zW$yF`9>cZd0RLXsUsnB>Ta(GoLPC|hVE0wIW2 z6=0;Po+f#{6VA;brrt(wL;(Q$&*mdC4`9`N;BQ}9d@V$Ls|&@0@Y-tT@|7^Jj#U(U z9CsHJ(^prd8E_+x#%J|3r-;dYrIMK*6VjOR34d5U^#fP9)Hsj$Kz~l$5=7ArHH2W2 z91nZYZ*Vn^8%qW`MG%QOnao_M*<T^%Y_RpllQ5MVP&>#<<_N9~FBbeL3j7xUv)kc3 z+p@$iLiFX=kYr*!u`)bFe@PJ;nsOa7HREU=rtuQ;fffXmmK>=*iBIF`mb_HWg;`k? zU%R3I8|&%M7P!GBd9bE99}t+!08w@}FiRsoG5IUFr)Q&FzchkPHx#qCnT>>|D;3_C z+nx~6+6Nuswlj4>bj6>;a>#bXL{^(}mDDDD7{tT23yNet+$TzoP(5ApVh#yR{^&ip z8k*IXeJ!$m9hZc7yW|p$eDN|6b3V@`X=6_jyhpae7!Pt^5`6bR_mheCIA9W0pyr16 z3#>vWHcX`=r^e>oB4qSK_J%K_!dIxq?P0f+L>{->h=nOxHiV?&U=rpE8MLq@b&fBl zVFIJ+ER{thwc>}fc{-e#wn(Xy9~f&ayhvZx7iG)3M8o|eVM%TT1szZQ6irL)GQ*G< z8HqT=Jo9~w4YIA=nQ_!9fiO#-GjKy)?6$L?FW;WZPe9CW{JRZR1CZBJPfy<*ejw6| z^7q6AGrG6aYHe>J2u^zS9P1m0S2^nP;^4VJC-va&6)m0w8d!wUl-3e)9UeL2>S-7i zSdpQN%iVA@Z~NzOoDY<UkR)kL9VY0l8_anZ;C=t>`0)5ghjix{(b|=Xg^ybiN_I7N z`Tm8^cV}BND?@I+myaLQ2_~&rc4A5h%NMxPt=n?2M96BSVpRuWH6qXA$vo|tz(oqt zh_=d=3wNfc`j?fGsULDvoocRoXL4)x8<m_R8}!Kv3=$k93FvOsLFR~11JP7J<5sK{ zB}s)xCc`wUi6zGLNK4|DeNS;x{gYB&8HVJPRQOg6R0)&A5Nmo`d%X9(Yb?o6spF&6 z!?Z4B#qAlFiZl>HNtviqhzmuc8p7iYNC04XAGd1}JSRyY!~GDRHP9tQ-$ya|6NIH! z@Aara^E2jRg)WGxK}Dr*&3cu(t#6^|+Nu~F<~%%RZQ0khxiUl@{GA;r_4CQ)nc;&1 zH0P_`ZEM9=QRY71Nnmm+#R4Z_<iGD<JkxhwhN0huaG5od>SjVz@~u801Xq9nj5^c$ zEDb8pR+*Z<UsI%{r?%^g1hr1fvgRS_hxs_2$J)zgpdaQ&IGbiAJ9{7VwUTV^ke)~? zngZ&cR0VMuUu%I~9X4RF7)D#zxh1(qbTjwbMaVJ5%9nJ9r)B*FwoORh$mvVK%ftoj zMj{K*85Y!)=&-pT<3bR$2n}@``5xIfk`FS39JYsqf%LiJD|pK4%LP`0J|Q&1eK&&$ zv><HKq&Mh02mPb-ec<|2ZpYAfw)<l?^Qcvq=7Ao<1gowKGJdu?yfPveiD|SR=j+F; zbBO3j7aO~|+x7!47M-O(D05pV$j3IYv^RY6#=!7BG{VMgo^l+R)1)BtKdE-L$qGJ9 zzXf~$2CZs?WF81w;)F?iqC?C@C&iX1reQgwMuSy&so790CYD(jU>xP$Al^;pMQH_B z=ZlRs%D7!rv4BTS?$IUB6H_fixiox?HLrv<R(Pgoga>Pv2Cb#85vb;zdT=2s4OHF_ z<!&A<H1mEle~WD4EO3_^teD&D&3tvsPW>plqcwkGb`yBU9%W|)KgvEo!I#$ZkJtgc z@1-P&aWW;QWFOFM$aBveMx|%~?ttthywS@;57(d2axCuQP9+PsnnW>~tZEsJ)b|>^ zixwA7Anx@9kwH+?WItdH%(E)hDF=E3#66rMIw?y5<1>)FjHP@I5Yfc$eWZbdg)^ zUqisjllA7~iZCUzh$I=!4i8B5$4ot1Gm*ygWz`PSeYV4CbgN}^XSU30P)jM5VTU3g z5rfE+W41zQ4&5_-8_5+km$HAA-8{2c*j|JkbNp%Is#4k00@zD8(NrBjEAWyL>IE4# zc^bI9AH6h>=SgtR!hKvmaiRZ=J`}maZgq9`&G7(n$D4xN$294Xlm-@_<lanYypixR z^9MhDVYjE&9b8qX3Y*YIq5gML_E8p2K>82zDm&gTD0{Q6V+@c?tNU2sm!AaF<Cg z2{nVv2}z?RtI88YEs2eAR@I}yxw<K9tDm9;^dZ7Z5|@)1fl9#wDCXPUkW)GngbfS% z9D`Sj&q}MsY+A$;jW9DN5M^~^ZAJHu&yFwyOnm5`bE8(_{HiI3r(C$9*|WfY+p1<x zk^uWfELE5Y5`J?;Sdo0Fr%cR9#M#5Zaz~CQYo_i2O~G<2lk+>pR=wIkQxknh-T^H? z<kBj37BaSSh-&aNC}+m|_E<Un1&id-FFgWwS{<j=wX)L&Y|3nc)-LHs(t?OPg(+r# zNX!}4{n;GHVm1}&oH&Ma?DB>PsYt3?^iS3Axh4gd0e`34A{3lf8~+^~Ktjwuriks} z?5O0QVX9BC?yZo8HFiDgE8K?---Ng;<TDYo!)tw!w!cG;fT>(Y!`s+`aE(W4b|gm? zzfulZ;Mz$Wm1B*yd%1Qm*Y4%oy<EGOU)jCHt=(ihu!Y3BaK!eYIFI&IlUt*eZMk_2 z+;$jVMZMgJ1#WmfhkN9E&t|jW87ZnGlhwezlz|SNZbQyCoU17MYR3X;!fs@K?<+m_ zkU5lBN808*J3EAfFP*<E@v&U7ycGgHzWTmMtY<{E@{3z!f$h3?Be-@HrRD{zC|<6f z(Nni3=L?7Q>gg2T2jI7?alD+R(a_Vk{usv>qq!fX*cvLmkA^7l+Pp#}A}3nWVsFwM zju#lLzE;4bxv%rHbN{E=X42N`J05vUD@WGj3E%4)<*%&K6H~FKNrPg|O8|taIml53 zo}~kmnpQ{=!;^*+J8?TZyYh;M8JZ~+GTG|sNe-RykU7B=HZm^;p#_mGD9`Mi)$=_G z{4GbaBsw)iV^3Jx@pV#Zw$ms*ba72-aW<F%l$#RV6TSe12iI<xA@bku^KJgS)9xT} zX}@S}N=g()D{G+4TGZz}c6q=g8Hqz}BA&j_4?C?^LxOutS*x|e3PCi!!Cl8=ACBIT zZD0cIP|+|l`8_6I&SAVLwJQ`dr`}+xC6R~>L6RSZXYUZos~EJjO|Z00{%Wldx>1}) zh0zwjq7|B*P-*k698(Je@v#PLzQPR32|DrTLv3SUnB8698#0s2d48#j{qu8{Aw>)# zVo>VqKWW|#m@1fRut#tW+dG-^%L~zTHc$0EStTn}IR15gps_9MWBG;1PgdzoN&JM4 zBn6rMq;h__GJh+KYE?3Ffcv?hthau6n%zd*%P!Hj2J9D^!?qr+f)4tZM;CuP-ya<9 zzuQ0Rb^e5JvGe=ixBfTn<DvUs+wE=h{oh`v+uA|=-`37L{^xf|@~)}6j-&a^f{l0X zoR(vGE*bxjezzTK<l{%jd8qhL$ldGoP-6M#+5dKZJS6|ET@(M0q`!;rgxf9B1#9{L z9;qV#B+4C<<((Eu^Wp~L-D-6miwoN<<k(DVuO<C|C$aqVX}$N&`glnGJDrZmf4AN4 zY~%ZXas<}#pT0|aOe?AQnwAT~17#I}|6$yl{Legzn+g4DhEWjFy1b+y_@73j;WQ`F zR5z&rAk7hW*H~eoV&{d^?(RBH^VW@<fj?@-+Vhi|cK>LK{C_3+{&V7b(c+u+@u2+g z?d0;m-P#rT-(Ac9_ee(mm9T(_{|`=kZ?E;79Q_wa&Yt72Kk#Jt$-!?g4ujYKxP3C- zd-?Nr^VQCgejR_9d3LwmUA#<w>Q4HL%NKXcQ@uCqb~@oz_qzLG`{wwh?cE$jlWr$| zzpoF^1_!?$rp>q4*TKolSIx^GFQd0-yYXuq9q;W`I~ENBoL=m9szrY^@z!ab)@hyA jX`R+-oz`ic)@hyAX`R+-oz`ic{^aT3I{PG}03ZVZbq3US literal 0 HcmV?d00001 diff --git a/unittest/data/etc.work.tgz b/unittest/data/etc.work.tgz new file mode 100644 index 0000000000000000000000000000000000000000..b58ab2f1d42dca5bdd2757aa9c726fa2715bcc76 GIT binary patch literal 785 zcmV+s1Md7EiwFQ}FTGs=1MOK`Z`v>v=9yoCG@hmo$G5o2TPeNJLAz#RgS`b}AR;6z z377WQ&rV>JUPdF4OsM`W<#w!`lYIV;j%^+fKq+Tv0*q35hKy=Y&l{8=rj#MXC?euH zBbb#cH1EJ9=22{il}c$m{dd28_9F!s{LkztS~~gpkOb?5yvaYtB;{XY7zyGvsOdT` zRkZv<+Tw%r|EUu5D6XVKpd1E4Y_3dAwB!<`$G+=>z8#Irj?JfmFM2@H);C}tg&-1L zwpYLpykO+|W%+bvBGD52>z$Sp`9d)a(!<Hgi3!HRlmlVdx(>PHMzSL^Oo(;U1PeO^ zp4$hh<>h<J|NGw_1El_=vF!xQ{Cw=wKSlV3{s~k1F9K`*r$Zpse^svr3+r?mU(c&n zcXj&x@d{4pu*Z!?lMl|Y-)4-EAiguahab!HPV;L}8?}2{>(RaFQ|wsQy><R@<=xgV zf3HBd(;9`{`(b0=8&Jc#xORFq6EeZHNHT_m$xPGSq?XUprk(EEWZrnyATorA_VUqa zdN;oo)d8#b2x!n2zX`iDr-mV3)}w1YK3jH-)x$T=W*8xVh)1|fCbxIcnY4m2M&Zv3 z-n7mye>Y>$9*w-adINN>Izijg!<!^p#IY-81=+8dnxYCZ)zb~@oD#t4D-SwZ) zKZcsrKcz_3e??%+f8>fx@P+$!-{a0fM?)t5b(#M$)~MpY2)xC=!-w|Vix1)>;h(X0 z&HoTYB@IEM>!k{+)z(qj3)}T%U<ZvWf#QFM|LvbXoR3WUN4nI%E>24S1>kM|52v(! z_5Vx$DOB_SVvx-LA}`l9z%2SF49oft5mo;cf(`!z9>!5lqaX|aaxFlE;!frNLa@t! z&5ay4JhYE2`9I|!5vtaI3c-f|S?Dh8_>fiK7Mb*qq13;)RQfLh&-u?I17y;F!ao)z zfYN^vko=FrU_Q&y$3FfcB)j{+P@@nj{)@nR{XZE3+wH%n8-0H_@%EXIYvEf#K|w)5 P;RE9jraRj~04M+e&}Nw5 literal 0 HcmV?d00001 diff --git a/unittest/data/example.html b/unittest/data/example.html new file mode 100644 index 0000000..7c734b2 --- /dev/null +++ b/unittest/data/example.html @@ -0,0 +1 @@ +<html><body><p>Hi</p></body></html> diff --git a/unittest/data/example.sh b/unittest/data/example.sh new file mode 100644 index 0000000..102b1c6 --- /dev/null +++ b/unittest/data/example.sh @@ -0,0 +1,2 @@ +#! /bin/sh +echo "Hi" diff --git a/unittest/data/example.tar b/unittest/data/example.tar new file mode 100644 index 0000000000000000000000000000000000000000..f79a1a3e59871d196aba3aee24e1075fc4384a58 GIT binary patch literal 20480 zcmeI3-EW&Z6u|S$U*W4e?d6JK48~X0a<yt|*GgRzt$EqgM1U0(zQjkH^snDBbZL`j zq3MEmtDQ(~0yZ|BbAI+Y_?XS2Sx^kiaM0g}!bfmEfG<;U`uJNGQIpbvY1y`*+%jz_ zHz^zq2<`8d9$K!;$}I>PgvrCXZNIw8w_WoJ&i(xtF=kO}%-k~6rpKR0MwR47QHI0n z`c4}fpXO6nnuk_;Ie7yWNSM5DZ01PT)Ly|O;w|ow0Gn0)!Vi5*#)bGf|1dZJ|CU9W z?J&!R`a)2K{O?fP!HV|h|91X<Rw4Wc)4=a-i1|HF{_uDW|DE$c6GD0b06y^V2%P_w z7{>nm`$d)nzFXiBphQ7Ecf!J;?%M(A;vaB_`JVywj%hlL|4$tU=l`Bn-!#8J|1kf1 z5jp@Wl%t>z7NvgVro9dUUHqH0_5Ya-_;X9f|J<U;zXJ6S_Tpckn4lZcz6ZcKK!tw# z?t>*kF|YfL0iMY}<E9WW1~mK&f&BNad)fXA__vXN6`E1g2TOyzs{a@BKjVVI^bhgB zNgay*zY+s@0sjv2uS7LUIsyI{eXjuQGyhu<3jqJ%3*h{(!T?^tzlHp((2SZsSQ*7Z zT9<vk0C*<<7Pn#Am+J%&V>-yc5(9Vv{~Y;Oq8cTguyVfalYRLRUHnsF$^0Kq1*6n< zB>yrc2=jjwSioNVH~BwcFu3%mKr?E(U{#j<ZvOAz|1-|0MceqFf*XMRD=>k*_;26; zGo1ewC`M5i$8p>@{_Xs~Bmd80a{Zs1mdR}<*Z&~@k753g5)0VP|LNtZPIK3rUxh(q zD33FO!F1R($%z}s&4{2C<udh%?%uQm1SASS<O+q5G-TY~-Wqv5iz83-+={$=cQU>r z|GhcA)Yfjp^NCTHg;9oX5g7RrvXP;8=(^5*Fjft&(!4OqeDcqN$g4jjK7+ey?*xu9 zEBV>BbCTYu;eSW|pW{gXAM$@70^nB5KgadIXY24+Dz1SSEb^?V$mGlPY<xL6o}P`2 zmfI32T!DDB?WEj55F9nw#?cJwR$-P}e0Ue-PtFecmQzL^yOqB9*ZJz$dK{(oLcdPy zZ55$?-Cw40=KA-=ndipFqi%=X@T0QQ<;dk;2j!DiHjB4BrFJijPvgt;>G<>4v+>n* z6uP(b&4Lf(%hU6ZBO|Chc<dxgqbe&5FH65g*F%4^_VHwLF<MpIEWH?iJUzcS85uWj zVZ_-r{K-dK)gQvS!wzmg_ILb>4Ru}H75|%#kn=z6{jnX(Zr1;ph3kJ^Vm(Ot8vjrB z|5QO)5&c^`pTB)eisZ+UCU;GFclSkZp2tLv03;16QUe0bm44=tVBsct9F$sI5<T`? zPvNjOd=NqWp^dD&cl`*V;eUt!FN1yH-<IzHxh3HJpJ`+Me?Pj|Y;}$QF2{TGVmyBJ z5`d6OzxP>sivzm&H=FnW+;Rj2{wD1IGfhr0{#RmKYy7t^zYM)wzY{hFAIaUpM@HHF zZ-VmGFZiK#jTq&!tb)X=Vp4!B;X-h+@@3HnP@-V$1SNnHKnb7(Py#3clmJQqC4dq@ P37`Z}0w{q$UIO0%s8=f_ literal 0 HcmV?d00001 diff --git a/unittest/data/example.tbz b/unittest/data/example.tbz new file mode 100644 index 0000000000000000000000000000000000000000..b215332f5bf9727701f3214638db0339e585b82f GIT binary patch literal 1025 zcmV+c1pfO%T4*^jL0KkKSzx|f#{dHrf9}Q*01$19|JQ5h-|+wM-|ztd5C8x|&;Vcn z00>Y79_t%y1i%7>k|0k}P31K`GbAHVQ)qgKzybpXOn?T585(57&}7Mk(;#Ft27uEh zh5$_ggr<TNOsA3q(i&*c01X2}KmZ1Zs2pTzlM_LcCKF77kkA?fOqv)0GzJk#8Umh~ z2B)PnB8*Q^ZB3{HLm*_+Mw)t>dYXPf9_jgjHULm9AQX~-VOW@`ysY!ptCh5(tm<zd z1%NMGL(SfzaWjk6?@Y=fo|-@`Kr#S?RzMzW&pMd;I{PAz=}$W<A?zjoW3Pvwu4qY) z%Yif~0M7uB0$%7l5}@sC)m2qj<S4S?UDL~<fdL}80TnEZb|CAKKFb}Ck6I_N7*H9K zUsk`o!U<bBfXK#^Rc_LPN|3>9qU*lGy4Cpm#^x=V<<l^?SZ;T88nze<Y_Ji%8gJo| zM5i^^6ni!^=;(I4$E6!@h7{ySCPOGOWnef0x+efNZ^R%rQve1`c>6Lq$oS$m9L=f# z*Z?~@NCcT7Bq1c401np@O$f8RDu-sIq4Xg;(43S+=#Akw$Ye8&m?mo|GoZdhAd!Or z1;kC$+Hnv|iK!5qcW;OHt!viD_dZ3RV9(WJY9<XZ^2V8Hf!UWE7M8*as}+rzdbMHb zy(I~%Cihm6h!ROsK<IUC#*hLk*u~I25cYg4s*q~)Vli1P_qDA!QzEkAinWZ<iao0| zWxVAf;T4+^R(2*p3vxk46XdeynaWY}gZHPd_80y{;fmk6YPq%kl56#4?ZSlFe7rg) z<mdsgNF=sR9d2ma4L&PWVo~^ce3U#)76N%=wi5M&SXwy$eW9iSp)X+S%O2ce16c^* zz2nun0>ox{Hw8%ZS|#uvE+HTe_;*%*jz^2SUm^zdeLY8|r_iBqXFTp(yW$czduRmf zJap2t>)Wax2|nPu<vw0dzbkdS!mE1?SC?gkw(9yDf>3S)W>@A;k5Mwpu(JKhPa)ea z+_AKs=;VMOya!PR?vM{?2BW>KEgl)JgnyUNDSbO?E!sY`BzVz-eZ`B%2?lHwXQZ{R zRPhSWj!^_gH9axUzyQaDiu))XDe^c*#XK;MJ3JDS1)~8}Tf!+Crw6}<e>R*U@Ii(_ zuz}~AQk=;pC{hKtdAQUSX;)Hbl_8&oLgq~Z3u>ce1IBQ&fQn!&luWH2QRcBLuRFS0 z3kuYz;0Xv24v5mg^B7tUhGc*xg;!&J2`HS3HS{g9NCj~tI-KMv3YoxDz*k{QtY`b) v%>`V}1(*TX8TX%H3K}Ebzm6+*EI@Mbkje+%M?F3ySN|7sML1B9V7^<&>FCJV literal 0 HcmV?d00001 diff --git a/unittest/data/example.tgz b/unittest/data/example.tgz new file mode 100644 index 0000000000000000000000000000000000000000..407e34ea9f5ab9ec9c8eab01c4084003bf2b80f0 GIT binary patch literal 1044 zcmV+v1nc`BiwFQdo|Rny1MQk!Z<|OIK>f_GnAKK&Sz(4523AqMYE&gnq|~+=zx1iX zKn*fqf+4Z<>w5<~PUEI=><zff-t%C?%m9PUnR{Vi(`XtL{W9z|2L&7<)VJvf`r~(v zNomjImgP{vO$*9R3Tr(=8|>4Ds+4X)NH0vDZ?~)dKgyqf5Mvsp#?&oCZF2f~V8|pl ziZblW*`3xmKFuetx;(Vf%gGyPK*HpGea)PxmfErVkNi8>R5mMrZ|%SHYyV+z0RK6s z%(5A`da5;L=zkmR@Bi)m{Zyj-9fA1`2mjtn<3Dp8l>vbG(Ast|{<p&Z{`*Ci1ip*$ zzZ0-9sG1%CcKHvw*Z$j%ZJKtS|5MvW|J&f8`0quS0CWTvrGDh5jSd03<NqrEXBOx$ zxbk0civG93UjFN26aE_q?*qC6OM+rvHG2>6LjRdC9S6pM+JDDE|J&dI|1I>tJFqm! zW%XOce<mCT;h!o0ZHoDSD;(gzjsABB@PE<t48T6|pTjZ~z%u&Z1_$`h(f{ti$|w%f zs%&xyc<KGWupsP<+JDDJ|6Abz{{{Nr9auSE9{>42SMUD_>M%+zTluf11kwK%*#G%I zTrg;TXJ9f*j_v;@|7Vob)%`z(=IDP591#B*#{cec8plU}|Bo{@|1V5#3X7@vKdR;d z(Em1g-v9IAph|Psn_q`PeWCm~BEY4?CP_}*IIc$ottgkNM|AgQ^P=#>yFwu>7`L~# zMqW+h$kRMml6UXUM%Uz@H|Im`Q8zrF7*$yqW#|@xkuRY)aCA6ymHX<zu$AV8N#>Kk z7DR3RA@LdXX4Ml|#{ucZoTWEMfB(<6`Re^2xC0P;<-frBzgBo8Ua-isLXz>9$;D_m zKAl_)*4~z2;YP%hrwY{$f|FX?IGRGS46~Gr{#}&+RN5C$6d8H!N`3LS{nfVgI7+L9 zK1-{o8p4|HFVi@4{dINbx$#ybKPsiJMlN?2lt1ldyZVU}y8kggjfR(#(dVxhqwC2a zbZ_U|6(2^!^UIF|Baj|!oMdSvv%>JQ^jkFR``fKg$K$KPrrCDw)#&5-<<;51xN+5K z&DN*3Y2Syk*B#vMNAp+R=@vWlf7?;<pPQCtbF0q(8OQm*R(L-C%b=7*|F(+fZ{Lz4 zxnI-NujbvYtK2+~i5dY&8c0&XwOViKXC4U_Zj#4Asl_GHV}G?V2I${~$LU=&$Jmws ztDAlB-%`&21$V%I)57=v+hMQ7``DdDULE*n_50-FSpJ*!^MAo@2b!6d@?TK&zZD+& zze@QZQfjC;S$&lCXFWD-mSAD$h*2&}86=*JNfD?YT9Sv{_t>Y7AP9mW2!bF8f*=U; Ozu-G!u#htVPyhgKi#9L- literal 0 HcmV?d00001 diff --git a/unittest/data/example.txt b/unittest/data/example.txt new file mode 100644 index 0000000..c981298 --- /dev/null +++ b/unittest/data/example.txt @@ -0,0 +1,35 @@ +hm@caribou:/home/ws/py/pyrshell/unittest$ ls +appl configuration __init__.py net __pycache__ UnitTestCase.py UnitTestSuite.py +base data __init__.pyc PackageTest.py sites-enabled UnitTestCase.pyc +hm@caribou:/home/ws/py/pyrshell/unittest$ cd data +hm@caribou:/home/ws/py/pyrshell/unittest/data$ X=$(pwd) +hm@caribou:/home/ws/py/pyrshell/unittest/data$ cd /etc +hm@caribou:/etc$ g_rsh +hm@caribou:/home/ws/py/pyrshell$ zip $X/example.zip *.sh + adding: 2biber.sh (deflated 3%) + adding: 2bt.sh (deflated 3%) + adding: 2dr.sh (deflated 3%) + adding: 2heu.sh (deflated 3%) + adding: 2indian.sh (deflated 3%) + adding: 2merkur.sh (deflated 3%) + adding: 2neptun.sh (deflated 3%) + adding: 2next.sh (deflated 3%) + adding: 2silenus.sh (deflated 3%) + adding: 2sky.sh (deflated 3%) + adding: 2tom.sh (deflated 3%) + adding: All.sh (deflated 21%) + adding: Env.sh (deflated 50%) + adding: MkTar.sh (deflated 16%) + adding: unpackShell.sh (stored 0%) + adding: upd-monitor.sh (deflated 20%) +hm@caribou:/home/ws/py/pyrshell$ tar $X/example.tar *.sh +tar: Ungültige Option -- â/â +âtar --helpâ oder âtar --usageâ gibt weitere Informationen. +hm@caribou:/home/ws/py/pyrshell$ tar cf $X/example.tar *.sh +hm@caribou:/home/ws/py/pyrshell$ tar czf $X/example.tgz *.sh +hm@caribou:/home/ws/py/pyrshell$ tar cjf $X/example.tbz *.sh +hm@caribou:/home/ws/py/pyrshell$ cd $X +hm@caribou:/home/ws/py/pyrshell/unittest/data$ cat >example.html +<html><body><p>Hi</p></body></html> +hm@caribou:/home/ws/py/pyrshell/unittest/data$ cat example.sh +cat: example.sh: Datei oder Verzeichnis nicht gefunden diff --git a/unittest/data/example.zip b/unittest/data/example.zip new file mode 100644 index 0000000000000000000000000000000000000000..a21aca61829e7f0c201e9fc4143933e60977ee82 GIT binary patch literal 3680 zcmb`JeN2;A6vp2QMJU>dEy|EN6i_IXqQoEy%%Wv0wgm*0L90+&r2@6Il#iik`PkIS zPy#9pl<$BHL^l_7n{x!*s33@lMldsojD?|@i1-0!_ZHsvg%_GEnC3L?Kfm*wbMHOp z@!x7-NJS9D80mC5z$-rau+$6u+XTV{u|n1gB|@>>SrNHC$OJJ6+fl%G2cb+fMF@ri z1_+}03fvLY?x$|K<LB#i>k%n-!KJ&5W_mk=cB|cyeWADG&S)!{?bKnQHebEez7jFF z7xith6|X8^^0_KVPC)#%0<1R?xbhFXk~f0no*;}g0^-o@1)T6u7<0^?vv;FyZm=pR z7MuE^_a#n8kd%zpu1PJ@7VaJUE+m-0ZLX=ys89Z?sIbKqzLHnR`IUC@j0Y$hDwq!o zj}$A>%4NN8-$!~_`QB5(s?I5;ldsL^s%akUH-p415G0sfA{9x(rD*<Y%D&J{5Oe7K zfRTlzh2<GXaQ-y(1Qgg4ut7#Sm7`ZXrpSm9%cGRqQz(8H6q*iV?lRytoLNSM(QA%l zf%M)K8B(z<PO0riNX^dBJMTq?!>lsafo`;n=eVLWjNze3#fjR-&;$h3zYh@>vmeO^ zUS?c2bHEc7VG2bkB++82Qh{ELn(+A0G7xh=6WXlJ_`5Kbx8xtn8+UL!;cU=64@FcG zs$o@8Dc|*6>0HMWb??p)6?Mwn_{eaSL*II%2kn3+bKL$IG_$4A_D~XtIahBn`&F>6 zea3Se_b{jgm3M)Fqlp<EjY_D#?<9DXh<lsdZ<yp%7wVQ#&nCAz*XNW|MwyL&@HpwF zIj4fXO}SfH9>0A2u>M<izj)UVgy^e;Wlznkhu;!bN8C=z<z%@zsyACWEj?HsQbWnN zO&p4P+I1$Xu<ORWdzM1)yy2VHVZWdIlcX*vTgG*~(p_3|c-W(KVdhgq%7Ij;pL%HP zZ?Kw64lP_kNJ|(nPoO`@h#n}cP0Jlz6Tq=|^}T;Bth~*qLNx4Je9KEW{e8I?IIX^( z_1&Px?+FuivZV27ouOH8W}1O9Cw@ED;6i1MrevfIzM%WC(w(+^M~};28}I7K?&eUA zvm-k$z8Rq~1&s~eeLcg=ds@6I{7s1Fmk%@;m2PySr>N8WnkqOhvV(hM7sF~(Hr&k1 z8C;Y2$@Q-GZgN?rM?Al}jTJ}f&-D4)%+P1sS6|+JUSDA25cEmWj**CxL&g2)%PLxJ zy4K~YI*kWU6ptMJvE`b;<nc-GwzgL*lR~neMR=3#`d`RNY4qmDaapP>4;bezFXkoq zM~yf>s5BYk28dpjXGnZr#^w*XjNMMIc^NSD<kPd<cLZH)R01nMr8=oW1X_~-0!{;N zR8Y8fGWK%z2>$neqOs}D0qxvCe}dY+vdV$_q#==Sca%kH-tJ%3n3`%uEHbMiB~{sx z%9HZ!!U`Q;H<PN&k6t0VS2gk7n@CsP9Tv6nHS3!MG$F|0&(`_b<P~41gWoX7K&4VD z3y+8jj1=oG?CzTxf})xgf@yEYW`#9FD3LOR;fhF7gvgqqRLB{M$Z)xsAxn}gAc5q} zP|8H$LXdQ==~L4E0+<_Yg&<)vnp7sD$LyC%;`VDB)RL7f2;7xCJ?T{EB3tlp-CYLV zxudPwkJmzA2Q!ZdT}-l>b%z(!_ECvPB7KAb=Tn&CfLNb>Jo4$YDH7(f<?W5V>fY?m zp1y3WjFQgY?8cJC8L8@?v7cMA$5uLn_7LVRH<-;}SpdZd0xml;q=LUNc8iE;?AEj{ zVR3l7q{EG?AaAX3VqXB(48{q|!uuZ`Ye+Q)S(9!Eu%H{#qYLk5u&ggYwOVfjV9?v> zf#KZ=7B&iD7_S0&(5vX-;cW(%mog9VFs1|Wpy|-V!}|p+k7xjR8212p&^_qk;TJ!a zcYK03X#)Ta+5kN?{07I;%nX4#>^=uTxX<+f@v9gM%+mlT7c+pt#jFR0->z5~g$UH4 zyB5IUuGIs>uSp$jTy+}2(4`4jaB1pc;a8E4HKaOZ1dQ_JVgf9<nDnslvm49GpO0WB zIiSHwJ|k_m8LOj>t2P-UxQPpxaN^F$oES^^5sGDAhRkUL6)@pI#W9JXGYGN>d<MZy J*lhy;{Rd}UNeKV| literal 0 HcmV?d00001 diff --git a/unittest/data/examples2.tgz b/unittest/data/examples2.tgz new file mode 100644 index 0000000000000000000000000000000000000000..e33740c2d51540c3c2d4cffe5598002716060e7a GIT binary patch literal 1004 zcmZwFe^8Tk90%|RXHY{JN76uWq#*_p7I$`l#v|y7To~F=qH7J9iFc4dXd}U%JB&10 zfp-eR#384E@ndz0coaOFbBPnSN)rm_h@ZiNv1hPf&+~jg?fkpXfA4=@_rCXi!mD4z zR1#8zaTYEc-<y0hU{Cq5aO&$ZkzrJVtCQ|W$0TP~{?L-V<TNwK=MI&uJ2V&EXxV#> z-}Tk?2PJWKnX;povO0G4&;tpfk%7Da%HZbt<Br0?bE_?Bva#T}iMy?}Hov*1dXX@R zubn|9ZfV;9w0s2L!h*d)U;6S;HTh#<(<aSZ=<qum(7G@!9HaOD);rUcO%b=e?E34I zmWW>ze>UzsTp=Sj)$Plz3)_9};(Ztq8Z%D_cNl|)PaV8vwZyjeztnw9pxcP11rhO~ zuc4cL@Vl$1N_#J%*=9;~DwB0Y;&Ql6rHz{)4?0XzwB6EqkFR<L7;M|?GDwg_a3_8b zBb`TQ^h8mG+##GLH8ye81^8S?3~~6Hv(U1DH|Rl?qF=qJ00AnfrA)(C4_QAu2*!uQ zlv!naLP)5Ll=J`KKGiGYpg=46hMbyxpI7;XnOv)}$Q%pq3^_LsjpRIG6y>$!h3PKK z33hX6JStvNjFa9`@B&g+M#1TBh`G`q%J8R%=xdB)i3pOp4G9#UACrC40KqHb=!glW z9MTj}A{+2+l|u+kL{}VTw>omv6X`Bx9k`O9bZwq{gsx%n^+%?OOFpr6QP4w?Uqa2O zJitVI!vW~H+sW`~g8#4`-hPA>Meky6hR5QG9=s9!Y-Fx|NW|9LUh&|~NMd{rS8agO z@1Bm68aK&>i=d{fb9F=yi-%D7mXg?4AH(a^Rpxn*pJ{T7<@?Jl7+t(p3vApx2${}& zBrY>a4MU76)bJ@MM~Ze8pC!Fht%h>M(YBGxtXSqCf{Kx*FT@JMWnw!Aw(SImCy7ZI z`)nDH#H?9rf7a9vSy#J1?wHNPeGAPKF;dLgDRV-Cn>+wFQVLKFYlGARf>0w<VmyUQ zKw9o>D=RqT=GCl(`vqDJzrv|IXmW=h9(l1Q^AR0%vWhYgda(*~<lls-KUTVCDGeH) z8DMdqXwtfs{=BiimJYkl_{WSOVE-qNA!Y?`ccEUE#(mU?<bLCvRYEFiTQv_O!xtdi z4CmoS?T3)cLgFO~FJSR2;F*yyraQ?G)~nLs!J?VSTQ%z1>b-(Thjqe~fIFya2VWDn z%?wjV+4}x*h97kMr<Q*e?y-|2nbQksUvq4D$J5iCe9y=AofHk&ZId(}wuZ^_qH*m1 Pu0n*c<AjR9nlbD@_CZcq literal 0 HcmV?d00001 diff --git a/unittest/data/monitor/localhost.dump.txt b/unittest/data/monitor/localhost.dump.txt new file mode 100644 index 0000000..510c939 --- /dev/null +++ b/unittest/data/monitor/localhost.dump.txt @@ -0,0 +1,68 @@ +receivergroup "": + error: default-error@hamatoma.de defaultroot@hamatoma.de + warning: default_warn@hamatoma.de second_warn@hamatoma.de +receivergroup "globalGroup": + error: global_warn@hamatoma.de global_pit@hamatoma.de + warning: global_warn@hamatoma.de global_monitor@hamatoma.de +receivergroup "group1": + error: critical1@hamatoma.de + warning: maybe1@hamatoma.de +smtphost "": + code: 1G.e.h.t.H.e.i.m + host: smtp.gmx.de + port: 587 + sender: hm.neutral@gmx.de + tls: True + user: hm.neutral@gmx.de +=== +host "": + address: localhost + receivergroup "": + error: default-error@hamatoma.de defaultroot@hamatoma.de + warning: default_warn@hamatoma.de second_warn@hamatoma.de + load1: 4.0 5.0 "" + load10: 2.2 3.2 "" + load5: 3.1 4.1 "" + processes: 500 1000 "" + swap: 50 100 "" + disk "": + receivergroup "": + error: + warning: + used: 81% 91% "" + disk "/": + receivergroup "globalGroup": + error: global_warn@hamatoma.de global_pit@hamatoma.de + warning: global_warn@hamatoma.de global_monitor@hamatoma.de + used: 70% 80% "globalGroup" +=== +host "localhost": + address: 127.0.0.1 + receivergroup "group1": + error: critical1@hamatoma.de + warning: maybe1@hamatoma.de + load1: 3.3 5.3 "group1" + load10: 1.5 2.5 "globalGroup" + load5: 2.4 3.4 "group1" + processes: 502 1002 "group1" + swap: 10 90 "group1" + disk "": + receivergroup "": + error: + warning: + used: 81% 91% "" + disk "/": + receivergroup "globalGroup": + error: global_warn@hamatoma.de global_pit@hamatoma.de + warning: global_warn@hamatoma.de global_monitor@hamatoma.de + used: 70% 80% "globalGroup" + disk "/home": + receivergroup "group1": + error: critical1@hamatoma.de + warning: maybe1@hamatoma.de + used: 83% 93% "" + disk "/opt": + receivergroup "": + error: + warning: + used: 84% 94% "" diff --git a/unittest/img/landscape.jpg b/unittest/img/landscape.jpg new file mode 100644 index 0000000000000000000000000000000000000000..54fd62e6e90ae0360c575de4c4f41a131d2a4c90 GIT binary patch literal 798 zcmc(bO-@2F6ot=iU!lDAJzHpzAsUD5;gOJFo(TzCFo8s;hQz6Ba2YPcIB*I6Ok9bz zHIRr3#EG2TNpHT}d+rHO;SJ_&T#rFf0~!F~9i7qD{CqK=^{+1%7yZ+EG7Ya7fispW zDvKqg5LqVjX0su6uC>aQD$2GgN?R&ry2f^#_g)_y_PVaucHX5=h{u^sw1jB6LKW`M z3iq%QGdyy#V3xS0@BknEV_h<~-H$H93p6J%!wo=p!}Xzmd?kD4yXl)~Pe#)u*l|EY x?C%ICs%SgHz91VHHLBI3jR;joY0)w_@R*Ugtxxt;J2El5Gg%>94*o=*KLEe9e6|1p literal 0 HcmV?d00001 diff --git a/unittest/img/portrait.jpg b/unittest/img/portrait.jpg new file mode 100644 index 0000000000000000000000000000000000000000..88d539b09a7262b550f0a18f7c156e3f64ad1200 GIT binary patch literal 798 zcmc(bO-@2F6ot=iU!lDAJzHpzAsUD5;gOJFo(TzCFo8s;hQz6Ba2YPcIB*I6Ok9bz zHIRr3#EG2TNpHT}d+rHO;SJ_&T#rFf0~!F~9i7qD{CqK=^{+1%7yZ+EG7Ya7fispW zDvKqg5LqVjX0su6uC>aQD$2GgN?R&ry2f^#_g)_y_PVaucHX5=h{u^sw1jB6LKW`M z3iq%QkC<_?V3xS0@BknEV_h<~-H$H93p6J%!wo=p!}Xzmd?kD4yXl)~Pe#)u*l|EY x?C%ICs%SgHz91VHHLBI3jR;joY0)w_@R*Ugtxxt;J2El5Gg%>94*o=*KLEgde6|1p literal 0 HcmV?d00001 diff --git a/unittest/net/FileTcpTaskHandlerTest.py b/unittest/net/FileTcpTaskHandlerTest.py new file mode 100644 index 0000000..1418890 --- /dev/null +++ b/unittest/net/FileTcpTaskHandlerTest.py @@ -0,0 +1,165 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +import os +import datetime +import shutil + +from unittest.UnitTestCase import UnitTestCase +from base.Logger import Logger +import base.StringUtils +import net.FileTcpTaskHandler + +class MockingServerThread: + def __init__(self): + self._answer = None + self._firstBlock = None + def send(self, message): + self._answer = message + def sendBinary(self, data): + self._answer = data.decode() + def readBinary(self): + return None + +class FileTcpTaskHandlerTest(UnitTestCase): + def __init__(self): + self._isRoot = os.geteuid() == 0 + self._logFile = self.tempFile('logger.log') + self._logger = Logger(self._logFile, True) + dir1 = self.tempDir('1', 'handlertest') + dir2 = self.tempDir('2', 'handlertest') + self._dirs = [dir1, dir2] + os.makedirs(dir1, 0o777, True) + os.makedirs(dir2, 0o777, True) + if os.path.isfile(self._logFile): + os.remove(self._logFile) + self._serverThread = MockingServerThread() + UnitTestCase.__init__(self) + + def _finish(self): + shutil.rmtree(self.tempDir('handlertest')) + + def hasAnswer(self, expected): + self.assertEquals(expected, self._serverThread._answer) + + def testCopy(self): + handler = net.FileTcpTaskHandler.FileTcpTaskHandler('test', self._dirs, self._logger) + source = self._dirs[0] + '/file1.txt' + target = self._dirs[1] + '/file2.txt' + content = 'xxxx' + base.StringUtils.toFile(source, content) + handler.fulfill(['file', 'test', 'copy', source, target], self._serverThread) + self.assertTrue(os.path.exists(target)) + self.assertEquals(content, base.StringUtils.fromFile(target)) + self.hasAnswer('OK') + + def testCopyFailure(self): + handler = net.FileTcpTaskHandler.FileTcpTaskHandler('test', self._dirs, self._logger) + source = '/tmp/file1.txt' + target = '/tmp/file2.txt' + handler.copy([source, target], self._serverThread) + self.hasAnswer('+++ "file": copy: invalid path') + source = self._dirs[0] + '/missing.txt' + handler.copy([source], self._serverThread) + self.hasAnswer('+++ "file": copy: missing target') + target = self._dirs[1] + '/missing.txt' + handler.copy([source, target], self._serverThread) + self.hasAnswer('+++ "file": copy failed No such file or directory') + + def testDelete(self): + handler = net.FileTcpTaskHandler.FileTcpTaskHandler('test', self._dirs, self._logger) + source = self._dirs[0] + '/todelete.txt' + content = 'yyy' + base.StringUtils.toFile(source, content) + self.assertTrue(os.path.exists(source)) + handler.fulfill(['file', 'test', 'delete', source], self._serverThread) + self.assertFalse(os.path.exists(source)) + self.hasAnswer('OK') + + def testMkDir(self): + handler = net.FileTcpTaskHandler.FileTcpTaskHandler('test', self._dirs, self._logger) + source = self._dirs[0] + '/subdir/dir1' + if os.path.isdir(source): + os.rmdir(source) + os.rmdir(os.path.dirname(source)) + self.assertFalse(os.path.isdir(source)) + handler.fulfill(['file', 'test', 'mkdir', source], self._serverThread) + self.assertTrue(os.path.isdir(source)) + self.hasAnswer('OK') + + def testMove(self): + handler = net.FileTcpTaskHandler.FileTcpTaskHandler('test', self._dirs, self._logger) + source = self._dirs[0] + '/file1.txt' + target = self._dirs[1] + '/file2.txt' + content = 'yyy' + base.StringUtils.toFile(source, content) + handler.fulfill(['file', 'test', 'move', source, target], self._serverThread) + self.assertFalse(os.path.exists(source)) + self.assertTrue(os.path.exists(target)) + self.assertEquals(content, base.StringUtils.fromFile(target)) + self.hasAnswer('OK') + + def checkFile(self, name, offset, length, info, expectedContent): + # 'OK' SIZE OFFSET LENGTH MTIME MODE OWNER GROUP '\n' + x, content = info.split('\n') + parts = x.split(' ') + info = os.stat(name) + self.assertEquals(expectedContent, content) + self.assertEquals('OK', parts[0]) + self.assertEquals('{:x}'.format(info.st_size), parts[1]) + self.assertEquals('{:x}'.format(offset), parts[2]) + self.assertEquals('{:x}'.format(length), parts[3]) + self.assertEquals('{:x}'.format(int(info.st_mtime_ns / 1000)), parts[4]) + self.assertEquals('{:x}'.format(info.st_mode), parts[5]) + self.assertEquals('{:x}'.format(info.st_uid), parts[6]) + self.assertEquals('{:x}'.format(info.st_gid), parts[7]) + + def testGet(self): + handler = net.FileTcpTaskHandler.FileTcpTaskHandler('test', self._dirs, self._logger, 10) + source = self._dirs[0] + '/fileGet.txt' + content = '1234567890abcdefghijklmnopqrstuvwxyz' + base.StringUtils.toFile(source, content) + handler.fulfill(['file', 'test', 'get', source, '0'], self._serverThread) + self.checkFile(source, 0, 10, self._serverThread._answer, '1234567890') + handler.fulfill(['file', 'test', 'get', source, '10'], self._serverThread) + self.checkFile(source, 10, 10, self._serverThread._answer, 'abcdefghij') + handler.fulfill(['file', 'test', 'get', source, '20'], self._serverThread) + self.checkFile(source, 20, 10, self._serverThread._answer, 'klmnopqrst') + handler.fulfill(['file', 'test', 'get', source, '30'], self._serverThread) + self.checkFile(source, 30, 6, self._serverThread._answer, 'uvwxyz') + + def testPut(self): + handler = net.FileTcpTaskHandler.FileTcpTaskHandler('test', self._dirs, self._logger, 10) + source = self._dirs[0] + '/filePut.txt' + if os.path.exists(source): + os.unlink(source) + self.assertFalse(os.path.exists(source)) + content = '1234567890abcdefghijklmnopqrstuvwxyz' + date = datetime.datetime(2018, 4, 1, 3, 44, 55, 123456) + mtime = int(date.timestamp() * 1000000) + self._serverThread._firstBlock = content.encode() + # source SIZE OFFSET LENGTH MTIME MODE OWNER GROUP + owner = 201 if self._isRoot else os.geteuid() + group = 202 if self._isRoot else os.getegid() + handler.fulfill(['file', 'test', 'put', source, '36', '0', '36', hex(mtime), hex(0o664), hex(owner), hex(group)], self._serverThread) + self.hasAnswer('OK') + info = os.stat(source) + self.assertEquals(mtime * 1000, info.st_mtime_ns) + self.assertEquals(owner, info.st_uid) + self.assertEquals(group, info.st_gid) + current = base.StringUtils.fromFile(source) + self.assertEquals(content, current) + + def testDiskFree(self): + handler = net.FileTcpTaskHandler.FileTcpTaskHandler('test', self._dirs, self._logger, 10) + handler.fulfill(['file', 'test', 'df'], self._serverThread) + lines = self._serverThread._answer.split('\n') + self.assertEquals('OK', lines[0]) + self.assertMatches(r'^\S+\t\d+\t\d$', lines[1]) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = FileTcpTaskHandlerTest() + tester.run() diff --git a/unittest/net/HttpClientTest.py b/unittest/net/HttpClientTest.py new file mode 100644 index 0000000..ea858a2 --- /dev/null +++ b/unittest/net/HttpClientTest.py @@ -0,0 +1,30 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +from unittest.UnitTestCase import UnitTestCase +import base.Logger +import net.HttpClient + +class HttpClientTest(UnitTestCase): + def __init__(self): + UnitTestCase.__init__(self) + + def testGetHead(self): + logger = base.Logger.Logger('/tmp/unittest.log', True) + client = net.HttpClient.HttpClient(3, logger) + aType = client.getHead('https://wiki.hamatoma.de', 5) + self.assertEquals('text/html; charset=UTF-8', aType) + + def testGetContent(self): + logger = base.Logger.Logger('/tmp/unittest.log', True) + client = net.HttpClient.HttpClient(3, logger) + content = client.getContent('https://wiki.hamatoma.de', 5).decode() + self.assertTrue(content.startswith('<!DOCTYPE html>')) + client.close() + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = HttpClientTest() + tester.run() diff --git a/unittest/net/LinuxTcpTaskHandlerTest.py b/unittest/net/LinuxTcpTaskHandlerTest.py new file mode 100644 index 0000000..d2fe9f2 --- /dev/null +++ b/unittest/net/LinuxTcpTaskHandlerTest.py @@ -0,0 +1,50 @@ +''' +Created on 12.04.2018 + +@author: hm +''' +import os +from unittest.UnitTestCase import UnitTestCase +from base.Logger import Logger +import net.LinuxTcpTaskHandler + +class MockingServerThread: + def __init__(self): + self._answer = None + self._firstBlock = None + def send(self, message): + self._answer = message + def sendBinary(self, data): + self._answer = data.decode() + def readBinary(self): + return None + +class LinuxTcpTaskHandlerTest(UnitTestCase): + def __init__(self): + self._isRoot = os.geteuid() == 0 + self._logFile = '/tmp/logger.log' + self._logger = Logger(self._logFile, True) + self._serverThread = MockingServerThread() + UnitTestCase.__init__(self) + + def hasAnswer(self, expected): + self.assertEquals(expected, self._serverThread._answer) + + def testDiskFree(self): + handler = net.LinuxTcpTaskHandler.LinuxTcpTaskHandler(self._logger) + handler.fulfill(['diskfree'], self._serverThread) + lines = self._serverThread._answer.split('\n') + self.assertEquals('OK', lines[0]) + self.assertMatches(r"(\t?\['[^']+', \d+, \d+, \d+\])+", lines[1]); + + def testUsers(self): + handler = net.LinuxTcpTaskHandler.LinuxTcpTaskHandler(self._logger) + handler.fulfill(['users'], self._serverThread) + lines = self._serverThread._answer.split('\n') + self.assertEquals('OK', lines[0]) + self.assertMatches(r'^\S+\t[:.\d]+(\t[\d:]+){2}', lines[1]) + +if __name__ == '__main__': + #import sys;sys.argv = ['', 'Test.testName'] + tester = LinuxTcpTaskHandlerTest() + tester.run() diff --git a/unittest/net/NetTester.py b/unittest/net/NetTester.py new file mode 100644 index 0000000..e4da574 --- /dev/null +++ b/unittest/net/NetTester.py @@ -0,0 +1,20 @@ +''' +Created on 22.04.2018 + +@author: hm +''' + +import unittest.PackageTest +import unittest.net.FileTcpTaskHandlerTest +import unittest.net.LinuxTcpTaskHandlerTest +import unittest.net.HttpClientTest + +def main(): + runner = unittest.PackageTest.PackageTest('NetTester') + runner.run(unittest.net.FileTcpTaskHandlerTest.FileTcpTaskHandlerTest) + runner.run(unittest.net.LinuxTcpTaskHandlerTest.LinuxTcpTaskHandlerTest) + runner.run(unittest.net.HttpClientTest.HttpClientTest) + runner.finish() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/unittest/net/__init__.py b/unittest/net/__init__.py new file mode 100644 index 0000000..e69de29 -- 2.39.5