From ab7374f3d483a56c4757ef66153b13464edf6e6a Mon Sep 17 00:00:00 2001 From: Hamatoma Date: Mon, 19 Feb 2024 22:15:47 +0100 Subject: [PATCH 1/1] Initial commit --- INSTALL.md | 27 + LICENSE | 121 +++ PyLint | 2 + README.md | 2 + app/BaseApp.py | 596 +++++++++++ app/DbApp.py | 611 +++++++++++ app/DirApp.py | 308 ++++++ app/EMailApp.py | 236 +++++ app/OperatingSystemApp.py | 628 +++++++++++ app/SatelliteApp.py | 534 ++++++++++ app/SpecialApp.py | 369 +++++++ app/SunMonitor.py | 78 ++ app/TextApp.py | 693 +++++++++++++ app/__init__.py | 0 base/BaseLogger.py | 109 ++ base/BaseRandom.py | 119 +++ base/Const.py | 24 + base/CryptoEngine.py | 408 ++++++++ base/CsvProcessor.py | 393 +++++++ base/DirTraverser.py | 267 +++++ base/FileHelper.py | 1201 ++++++++++++++++++++++ base/JavaConfig.py | 118 +++ base/JobController.py | 123 +++ base/LinuxUtils.py | 360 +++++++ base/Logger.py | 56 + base/MemoryLogger.py | 102 ++ base/PortableRandom.py | 73 ++ base/ProcessHelper.py | 237 +++++ base/Scheduler.py | 128 +++ base/SearchRule.py | 639 ++++++++++++ base/SearchRuleList.py | 869 ++++++++++++++++ base/StringUtils.py | 767 ++++++++++++++ base/TextProcessor.py | 314 ++++++ base/UsageInfo.py | 552 ++++++++++ base/__init__.py | 0 data/.gitignore | 2 + data/config/pylint3.conf | 585 +++++++++++ data/private.template.conf | 3 + db/DbDriver.py | 53 + db/MySqlDriver.py | 103 ++ db/PgDriver.py | 134 +++ net/EMail.py | 170 +++ net/HttpClient.py | 189 ++++ net/RestServer.py | 195 ++++ net/SimpleRestServer.py | 152 +++ net/__init__.py | 0 snake_install.sh | 31 + tools/PrepUTest.sh | 2 + unittest/UnitTestCase.py | 404 ++++++++ unittest/UnitTestSuite.py | 126 +++ unittest/__init__.py | 0 unittest/app/DbAppTest.py | 377 +++++++ unittest/app/DirAppTest.py | 229 +++++ unittest/app/EMailAppTest.py | 171 +++ unittest/app/OperatingsSystemAppTest.py | 176 ++++ unittest/app/SatelliteAppTest.py | 182 ++++ unittest/app/TextAppTest.py | 356 +++++++ unittest/base/BaseRandomTest.py | 43 + unittest/base/CryptoEngineTest.py | 178 ++++ unittest/base/CsvProcessorTest.py | 178 ++++ unittest/base/DirTraverserTest.py | 202 ++++ unittest/base/FileHelperTest.py | 622 +++++++++++ unittest/base/JavaConfigTest.py | 74 ++ unittest/base/JobControllerTest.py | 57 + unittest/base/LinuxUtilsTest.py | 117 +++ unittest/base/LoggerTest.py | 99 ++ unittest/base/MemoryLoggerTest.py | 77 ++ unittest/base/PortableRandomTest.py | 43 + unittest/base/ProcessHelperTest.py | 76 ++ unittest/base/SchedulerTest.py | 50 + unittest/base/StringUtilsTest.py | 516 ++++++++++ unittest/base/TextProcessorTest.py | 329 ++++++ unittest/base/UsageInfoTest.py | 264 +++++ unittest/data/etc.tgz | Bin 0 -> 25579 bytes unittest/data/etc.work.tgz | Bin 0 -> 785 bytes unittest/data/example.html | 1 + unittest/data/example.sh | 2 + unittest/data/example.tar | Bin 0 -> 20480 bytes unittest/data/example.tbz | Bin 0 -> 1025 bytes unittest/data/example.tgz | Bin 0 -> 1044 bytes unittest/data/example.txt | 35 + unittest/data/example.zip | Bin 0 -> 3680 bytes unittest/data/examples2.tgz | Bin 0 -> 1004 bytes unittest/data/monitor/localhost.dump.txt | 68 ++ unittest/net/EMailTest.py | 85 ++ unittest/net/HttpClientTest.py | 54 + wirnoi.sql | 771 ++++++++++++++ 87 files changed, 18645 insertions(+) create mode 100644 INSTALL.md create mode 100644 LICENSE create mode 100755 PyLint create mode 100644 README.md create mode 100644 app/BaseApp.py create mode 100755 app/DbApp.py create mode 100755 app/DirApp.py create mode 100755 app/EMailApp.py create mode 100755 app/OperatingSystemApp.py create mode 100755 app/SatelliteApp.py create mode 100755 app/SpecialApp.py create mode 100644 app/SunMonitor.py create mode 100755 app/TextApp.py create mode 100644 app/__init__.py create mode 100644 base/BaseLogger.py create mode 100644 base/BaseRandom.py create mode 100644 base/Const.py create mode 100644 base/CryptoEngine.py create mode 100644 base/CsvProcessor.py create mode 100644 base/DirTraverser.py create mode 100644 base/FileHelper.py create mode 100644 base/JavaConfig.py create mode 100644 base/JobController.py create mode 100644 base/LinuxUtils.py create mode 100644 base/Logger.py create mode 100644 base/MemoryLogger.py create mode 100644 base/PortableRandom.py create mode 100644 base/ProcessHelper.py create mode 100644 base/Scheduler.py create mode 100644 base/SearchRule.py create mode 100644 base/SearchRuleList.py create mode 100644 base/StringUtils.py create mode 100644 base/TextProcessor.py create mode 100644 base/UsageInfo.py create mode 100644 base/__init__.py create mode 100644 data/.gitignore create mode 100644 data/config/pylint3.conf create mode 100644 data/private.template.conf create mode 100644 db/DbDriver.py create mode 100644 db/MySqlDriver.py create mode 100644 db/PgDriver.py create mode 100644 net/EMail.py create mode 100644 net/HttpClient.py create mode 100644 net/RestServer.py create mode 100644 net/SimpleRestServer.py create mode 100644 net/__init__.py create mode 100755 snake_install.sh create mode 100755 tools/PrepUTest.sh create mode 100644 unittest/UnitTestCase.py create mode 100644 unittest/UnitTestSuite.py create mode 100644 unittest/__init__.py create mode 100644 unittest/app/DbAppTest.py create mode 100644 unittest/app/DirAppTest.py create mode 100644 unittest/app/EMailAppTest.py create mode 100644 unittest/app/OperatingsSystemAppTest.py create mode 100644 unittest/app/SatelliteAppTest.py create mode 100644 unittest/app/TextAppTest.py create mode 100644 unittest/base/BaseRandomTest.py create mode 100644 unittest/base/CryptoEngineTest.py create mode 100644 unittest/base/CsvProcessorTest.py create mode 100644 unittest/base/DirTraverserTest.py create mode 100644 unittest/base/FileHelperTest.py create mode 100644 unittest/base/JavaConfigTest.py create mode 100644 unittest/base/JobControllerTest.py create mode 100644 unittest/base/LinuxUtilsTest.py create mode 100644 unittest/base/LoggerTest.py create mode 100644 unittest/base/MemoryLoggerTest.py create mode 100644 unittest/base/PortableRandomTest.py create mode 100644 unittest/base/ProcessHelperTest.py create mode 100644 unittest/base/SchedulerTest.py create mode 100644 unittest/base/StringUtilsTest.py create mode 100644 unittest/base/TextProcessorTest.py create mode 100644 unittest/base/UsageInfoTest.py create mode 100644 unittest/data/etc.tgz create mode 100644 unittest/data/etc.work.tgz create mode 100644 unittest/data/example.html create mode 100644 unittest/data/example.sh create mode 100644 unittest/data/example.tar create mode 100644 unittest/data/example.tbz create mode 100644 unittest/data/example.tgz create mode 100644 unittest/data/example.txt create mode 100644 unittest/data/example.zip create mode 100644 unittest/data/examples2.tgz create mode 100644 unittest/data/monitor/localhost.dump.txt create mode 100644 unittest/net/EMailTest.py create mode 100644 unittest/net/HttpClientTest.py create mode 100644 wirnoi.sql diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..e24b3d3 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,27 @@ +# Creation of a Script for Cloning and Update + +Be root and copy the following text into a terminal. This will create the script SnakeBoxx which inititializes and/or updates the package. +~~~ +FN=/usr/local/bin/SnakeBoxx +cat <<'EOS' >$FN +URL=https://github.com/hamatoma/snakeboxx.git +BASE=/usr/share +if [ $(id -u) != 0 ]; then + echo "be root!" +else + cd $BASE + if [ -d snakeboxx ]; then + cd snakeboxx + git pull $URL + else + git clone $URL + cd snakeboxx + bash snake_install.sh DirApp + bash snake_install.sh TextApp + bash snake_install.sh OperatingSystemApp + fi +fi +EOS +chmod uog+rwx $FN +$FN +~~~ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/PyLint b/PyLint new file mode 100755 index 0000000..a8f7b75 --- /dev/null +++ b/PyLint @@ -0,0 +1,2 @@ +#! /bin/bash +pylint3 --rcfile=data/config/pylint3.conf app | less diff --git a/README.md b/README.md new file mode 100644 index 0000000..31f083c --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# snakeboxx +A collection of useful Python modules and applications diff --git a/app/BaseApp.py b/app/BaseApp.py new file mode 100644 index 0000000..2c1bbad --- /dev/null +++ b/app/BaseApp.py @@ -0,0 +1,596 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import time +import os.path +import posix +import pwd +import tempfile +import traceback + +import base.Const +import base.Logger +import base.StringUtils +import base.FileHelper +import base.UsageInfo +import base.ProcessHelper + +VERSION = '2022.10.07.00' + + +class BaseApp: + '''The base class of all applications. + ''' + __appLatestInstance = None + __underTest = False + + def __init__(self, mainClass, args, isService=False, progName=None): + '''Constructor. + @param mainClass the application name, e.g. dbtool + @param args: the program arguments + @param isService: True: the application offers a systemd service + @param progName: name of the binary None: built from the mainClass + ''' + BaseApp.__appLatestInstance = self + self._mainClass = mainClass + self._appName = ( + mainClass[0:-3] if mainClass.endswith('App') else mainClass).lower() + self._programName = self._appName + 'boxx' if progName is None else progName + self._args = args + self._logger = base.MemoryLogger.MemoryLogger(1) + base.FileHelper.setLogger(self._logger) + self._appBaseDirectory = '/usr/share/snakeboxx' + self._programArguments = [] + self._programOptions = [] + self._start = time.process_time() + self._startReal = time.time() + self._doExit = True + self._usageInfo = None + self._configDirectory = '/etc/snakeboxx' + self._mainMode = None + self._processHelper = None + self._resultText = None + self._resultLines = None + self._configuration = None + self._start = time.process_time() + self._startReal = time.time() + self._userId = posix.getuid() + self._isRoot = self._userId == 0 + self._usageInfo = base.UsageInfo.UsageInfo(self._logger) + self._serviceName = None + self._isService = isService + self._baseTestDir = None + self._daemonSteps = 0x100000000 if not BaseApp.__underTest else 1 + self._optionProcessor = None + + def abort(self, message): + '''Displays a message and stops the programm. + @param message: the error message + ''' + self._logger.error(message) + if self._doExit: + exit(1) + + def argumentError(self, message): + '''Handles a severe error with program exit. + @param message: the error message + ''' + print('Tip: try "{} help [pattern [pattern2]]"'.format( + self._programName)) + self.abort(message) + + def buildConfig(self): + '''Dummy method. + ''' + base.StringUtils.avoidWarning(self) + raise Exception('BaseApp.buildConfig() is not overridden') + + def buildStandardConfig(self, content): + '''Writes a useful configuration into the application specific configuration file. + @param content: the configuration content + ''' + fn = f'{self._configDirectory}{os.sep}{self._appName}.conf' + if os.path.exists(fn): + fn = fn.replace('.conf', '.example') + self._logger.log('creating {} ...'.format(fn), + base.Const.LEVEL_SUMMARY) + base.StringUtils.toFile(fn, content) + + def buildUsage(self): + '''Builds the usage message. + ''' + raise NotImplementedError('buildUsage() not implemented by sub class') + + def buildUsageOptions(self, mode=None, subMode=None): + '''Adds the options for a given mode. + @param mode: None or the mode for witch the option is added + @param subMode: None or the submode for witch the option is added + ''' + raise NotImplementedError( + 'buildUsageOptions() not implemented by sub class') + + def buildUsageCommon(self, isService=False): + '''Appends usage info common for all applications. + @param isService: True: the application is already used as service + ''' + self._usageInfo.addMode( + 'install', 'install\n Installs the application.', 'APP-NAME install') + self._usageInfo.addMode('uninstall', + 'uninstall [--service=]\n Removes the application.', + 'APP-NAME uninstall --service=emailboxx') + self._usageInfo.addMode( + 'build-config', 'build-config\n Creates a useful configuration file.', 'APP-NAME build-config') + self._usageInfo.addMode( + 'version', 'version\n Prints the version number.', 'APP-NAME version') + if isService: + self._usageInfo.addMode( + 'daemon', 'daemon \n start the service daemon') + self._usageInfo.addMode( + 'reload', 'reload \n requests to reload configuration data') + self._usageInfo.addMode('help', r'''help [ []] + Prints a description of the application + + if given: each mode is listed if the pattern matches + : + if given: only submodes are listed if this pattern matches +''', 'APP-NAME help\nAPP-NAME help help sub') + + def checkGlobalOptions(self): + '''Checks the validity of the global options. + Can be overridden. + ''' + pass + + def createPath(self, directory, node=None): + '''Returns a full name of a file or directory depending on normal or unittest environment. + In normal environment the result is [directory] or [directory]/[node]. + If the the global parameter --dir-unittest exists: this option defines the base directory. + In this base directory will be created a subdirectory with the last node of the parameter directory. + For unit tests the result is taken from the global options (_testTargetDir) + @param directory: the original directory's name, e.g. '/etc/nginx' + @param node: None or the filename without path, e.g. 'nginx.conf' + @return: the directory name (node is None) or the filename (directory + os.sep + node) + ''' + if self._baseTestDir is None: + rc = directory + else: + rc = self._baseTestDir + os.sep + rc += 'root' if directory == '/' else os.path.basename(directory) + base.FileHelper.ensureDirectory(rc) + if node is not None: + if rc.endswith(os.sep): + rc += node + else: + rc += os.sep + node + return rc + + def createSystemDScript(self, serviceName, starter, user, group, description): + '''Creates the file controlling a systemd service. + @param serviceName: used for syslog and environment file + @param starter: name of the starter script without path, e.g. 'pymonitor' + @param user: the service is started with this user + @param group: the service is started with this group + @param description: this string is showed when the status is requestes. + ''' + systemDPath = self.createPath('/etc/systemd/system', None) + systemdFile = f'{systemDPath}{os.sep}{serviceName}.service' + script = '''[Unit] +Description={}. +After=syslog.target +[Service] +Type=simple +User={} +Group={} +WorkingDirectory=/etc/snakeboxx +#EnvironmentFile=-/etc/snakeboxx/{}.env +ExecStart=/usr/local/bin/{} daemon {} {} +ExecReload=/usr/local/bin/{} reload {} {} +SyslogIdentifier={} +#StandardOutput=syslog +#StandardError=syslog +Restart=always +RestartSec=3 +[Install] +WantedBy=multi-user.target +'''.format(description, user, group, serviceName, starter, serviceName, user, starter, serviceName, user, serviceName) + with open(systemdFile, 'w') as fp: + fp.write(script) + print('systemd script created: ' + systemdFile) + uid = None + try: + uid = pwd.getpwnam(user) + self._logger.log('user {} ({}) already exists'.format( + user, uid), base.Const.LEVEL_DETAIL) + except KeyError: + if self._isRoot: + self._logger.log('creating user {} ...'.format( + user), base.Const.LEVEL_SUMMARY) + self._processHelper.execute(['/usr/sbin/useradd', user], True) + + def defaultConfigurationFile(self): + '''Returns the name of the file for the configuration example. + ''' + rc = f'{self._configDirectory}{os.sep}{self._appName}.conf' + if os.path.exists(rc): + rc = rc[0:-4] + 'example' + return rc + + def daemon(self): + '''Waits for jobs and executes them + ''' + serviceName = self._programArguments[0] if len( + self._programArguments) >= 1 else self._appName + self._logger.log('starting {} with version {}'.format( + serviceName, VERSION), base.Const.LEVEL_SUMMARY) + fileReloadRequest = self.reloadRequestFile(serviceName) + if self._daemonSteps is None: + self._daemonSteps = 0x7fffffffffff + while self._daemonSteps > 0: + self._daemonSteps -= 1 + hasRequest = fileReloadRequest is not None and os.path.exists( + fileReloadRequest) + if hasRequest: + if not self.handleReloadRequest(fileReloadRequest): + fileReloadRequest = None + self.daemonAction(hasRequest) + interval = self._configuration.getInt('daemon.interval', 3) + time.sleep(interval) + + def daemonAction(self, reloadRequest): + '''Does the real thing in the daemon (= service). + @param reloadRequest: True: a reload request has been done + ''' + base.StringUtils.avoidWarning(self) + base.StringUtils.avoidWarning(reloadRequest) + raise Exception('BaseApp.daemonAction() is not overridden') + + def enableAndStartService(self, serviceName): + '''Enables (makeing "autostart") and starts a service given by name. + ''' + if not self._isRoot: + self._logger.log(f'+ not root: ignoring enable and start service {serviceName}') + else: + self._processHelper.execute(['/bin/systemctl', 'enable', serviceName], True) + self._processHelper.execute(['/bin/systemctl', 'start', serviceName], True) + self._processHelper.execute(['/bin/systemctl', 'status', serviceName], True) + + def getSubMode(self): + '''Returns the sub mode of the program: this is the first argument. + @returns: None: error occurred Otherwise: the submode + ''' + rc = self.shiftProgramArgument() + if rc is None: + self.abort('missing ') + else: + self._usageInfo._activeMode = self._mainMode + self.buildUsage() + self._usageInfo._activeMode = None + info = self._usageInfo._nested[self._mainMode] + names = [] + for item in info._descriptions: + if base.UsageInfo.UsageInfo.matches(rc, item): + names.append(item) + if len(names) == 0: + self.abort(f'unknown ={rc} for ={self._mainMode}') + rc = None + elif len(names) > 1: + self.abort(f'{rc} is ambigious: {" ".join(names)}') + rc = None + else: + rc = names[0] + self._usageInfo = base.UsageInfo.UsageInfo(self._logger) + return rc + + def handleCommonModes(self): + '''Handles the modes common to all application like 'install' + @return: True: mode is a common mode and already handled + ''' + rc = True + if self._mainMode is None: + self.help() + else: + if self._mainMode == 'install': + self.install() + elif self._mainMode == 'uninstall': + self.uninstall() + elif self._mainMode == 'build-config': + self.buildConfig() + elif self._mainMode == 'help': + self.help() + elif self._mainMode == 'reload': + self.requestReload() + elif self._mainMode == 'version': + print('version: ' + VERSION) + else: + rc = False + return rc + + def handleReloadRequest(self, fileReloadRequest): + '''Handles a request for reloading configuration inside the daemon. + @param fileReloadRequest: the name of the file defining a request + @return: True: success False: the file could not be deleted/modified + ''' + rc = True + try: + os.unlink(fileReloadRequest) + except OSError: + base.StringUtils.toFile(fileReloadRequest, 'found') + self._configuration.readConfig(self._configuration._filename) + if os.path.exists(fileReloadRequest) and os.path.getsize(fileReloadRequest) == 0: + self._logger.error( + 'cannot delete/rewrite reload request file: ' + fileReloadRequest) + rc = False + return rc + + def handleGlobalOptions(self): + '''Search for the global options and handles it. + Divides the rest of arguments into _programArguments and _programOptions + ''' + mode = '' + self._usageInfo.addMode(mode, + r''': this options can be used for all modes and must positioned in front of the mode. +''', '') + option = base.UsageInfo.Option('config-directory', 'c', 'the directory containing the configuration file', + 'string', self._configDirectory) + self._usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option('runtime', 'r', 'the runtime will be displayed at the end of the program', + 'bool') + self._usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option( + 'dir-unittest', None, 'a directory used for unit tests') + self._usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option( + 'log-file', None, 'the file for logging messages. If empty the MemoryLogger will be used', mayBeEmpty=True) + self._usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option('verbose-level', 'v', 'sets the amount of logs: only log messages with a level <= will be displayed', + 'int', base.Const.LEVEL_SUMMARY) + self._usageInfo.addModeOption(mode, option) + opts = [] + while self._args and self._args[0].startswith('-'): + opts.append(self._args[0]) + self._args = self._args[1:] + optionProcessor = self._usageInfo.currentOptionProcessor(mode) + if not optionProcessor.scan(opts): + self.abort('error on global options') + else: + self._configDirectory = optionProcessor.valueOf('config-directory') + self._baseTestDir = optionProcessor.valueOf('dir-unittest') + if self._args: + self._mainMode = self._args[0] + self._args = self._args[1:] + for arg in self._args: + if arg.startswith('-'): + self._programOptions.append(arg) + else: + self._programArguments.append(arg) + # === configuration + fn = None + if self._mainMode not in ['install', 'uninstall', 'help']: + fn = f'{self._configDirectory}{os.sep}{self._appName}.conf' + if not os.path.exists(fn): + self.buildConfig() + if not os.path.exists(fn): + fn = None + self._configuration = base.JavaConfig.JavaConfig(fn, self._logger) + # === logger + logFile = optionProcessor.valueOf('log-file') + if (self._appName == '!unittest' or self.__underTest) and logFile is not None: + self._logger._verboseLevel = base.Const.LEVEL_FINE + else: + if logFile is None: + logFile = self._configuration.getString('logfile') + if logFile is None: + logFile = f'/var/log/local/{self._programName}.log' + # '' or '-' means memory logger + if logFile != '' or logFile == '-': + oldLogger = self._logger + self._logger = base.Logger.Logger( + logFile, optionProcessor.valueOf('verbose-level')) + oldLogger.derive(self._logger) + base.FileHelper.setLogger(self._logger) + base.StringUtils.setLogger(self._logger) + self._processHelper = base.ProcessHelper.ProcessHelper( + self._logger) + if not optionProcessor.valueOf('runtime'): + self._start = None + + def handleOptions(self, subMode=None): + '''Compares the current options with the specified options. + @return: True: success + ''' + self.buildUsageOptions(None, subMode) + self._optionProcessor = processor = self._usageInfo.currentOptionProcessor( + self._mainMode, subMode) + rc = processor.scan(self._programOptions) + self.checkGlobalOptions() + return rc + + def help(self): + '''Prints the usage message. + ''' + self.buildUsageCommon(self._isService) + self.buildUsage() + for mode in self._usageInfo._descriptions: + self.buildUsageOptions(mode) + self._usageInfo.extendUsageInfoWithOptions(mode) + self._usageInfo.replaceMacro('APP-NAME', self._programName) + pattern = '' if not self._programArguments else self._programArguments[0] + pattern2 = '' if len( + self._programArguments) < 2 else self._programArguments[1] + info = self._usageInfo.asString(pattern, 0, pattern2) + self._resultText = info + print(info) + + def install(self): + '''Installs the program. + ''' + if not self._isRoot and not self.__underTest: + self.argumentError('be root!') + else: + configDir = self._configDirectory + base.FileHelper.ensureDirectory(configDir) + self.buildConfig() + app = self.createPath('/usr/local/bin', self._programName) + source = f'{self._appBaseDirectory}{os.sep}app{os.sep}{self._mainClass}.py' + if os.path.exists(app): + base.FileHelper.ensureFileDoesNotExist(app) + self._logger.log( + f'creating starter {app} -> {source}', base.Const.LEVEL_SUMMARY) + os.symlink(source, app) + fn = '/usr/lib/python3/dist-packages/snakeboxx.py' + if not os.path.exists(fn): + self._logger.log('creating {} ...'.format(fn)) + base.StringUtils.toFile(fn, """'''Prepare for usage of snakeboxx modules. +''' +import sys + +if '/usr/share/snakeboxx' not in sys.path: + sys.path.insert(0, '/usr/share/snakeboxx') + +def startApplication(): + '''Starts the application. + In this version: do nothing + ''' +""") + + @staticmethod + def isUnderTest(): + '''Returns wether the application runs as unittest. + @return: True: application is under test + ''' + return BaseApp.__underTest + + def installAsService(self, service, startAtOnce=False): + '''Enables the service and start it (if wanted): + @param service: name of the service + @param startAtOnce: True: the service will be started False: do not start the service + ''' + if self._isRoot: + self._processHelper.execute( + ['/usr/bin/systemctl', 'enable', service], True) + if startAtOnce: + self._processHelper.execute( + ['/usr/bin/systemctl', 'start', service], True) + self._processHelper.execute( + ['/usr/bin/systemctl', 'status', service], True) + else: + print( + 'please inspect the configuration and then execute "/usr/bin/systemctl start {}"'.format(service)) + + @staticmethod + def lastInstance(): + '''Returns the last instantiated instance of BaseApp. + @returns: the last instantiated instance of BaseApp + ''' + return BaseApp.__appLatestInstance + + def main(self): + '''The main function: handles the complete application process. + ''' + try: + self.handleGlobalOptions() + if not self.handleCommonModes(): + if self._mainMode is None: + self.help() + else: + # Marks that onyl modes will be assembled + self._usageInfo._modeNames = [] + self.buildUsage() + self._mainMode = self._usageInfo.findMode(self._mainMode) + self._usageInfo = base.UsageInfo.UsageInfo(self._logger) + self.run() + if self._start is not None: + real = base.StringUtils.secondsToString( + int(time.time() - self._startReal)) + cpu = base.StringUtils.secondsToString( + int(time.process_time() - self._start)) + self._logger.log( + 'runtime (real/process): {}/{}'.format(real, cpu), base.Const.LEVEL_SUMMARY) + except Exception as exc: + self._logger.error(f'{type(exc)}: {exc}') + traceback.print_exc() + + def reloadRequestFile(self, serviceName): + '''Return the name of the file for request a reload. + @param serviceName: name of the service + @return the filename + ''' + base.StringUtils.avoidWarning(self) + rc = '{}/{}/reload.request'.format(tempfile.gettempdir(), serviceName) + return rc + + def requestReload(self): + '''Requests a reading of the configuration data. + ''' + service = self.shiftProgramArgument(self._programName) + fn = self.reloadRequestFile(service) + base.FileHelper.ensureDirectory(os.path.dirname(fn), 0o777) + self._logger.log('reload requested', base.Const.LEVEL_SUMMARY) + base.StringUtils.toFile(fn, '') + os.chmod(fn, 0o666) + if not self._isRoot: + count = 10 if not self.__underTest else 1 + self._logger.log( + f'waiting for answer (max {count} sec)', base.Const.LEVEL_SUMMARY) + for ix in range(count): + if not os.path.exists(fn): + self._logger.log(f'{ix+1}: request not processed', + base.Const.LEVEL_SUMMARY) + break + time.sleep(1) + if ix <= 0: + self._logger.error('reload request was not processed') + base.FileHelper.ensureFileDoesNotExist(fn) + + def run(self): + '''Starts the application specific work. + Note: must be overwritten by the sub class. + ''' + raise NotImplementedError('BaseApp.run() is not overridden') + + @staticmethod + def setUnderTest(status=True): + ''' Marks the application to run under a unittest. + @param status: True: set test status False: release test status + ''' + BaseApp.__underTest = status + + def shiftProgramArgument(self, defaultValue=None): + '''Returns the next program argument. + @param defaultValue: the return value if no program argument is available + @return None: no more arguments otherwise: the next program argument which is already removed from _programArguments + ''' + rc = defaultValue + if self._programArguments: + rc = self._programArguments[0] + del self._programArguments[0] + return rc + + def uninstall(self): + '''Uninstalls the program. + ''' + if not self._isRoot and not self.__underTest: + self.argumentError('be root!') + else: + app = self.createPath('/usr/local/bin', self._programName) + if os.path.exists(app): + self._logger.log( + f'removing starter {app} ...', base.Const.LEVEL_SUMMARY) + base.FileHelper.ensureFileDoesNotExist(app) + if self._programOptions and self._programOptions[0].startswith('--service='): + serviceName = self._programOptions[0][10:] + serviceFile = self.createPath( + '/etc/systemctl/system', f'{serviceName}.service') + if os.path.exists(serviceFile): + self._logger.log('removing service file {} ...'.format( + serviceFile), base.Const.LEVEL_SUMMARY) + os.unlink(serviceFile) + else: + self._logger.log('not found: ' + serviceFile, + base.Const.LEVEL_SUMMARY) + + def unknownMode(self): + '''Handles the error "unknown mode". + ''' diff --git a/app/DbApp.py b/app/DbApp.py new file mode 100755 index 0000000..7f858e5 --- /dev/null +++ b/app/DbApp.py @@ -0,0 +1,611 @@ +#! /usr/bin/python3 +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import sys +import os.path +import datetime +# see snake_install.sh +import snakeboxx + +import base.UsageInfo +import app.BaseApp +from db.MySqlDriver import MySqlDriver +from db.PgDriver import PgDriver +from db.DbDriver import DbDriver +from typing import List + + +class DbApp(app.BaseApp.BaseApp): + '''Performs some tasks with mysql databases: creation, import, export... + ''' + + def __init__(self, args: list[str]): + '''Constructor. + @param args: the program arguments, e.g. ['test', 'a@bc.de'] + ''' + app.BaseApp.BaseApp.__init__(self, 'DbApp', args, None, 'dbboxx') + self._processTool = base.ProcessHelper.ProcessHelper(self._logger) + self._driverName = 'mysql' + + def _getDriver(self) -> DbDriver: + '''Returns the driver given by the global option "driver". + ''' + name = self._driverName; + if name == 'postgresql': + rc = PgDriver(self) + else: + rc = MySqlDriver(self) + return rc + + def _getFileNameWebappConfig(self, domain: str) -> str: + '''Returns the name of the web application configuration file. + @param domain the domain of the application + @return the filename + ''' + fnConfig = self.createPath(f'/etc/snakeboxx/webapps.d', domain) + return fnConfig + + def _getWebAppConfiguration(self, domain: str) -> base.JavaConfig.JavaConfig: + '''Returns the configuration of a web application as a JavaConfig instance. + @param domain the domain of the application + @return: the JavaConfig instance + ''' + fn = self._getFileNameWebappConfig(domain) + rc = base.JavaConfig.JavaConfig(fn, self._logger) + return rc + + def allDbs(self): + '''Lists the databases. + ''' + if self.handleOptions(): + driver = self._getDriver() + rc = self._resultLines = driver.allDbs() + print('\n'.join(rc)) + + def buildConfig(self): + '''Creates an useful configuration example. + ''' + content = '''# created by DbApp +logfile=/var/log/local/dbboxx.log +mysql.host=localhost +mysql.port=3306 +mysql.admin.user=dbadm +mysql.admin.code=TopSecret +postgresql.host=localhost +postgresql.port=5432 +postgresql.admin.user=dbadm +postgresql.admin.code=TopSecret +postgresql.os.user=postgres +webapp.base=/srv/www +''' + self.buildStandardConfig(content) + + def buildArgvMysql(self, command: str, needsAdmin=False): + '''Builds an argument vector for a mysql relevant command (mysql or mysqldump...). + @param command: 'mysql' or 'mysqldump' + @param needsAdmin: True: the operation needs administrator rights + ''' + rc = None + user = self._optionProcessor.valueOf( + 'adm-name' if needsAdmin else 'user-name') + if user is not None: + code = self._optionProcessor.valueOf( + 'adm-password' if needsAdmin else 'user-password') + else: + user = self._configuration.getString('mysql.admin.user') + code = self._configuration.getString('mysql.admin.code') + if user is None: + self.abort('missing user (option or configuration file)') + elif code is None: + self.abort('missing password (option or configuration file)') + else: + rc = [command] + rc.append(f'-u{user}') + if code != '': + rc.append(f'-p{code}') + return rc + + def buildUsage(self): + '''Builds the usage message. + ''' + self._usageInfo.appendDescription('''APP-NAME [] + Info and manipulation of mysql databases. +''') + self._usageInfo.addMode('all-dbs', '''all-dbs + Lists all databases. + ''', '''APP-NAME all-dbs +APP-NAME all-dbs --admin=jonny --adm-password=ForgetNever +''') + self._usageInfo.addMode('create-admin', '''APP-NAME create-admin [] + Create a database user with admin rights. + ''', '''APP-NAME create-admin dbadm TopSecret +APP-NAME create-admin dbadmin MoreSecret --admin=root --adm-password=ForgetNever +''') + self._usageInfo.addMode('create-db-and-user', '''APP-NAME create-db-and-user [ ] [] + Create a database and a user with rights to modify this db. + ''', '''APP-NAME create-db-and-user dbcompany +APP-NAME create-db-and-user dbcompany dbadmin MoreSecret --adm-name=root --adm-password=ForgetNever +''') + self._usageInfo.addMode('create-webapp', '''APP-NAME create-webapp [ []] [] + Create a web application with db, user, password and the base directory . + If missing .. the data will be taken from configuration. + Default : /srv/www/ + ''', '''APP-NAME create-webapp example.com dbexample example TopSecret +APP-NAME create-webapp example.com dbexample example TopSecret --no-dir +APP-NAME create-webapp example.com --no-dir +''') + self._usageInfo.addMode('delete-db', '''APP-NAME delete-db [] [] + Deletes a database (with or without a confirmation and/or a backup). + : name of the database + : before deleting the database is dumped to this file. Should end with '.sql' or '.sql.gz' + if ending with '.gz' the dump will be compressed via gzip. Default: /.sql.gz + ''', '''APP-NAME delete-db dbcompany ../db/dbcompany.sql.gz +APP-NAME delete-db dbcompany --no-backup --force --user-name=root --user-password=ForgetNever +''') + self._usageInfo.addMode('delete-user', '''APP-NAME delete-user + Deletes a database user with/without confirmation. + : name of the database +''', '''APP-NAME delete-user jonny +APP-NAME delete-user eve --force --user-name=root --user-password=ForgetNever +''') + self._usageInfo.addMode('export-db', '''APP-NAME export-db [] + Exports a database into a file. + : name of the database + : the content is dumped to this file. If ending with '.gz' the content will be compressed with gzip +''', '''APP-NAME export-db dbcompany dbc.sql.gz +APP-NAME export-db dbcompany /data/version3.sql --force --user-name=root --user-password=ForgetNever +''') + self._usageInfo.addMode('export-webapp', '''APP-NAME export-webapp [] [] + Exports a database into a file. + : the domain of the web application, e.g. 'example.com' + : the content is dumped to this file. If ending with '.gz' the content will be compressed with gzip + if not given: it will be created in the tempororary directory with the name .sql.gz +''', '''APP-NAME export-webapp example.com +APP-NAME export-webapp example.com /data/version9.sql --force --user-name=root --user-password=ForgetNever +''') + self._usageInfo.addMode('import-db', '''APP-NAME import-db [] + Imports a file into a database. + : name of the database + : the new content as SQL statements. If ending with '.gz' the content is compressed with gzip +''', '''APP-NAME export-db dbcompany dbc.sql.gz +APP-NAME import-db dbcompany company_update1.sql --user-name=root --user-password=ForgetNever +''') + self._usageInfo.addMode('import-webapp', '''APP-NAME import-webapp [] + Imports a file into a database of a web application. + : domain of the web application, e.g. 'example.com' + : the new content as SQL statements. If ending with '.gz' the content is compressed with gzip +''', '''APP-NAME import-webapp www.example.com dbc.sql.gz +''') + self._usageInfo.addMode('restore-webapps', '''APP-NAME restore-webapps [] [] + Restores all web applications defined by given database dumps in . DB, user and password will be created. + Note: the db dumps needs the filename .sql.gz + Note: the webapp description must exist: /etc/snakeboxx/webapp.d/.conf + : domain of the web application, e.g. 'example.com' + : the new content as SQL statements. If ending with '.gz' the content is compressed with gzip +''', '''APP-NAME import-webapp www.example.com dbc.sql.gz +''') + + def buildUsageOptions(self, mode: str=None, subMode: str=None): + '''Adds the options for a given mode. + @param mode: None or the mode for which the option is added + @param subMode: None or the submode for which the option is added + ''' + def add(mode: str, opt: str): + self._usageInfo.addModeOption(mode, opt) + + def addAdminOpts(mode: str): + add(mode, base.UsageInfo.Option('adm-name', + 'N', 'name of a db user with admin rights')) + add(mode, base.UsageInfo.Option( + 'adm-password', 'P', 'password of the admin')) + add(mode, base.UsageInfo.Option( + 'adm-group', 'g', 'group of the administrators')) + + def addDriver(mode: str): + add(mode, base.UsageInfo.Option('driver', 'D', + 'database driver: mysql or pg', 'string', defaultValue='mysql')) + + def addForce(mode: str): + add(mode, base.UsageInfo.Option('force', 'f', + 'delete without confirmation', 'bool')) + + def addNoBackup(mode: str): + add(mode, base.UsageInfo.Option( + 'no-backup', 'n', 'no backup export is done', 'bool')) + + def addUserOpts(mode: str): + add(mode, base.UsageInfo.Option('user-name', 'n', + 'name of a db user with rights for the related db')) + add(mode, base.UsageInfo.Option( + 'user-password', 'P', 'password of the user')) + + base.StringUtils.avoidWarning(subMode) + if mode is None: + mode = self._mainMode + if mode not in ['', 'install', 'uninstall', 'build-config', 'version', 'help']: + addDriver(mode) + if mode == 'all-dbs': + addAdminOpts(mode) + add(mode, base.UsageInfo.Option('internal-too', 'i', + 'the internal databases will be listed too', 'bool')) + elif mode == 'create-admin': + addAdminOpts(mode) + add(mode, base.UsageInfo.Option('readonly', 'r', + 'the admin gets only read rights (e.g. for backup)', 'bool')) + elif mode == 'create-db-and-user': + addAdminOpts(mode) + elif mode == 'delete-db': + addUserOpts(mode) + addForce(mode) + addNoBackup(mode) + elif mode == 'delete-user': + addAdminOpts(mode) + addForce(mode) + elif mode == 'export-db': + addUserOpts(mode) + elif mode == 'export-webapp': + self._usageInfo.addModeOption(mode, None) + elif mode == 'import-db': + addUserOpts(mode) + addForce(mode) + addNoBackup(mode) + elif mode == 'import-webapp': + addForce(mode) + addNoBackup(mode) + elif mode == 'create-webapp': + add(mode, base.UsageInfo.Option('no-dir', 'n', + 'the directory will not created', 'bool')) + elif mode == 'restore-webapps': + addAdminOpts(mode) + + def checkGlobalOptions(self): + '''Checks the validity of the global options. + Can be overridden. + ''' + driverName = self._optionProcessor.valueOf('driver') + if 'mysql'.startswith(driverName): + self._driverName = 'mysql' + elif 'pg'.startswith(driverName) or 'postgresql'.startswith(driverName): + self._driverName = 'postgresql' + else: + self.abort('unknown driver name: {driverName} known: "mysql" "pg"') + + def confirm(self, message: str, expected: str) -> bool: + '''Confirms a critical action. + @param message: the message of confirmation + @param expected: this value must be typed by the user + @return: True: confirmation successful + ''' + answer = input(message + ': ') + rc = answer.strip() == expected + if not rc: + self.abort('confirmation failed') + return rc + + def createAdmin(self): + '''Creates an user able to process all databases + ''' + user = self.shiftProgramArgument() + code = self.shiftProgramArgument() + if code is None: + self.abort('too few arguments') + else: + if self.handleOptions(): + readOnly = self._optionProcessor.valueOf('readonly') + driver = self._getDriver() + driver.createAdmin(user, code, readOnly) + + def createDbAndUser(self): + '''Creates a database and a user with rights to modify this db. + ''' + db = self.shiftProgramArgument() + user = self.shiftProgramArgument() + code = self.shiftProgramArgument() + if db is None: + self.abort('too few arguments') + elif user is not None and code is None: + self.abort('missing password') + elif code is not None and code.find("'") >= 0: + self.abort('password must not contain "\'"') + elif self.handleOptions(): + driver = self._getDriver() + driver.createDbAndUser(db, user, code) + + def createWebApp(self): + '''Creates a database and a user with rights to modify this db. + ''' + domain = self.shiftProgramArgument() + db = self.shiftProgramArgument() + user = self.shiftProgramArgument() + code = self.shiftProgramArgument() + home = self.shiftProgramArgument() + testConfig = True + if home is None and domain is not None: + home = f'/srv/www/{domain}' + fnConfig = self._getFileNameWebappConfig(domain) + if db is None and os.path.exists(fnConfig): + config = self.webApp(domain) + db = config.getString('db') + user = config.getString('user') + code = config.getString('password') + home = config.getString('directory') + driver = 'mysql' + testConfig = False + if code is None: + self.abort(f'too few arguments or {config} does not exist') + elif testConfig and os.path.exists(fnConfig): + self.abort(f'webapp {domain} already exist: {fnConfig}') + elif os.path.exists(fnConfig + '.conf'): + self.abort(f'webapp {domain} already exist: {fnConfig}.conf') + elif self.handleOptions(): + self._logger.log( + f'creating {fnConfig} ...', base.Const.LEVEL_SUMMARY) + if testConfig: + driver = self.driverName() + base.StringUtils.toFile(fnConfig, f'''db={db} +user={user} +password={code} +sql.file={domain}-{db} +directory={home} +excluded= +driver={driver} +''') + sql = f'''GRANT ALL ON {db}.* TO '{user}'@'localhost' IDENTIFIED BY '{code}'; +flush privileges; +create database if not exists {db};''' + self._logger.log( + f'creating db {db}...', base.Const.LEVEL_SUMMARY) + self._logger.log(sql, base.Const.LEVEL_FINE) + argv = self.buildArgvMysql('mysql', True) + self._processTool.executeInput( + argv, self._logger._verboseLevel >= base.Const.LEVEL_DETAIL, sql) + if not self._optionProcessor.valueOf('no-dir') and not os.path.exists(home): + base.FileHelper.ensureDirectory(home) + + def deleteDb(self): + '''Creates a database and a user with rights to modify this db. + ''' + db = self.shiftProgramArgument() + backupFile = self.shiftProgramArgument() + if db is None: + self.abort('too few arguments') + elif self.handleOptions(): + force = self._optionProcessor.valueOf('force') + if not self._optionProcessor.valueOf('no-backup'): + self.export(db, backupFile) + if force or self.confirm('If you want to delete you must type the name of the db', db): + driver = self._getDriver() + driver.deleteDb(db) + + def deleteUser(self): + '''Deletes a database user. + ''' + user = self.shiftProgramArgument() + if user is None: + self.abort('too few arguments') + else: + if self.handleOptions(): + force = self._optionProcessor.valueOf('force') + if force or self.confirm('If you want to delete you must type the name of the user', user): + sql = f'''DROP user {user}@localhost; +flush privileges; +''' + self._logger.log( + f'deleting user {user}...', base.Const.LEVEL_SUMMARY) + self._logger.log(sql, base.Const.LEVEL_FINE) + argv = self.buildArgvMysql('mysql', True) + argv.append('mysql') + self._processTool.executeInput( + argv, self._logger._verboseLevel >= base.Const.LEVEL_DETAIL, sql) + + def driverName(self): + '''Returns the name of the current driver, e.g. "mysql + ''' + return self._driverName + + def export(self, db: str, target: str, user: str=None, code: str=None): + '''Exports a database into a file. + @param db: the name of the database + @param target: the file to export. None: a file in the temp directory will be written + @param user: None: will be got from options or configuration. Otherwise: a user with read rights for db + @param code: None: will be got from options or configuration. Otherwise: the password of user + ''' + if target is None: + now = datetime.datetime.now().strftime('%y.%m.%d-%H_%M') + target = base.FileHelper.tempFile(f'{db}.{now}.sql.gz') + if user is None: + user = self._optionProcessor.valueOf('user-name') + if user is not None: + code = self._optionProcessor.valueOf('user-password') + else: + user = self._configuration.getString('mysql.admin.user') + code = self._configuration.getString('mysql.admin.code') + self._logger.log(f'exporting {db} to {target}') + if target.endswith('.gz'): + self._processHelper.executeScript(f'''#! /bin/bash +/usr/bin/mysqldump --default-character-set=utf8mb4 --single-transaction -u{user} '-p{code}' {db} | gzip -c > {target} +''') + else: + contents = f'''#! /bin/bash +/usr/bin/mysqldump --default-character-set=utf8mb4 --single-transaction -u{user} '-p{code}' {db} > {target} +''' + self._processHelper.executeScript(contents) + + def exportDb(self): + '''Exports a database into a file. + ''' + db = self.shiftProgramArgument() + file = self.shiftProgramArgument() + if db is None: + self.abort('too few arguments') + else: + if self.handleOptions(): + self.export(db, file) + + def exportWebApp(self): + '''Exports a database into a file. + ''' + domain = self.shiftProgramArgument() + file = self.shiftProgramArgument() + if domain is None: + self.abort('too few arguments') + else: + if self.handleOptions(): + config = self.webApp(domain) + if config is None: + self.abort(f'missing configuration for {domain}') + else: + self.export(config.getString('db'), file, config.getString( + 'user'), config.getString('password')) + + def importDb(self): + '''Imports a file into a database. + ''' + db = self.shiftProgramArgument() + file = self.shiftProgramArgument() + if file is None: + self.abort('too few arguments') + else: + if self.handleOptions(): + force = self._optionProcessor.valueOf('force') + if not self._optionProcessor.valueOf('no-backup'): + self.export(db, None) + if force or self.confirm('If you want to import you must type the name of the db', db): + self.importSql(db, file) + + def importSql(self, db: str, source: str, user: str=None, code: str=None): + '''Exports a database into a file. + @param db: the name of the database + @param source: the file to import + @param user: None: will be got from options or configuration. Otherwise: a user with read rights for db + @param code: None: will be got from options or configuration. Otherwise: the password of user + ''' + if user is None: + user = self._optionProcessor.valueOf('user-name') + if user is not None: + code = self._optionProcessor.valueOf('user-password') + else: + user = self._configuration.getString('mysql.admin.user') + code = self._configuration.getString('mysql.admin.code') + self._logger.log(f'import {db} from {source}') + if source.endswith('.gz'): + self._processHelper.executeScript(f'''#! /bin/bash +/usr/bin/zcat {source} | /usr/bin/mysql -u{user} '-p{code}' {db} +''') + else: + self._processHelper.executeScript(f'''#! /bin/bash +/usr/bin/mysql -u{user} '-p{code}' {db} < {source} +''') + + def importWebApp(self): + '''Imports a file into a database. + ''' + domain = self.shiftProgramArgument() + file = self.shiftProgramArgument() + if file is None: + self.abort('too few arguments') + else: + if self.handleOptions(): + config = self.webApp(domain) + if config is None: + self.abort('missing configuration for {domain}') + else: + db = config.getString('db') + force = self._optionProcessor.valueOf('force') + if not self._optionProcessor.valueOf('no-backup'): + self.export(db, None) + if force or self.confirm('If you want to import you must type the name of the db', db): + self.importSql(db, file, config.getString( + 'user'), config.getString('password')) + + def restoreWebApps(self): + '''Restore many webapps by creating DB, user and password. + ''' + dirDumps = self.shiftProgramArgument('.') + if dirDumps != '.' and not os.path.isdir(dirDumps): + self.abort(f'not a directory: {dirDumps}') + elif self.handleOptions(): + for fnDump in os.listdir(dirDumps): + if fnDump.endswith('.sql.gz'): + full = os.path.join(dirDumps, fnDump) + domain = fnDump[0:-7] + fnConfig = self._getFileNameWebappConfig(domain) + if not os.path.exists(fnConfig): + self._logger.error( + f'missing configuration file {fnConfig}') + else: + self._logger.log('= restoring {domain}...') + config = self.webApp(domain) + db = config.getString('db') + user = config.getString('user') + code = config.getString('password') + if db is None or user is None or code is None: + self._logger.error( + f'incomplete configuration in {fnConfig}: db: {db} user: {user} pw: {code}') + else: + self.createDbAndUser2(db, user, code) + self.importSql(db, full, user, code) + + def run(self): + '''Implements the tasks of the application. + ''' + if self._mainMode == 'all-dbs': + self.allDbs() + elif self._mainMode == 'create-admin': + self.createAdmin() + elif self._mainMode == 'create-db-and-user': + self.createDbAndUser() + elif self._mainMode == 'create-webapp': + self.createWebApp() + elif self._mainMode == 'delete-db': + self.deleteDb() + elif self._mainMode == 'delete-user': + self.deleteUser() + elif self._mainMode == 'export-db': + self.exportDb() + elif self._mainMode == 'export-webapp': + self.exportWebApp() + elif self._mainMode == 'import-db': + self.importDb() + elif self._mainMode == 'import-webapp': + self.importWebApp() + elif self._mainMode == 'restore-webapps': + self.restoreWebApps() + else: + self.abort('unknown mode: ' + self._mainMode) + + def webApp(self, domain: str) -> base.JavaConfig.JavaConfig: + '''Returns the web application configuration. + @param domain: the domain of the web application + @return: None: not found Otherwise: the JavaConfig instance + ''' + rc = None + fnConfig = self._getFileNameWebappConfig(domain) + if not os.path.exists(fnConfig): + fnConfig += '.conf' + if os.path.exists(fnConfig): + rc = base.JavaConfig.JavaConfig(fnConfig, self._logger) + config = base.JavaConfig.JavaConfig(fnConfig, self._logger) + db = config.getString('db') + user = config.getString('user') + code = config.getString('password') + if db is None or user is None or code is None: + self._logger.error(f'incomplete data in {fnConfig}') + rc = None + return rc + + +def main(args: List[str]): + '''Main function. + @param args: the program arguments + ''' + snakeboxx.startApplication() + application = DbApp(args) + application.main() + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/app/DirApp.py b/app/DirApp.py new file mode 100755 index 0000000..c6bd9d2 --- /dev/null +++ b/app/DirApp.py @@ -0,0 +1,308 @@ +#! /usr/bin/python3 +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import sys +import os.path +import time +import fnmatch +# see snake_install.sh +import snakeboxx + + +import base.DirTraverser +import base.FileHelper +import app.BaseApp + + +class MetaData: + '''Stores the meta data of a file. + ''' + + def __init__(self, name, statInfo, value): + '''Adds a file to the list if it is a extremum. + @param name: the filename + @param statInfo: the meta data of the file + @param value: the value of the sort criterion + ''' + self._name = name + self._stat = statInfo + self._value = value + + +class ExtremaList: + '''Stores a list of files sorted by a given criteria. + ''' + + def __init__(self, size, criterion, descending): + '''Constructor. + @param size: maximal length of the internal list + @param criterion: t(ime), s(ize) + @param descending: True: + ''' + self._list = [] + self._size = size + self._criterion = criterion + self._descending = descending + self._limit = None + self._factor = -1 if self._descending else 1 + self._minLength = 0 + + def merge(self, name, statInfo): + '''Adds a file to the list if it is a extremum. + @param name: the filename + @param statInfo: the meta data of the file + ''' + sortIt = False + if len(self._list) < self._size: + value = self._factor * \ + (statInfo.st_mtime if self._criterion == 't' else statInfo.st_size) + self._list.append(MetaData(name, statInfo, value)) + sortIt = True + else: + value = self._factor * \ + (statInfo.st_mtime if self._criterion == 't' else statInfo.st_size) + if value > self._limit: + sortIt = True + # replace the "smallest" element: + self._list[0] = MetaData(name, statInfo, value) + if sortIt: + self._list.sort(key=lambda info: info._value) + self._limit = self._list[0]._value + + def show(self, title, lines): + '''Shows the list of files + @param title: the text above the file list + @param lines: IN/OUT: the info is stored there + ''' + lines.append('== ' + title) + self._list.reverse() + for item in self._list: + info = base.FileHelper.listFile( + item._stat, item._name, self._criterion == 't', True) + lines.append(info) + + +class DirApp(app.BaseApp.BaseApp): + '''Performs some tasks with text files: searching, modification... + ''' + + def __init__(self, args): + '''Constructor. + @param args: the program arguments, e.g. ['test', 'a@bc.de'] + ''' + app.BaseApp.BaseApp.__init__(self, 'DirApp', args, None, 'dirboxx') + self._traverser = None + self._processor = None + self._hostname = None + + def adjust(self): + '''Adjusts the file modifcation datetime in a given directory. + ''' + self._resultLines = [] + source = self.shiftProgramArgument() + target = self.shiftProgramArgument() + if target is None: + self.abort('too few arguments') + elif not os.path.isdir(source): + self.abort(f'not a directory: {source}') + elif not os.path.isdir(target): + self.abort(f'not a directory: {target}') + if self.handleOptions(): + pattern = self._optionProcessor.valueOf('pattern') + self.adjustDir(pattern, source, target) + + def adjustDir(self, pattern, source, target): + '''For all files in target: if a file exists in source the modification time is transfered from source to target. + @param source: the directory with files having the needed modification time + @param target: the directory with the files to change + @param pattern: a shell pattern for the files to process, e.g. "*.jpg" + ''' + info = f'= {target}:' + self._resultLines.append(info) + self._logger.log(info, base.Const.LEVEL_DETAIL) + dirs = [] + for file in os.listdir(target): + full = os.path.join(target, file) + if os.path.isdir(full): + if self._optionProcessor.valueOf('recursive'): + dirs.append(file) + elif not fnmatch.fnmatch(file, pattern): + info = f'{file} ignored' + self._resultLines.append(info) + self._logger.log(info, base.Const.LEVEL_FINE) + else: + src = os.path.join(source, file) + if os.path.exists(src): + timeSrc = os.stat(src).st_mtime + timeTrg = os.stat(full).st_mtime + base.FileHelper.setModified(full, timeSrc) + info = f'{file}: {time.strftime("%Y.%m.%d-%H:%M:%S", time.localtime(timeTrg))} -> {time.strftime("%Y.%m.%d-%H:%M:%S", time.localtime(timeSrc))}' + self._resultLines.append(info) + self._logger.log(info, base.Const.LEVEL_LOOP) + for item in dirs: + self.adjustDir(pattern, os.path.join(source, item), os.path.join(target, item)) + + def buildConfig(self): + '''Creates an useful configuration example. + ''' + content = '''# created by DirApp +logfile=/var/log/local/dirboxx.log +''' + self.buildStandardConfig(content) + + def buildUsage(self): + '''Builds the usage message. + ''' + self._usageInfo.appendDescription('''APP-NAME [] + Searching and modifying in text files. +''') + self._usageInfo.addMode('describe-rules', ''' + Describes the syntax and the meaning of the rules + ''', '''APP-NAME describe-rules +''') + self._usageInfo.addMode('extrema', '''extrema [ []] [] + Find the oldest, youngest, largest, smallest files. + : 'all' or a comma separated list of requests: o or oldest, y or youngest, l or largest, s or smallest + : the base directory for searching. Default: the current directory + ''', '''APP-NAME extrema o,y,l +APP-NAME extrema oldest,largest /home --exclude-dir=.git +''') + + self._usageInfo.addMode('list', '''list [] [] + Displays the metadata (size, date/time...) of the specified dirs/files. + : defines the files/dirs to display. Default: the current directory + ''', '''APP-NAME list +APP-NAME list *.txt --exclude-dirs=.git --file-type=fl --min-size=20k --younger-than=2020.01.30-05:00:00 +''') + self._usageInfo.addMode('adjust', '''adjust [] + Search for each file in a file with the same name in and sets the modification datetime of the 2nd file to the first. + : the modifiction datetime is taken from the file in this directory + : files in this directory will be changed (modifiction datetime) + ''', '''APP-NAME list +APP-NAME adjust Bilder Bilder/fullsize +APP-NAME adjust /tmp/pic/eastern /home/pictures/eastern/fullhd +''') + + def buildUsageOptions(self, mode=None, subMode=None): + '''Adds the options for a given mode. + @param mode: None or the mode for which the option is added + @param subMode: None or the submode for which the option is added + ''' + def add(mode, opt): + self._usageInfo.addModeOption(mode, opt) + + base.StringUtils.avoidWarning(subMode) + if mode is None: + mode = self._mainMode + if mode == 'describe-rules': + pass + elif mode == 'extrema': + base.DirTraverser.addOptions(mode, self._usageInfo) + add(mode, base.UsageInfo.Option('count', None, + 'max. count of elements in the extrema list', 'int', 5)) + add(mode, base.UsageInfo.Option('min-length', None, + 'relevant for "smallest": only file larger than will be inspected', 'int', 0)) + elif mode == 'list': + base.DirTraverser.addOptions(mode, self._usageInfo) + elif mode == 'adjust': + add(mode, base.UsageInfo.Option('pattern', 'p', + 'specifies the filenames to change, with shell wildcards: "*": any string "?": one char...', 'string', '*')) + add(mode, base.UsageInfo.Option('recursive', 'r', + 'files will be processed in subdirectories', 'bool')) + + def extrema(self): + '''Searches the "extremest" (youngest, oldest, ...) files. + ''' + what = self.shiftProgramArgument('o,y,l,s') + if what == 'all': + what = 'o,y,l,s' + directory = self.shiftProgramArgument('.') + what1 = '' + for item in what.split(','): + if item in ('o', 'y', 'l', 's', 'oldest', 'youngest', 'largest', 'smallest'): + what1 += item[0] + else: + self.argumentError( + 'unknown item in : {} use oldest, youngest, largest or smallest'.format(item)) + if self.handleOptions(): + self._traverser = base.DirTraverser.buildFromOptions( + directory, self._usageInfo, 'extrema') + count = self._optionProcessor.valueOf('count') + oldest = ExtremaList(count, 't', True) if what1.find( + 'o') >= 0 else None + youngest = ExtremaList( + count, 't', False) if what1.find('y') >= 0 else None + largest = ExtremaList(count, 's', False) if what1.find( + 'l') >= 0 else None + smallest = ExtremaList(count, 's', True) + smallest._minLength = self._optionProcessor.valueOf('min-length') + for filename in self._traverser.next(self._traverser._directory, 0): + if oldest is not None: + oldest.merge(filename, self._traverser._statInfo) + if youngest is not None: + youngest.merge(filename, self._traverser._statInfo) + if not self._traverser._isDir: + if largest is not None: + largest.merge(filename, self._traverser._statInfo) + if smallest is not None and ( + smallest._minLength is None + or self._traverser._statInfo.st_size >= smallest._minLength): + smallest.merge(filename, self._traverser._statInfo) + summary = self._traverser.summary() + self._logger.log(summary, base.Const.LEVEL_SUMMARY) + self._resultLines = summary.split('\n') + if oldest is not None: + oldest.show('the oldest files:', self._resultLines) + if smallest is not None: + smallest.show('the smallest files:', self._resultLines) + if youngest is not None: + youngest.show('the youngest files:', self._resultLines) + if largest is not None: + largest.show('the largest files:', self._resultLines) + print('\n'.join(self._resultLines)) + + def list(self): + '''Displays the meta data of the specified files/dirs. + ''' + self._resultLines = [] + directory = self.shiftProgramArgument('.') + if self.handleOptions(): + self._traverser = base.DirTraverser.buildFromOptions( + directory, self._usageInfo, 'list') + for filename in self._traverser.next(self._traverser._directory, 0): + info = base.FileHelper.listFile( + self._traverser._statInfo, filename, orderDateSize=True, humanReadable=True) + self._resultLines.append(info) + print(info) + summary = self._traverser.summary() + self._logger.log(summary, base.Const.LEVEL_SUMMARY) + self._resultLines += summary.split('\n') + + def run(self): + '''Implements the tasks of the application. + ''' + self._hostname = self._configuration.getString('hostname', '') + if self._mainMode == 'adjust': + self.adjust() + elif self._mainMode == 'extrema': + self.extrema() + elif self._mainMode == 'list': + self.list() + else: + self.abort('unknown mode: ' + self._mainMode) + + +def main(args): + '''Main function. + @param args: the program arguments + ''' + snakeboxx.startApplication() + application = DirApp(args) + application.main() + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/app/EMailApp.py b/app/EMailApp.py new file mode 100755 index 0000000..079a84d --- /dev/null +++ b/app/EMailApp.py @@ -0,0 +1,236 @@ +#! /usr/bin/python3 +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import sys +import os +import snakeboxx + +import base.StringUtils +import base.JobController +import net.EMail +import app.BaseApp + + +class EmailJobController(base.JobController.JobController): + '''Executes email tasks. + ''' + + def __init__(self, emailApp, jobDirectory, cleanInterval): + '''Constructor. + @param emailApp: the parent process + @param jobDirectory: the directory hosting the job files + @param cleanInterval: files older than this amount of seconds will be deleted + ''' + base.JobController.JobController.__init__( + self, jobDirectory, cleanInterval, emailApp._logger) + self._emailApp = emailApp + + def process(self, name, args): + '''Processes a job found in a job file. + @param name: the job name, e.g. 'send' + @param args: the job arguments, e.g. ['a@bc.de', 'subject', 'body'] + @return: True: success + ''' + rc = self._emailApp.process(name, args) + return rc + + +class EMailApp(app.BaseApp.BaseApp): + '''Sends an email. + ''' + + def __init__(self, args): + '''Constructor. + @param args: the program arguments, e.g. ['test', 'a@bc.de'] + ''' + app.BaseApp.BaseApp.__init__(self, 'EMailApp', args, True) + self._daemonJobController = None + + def buildConfig(self): + '''Creates an useful configuration example. + ''' + content = '''# created by EMailApp +smtp.host=smtp.gmx.de +smtp.port=587 +smtp.user=hm.neutral@gmx.de +smtp.code=TopSecret +smtp.with.tls=True +sender=hm.neutral@gmx.de +# jobs should be written to this dir: +job.directory=/tmp/emailboxx/jobs +# files older than this amount of seconds will be deleted (in job.directory): +job.clean.interval=3600 +''' + self.buildStandardConfig(content) + + def buildUsage(self): + '''Builds the usage message. + ''' + self._usageInfo.appendDescription('''APP-NAME [] + Offers email services. +''') + self._usageInfo.addMode('send', '''send [] + : the email address of the receipient. Separate more than one with '+' + : the text of the subject field + : the body of the email or (if preceeded by '@' a filename with the body. +''', '''APP-NAME send --carbon-copy=jonny@x.de eva@x.com+adam@x.fr "Greetings" "Hi guys" +APP-NAME send --blind-carbon-copy=jonny@x.de+joe@x.de eva@x.com+adam@x.fr "Greetings" @birthday.html +''') + + def buildUsageOptions(self, mode=None, subMode=None): + '''Adds the options for a given mode. + @param mode: None or the mode for which the option is added + @param subMode: None or the submode for which the option is added + ''' + def add(mode, opt): + self._usageInfo.addModeOption(mode, opt) + + base.StringUtils.avoidWarning(subMode) + if mode is None: + mode = self._mainMode + if mode == 'send': + add(mode, base.UsageInfo.Option('carbon-copy', 'c', + 'additional recipient(s) seen by all recipients. Separate more than one with "+"')) + add(mode, base.UsageInfo.Option('blind-carbon-copy', 'b', + 'additional recipient(s) not seen by all recipients. Separate more than one with "+"')) + + def daemonAction(self, reloadRequest): + '''Does the real thing in the daemon (= service). + @param reloadRequest: True: a reload request has been done + ''' + if reloadRequest or self._daemonJobController is None: + jobDir = self._configuration.getString('job.directory') + cleanInterval = self._configuration.getInt('job.clean.interval') + self._daemonJobController = EmailJobController( + self, jobDir, cleanInterval) + self._daemonJobController.check() + + def install(self): + '''Installs the application and the related service. + ''' + app.BaseApp.BaseApp.install(self) + self.createSystemDScript( + 'emailboxx', 'emailboxx', 'emailboxx', 'emailboxx', 'Offers an email send service') + self.installAsService('emailboxx', True) + + def process(self, name, args): + '''Processes a job found in a job file. + @param name: the job name, e.g. 'send' + @param args: the job arguments, e.g. ['a@bc.de', 'subject', 'body'] + @return: True: success + ''' + rc = False + if name == 'send': + args2 = [] + options = [] + for arg in args: + if arg.startwith('-'): + options.append(arg) + else: + args2.append(arg) + self.send(args2, options) + rc = True + elif name == 'test': + recipient = 'test@hamatoma.de' if not args else args[0] + self._logger.log('job "test" recogniced: ' + + ' '.join(args), base.Const.LEVEL_SUMMARY) + host = self._configuration.getString('smtp.host') + sender = self._configuration.getString('smtp.sender') + port = self._configuration.getInt('smtp.port') + user = self._configuration.getString('smtp.user') + sender = self._configuration.getString('smtp.sender', user) + code = self._configuration.getString('smtp.code') + withTls = self._configuration.getBool('smtp.with.tls') + net.EMail.sendSimpleEMail(recipient, 'Test EMailApp Daemon', 'it works', sender, host, + port, user, code, withTls, self._logger) + rc = True + else: + self._logger.error() + return rc + + def run(self): + '''Implements the tasks of the application + ''' + if self._mainMode == 'send': + self.send() + elif self._mainMode == 'daemon': + self.daemon() + else: + self.abort('unknown mode: ' + self._mainMode) + + def send(self, args=None, options=None): + '''Sends an email. + ''' + cc = [] + bcc = [] + if args is None: + args = self._programArguments + if options is None: + options = self._programOptions + stopped = False + for opt in options: + if opt.startswith('--cc='): + cc.append(opt[5:]) + elif opt.startswith('-c'): + cc.append(opt[2:]) + elif opt.startswith('--bcc='): + bcc.append(opt[6:]) + elif opt.startswith('-b'): + bcc.append(opt[2:]) + else: + self.abort('unknown option: ' + opt) + stopped = True + break + cc = None if not cc else '+'.join(cc) + bcc = None if not bcc else '+'.join(bcc) + if not stopped: + if len(args) < 3: + self.abort('too few arguments') + else: + receipient = args[0] + subject = args[1] + body = args[2] + if body.startswith('@'): + fn = body[1:] + if not os.path.exists(fn): + self.abort('body file not found: ' + fn) + stopped = True + else: + body = base.StringUtils.fromFile(body) + if not stopped: + isHtml = body.startswith('<') + if isHtml: + text = None + html = body + else: + text = body + html = None + email = net.EMail.EMail(subject, text, html) + host = self._configuration.getString('smtp.host') + port = self._configuration.getInt('smtp.port') + username = self._configuration.getString('smtp.user') + code = self._configuration.getString('smtp.code') + withTls = self._configuration.getBool('smtp.with.tls') + sender = self._configuration.getString('sender', username) + if host is None or port is None or username is None or code is None: + self.abort('missing email configuration') + else: + email.setSmtpLogin( + host, port, username, code, withTls, sender) + email.sendTo(receipient, cc, bcc, self._logger) + + +def main(args): + '''Main function. + @param args: the program arguments + ''' + snakeboxx.startApplication() + application = EMailApp(args) + application.main() + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/app/OperatingSystemApp.py b/app/OperatingSystemApp.py new file mode 100755 index 0000000..dbbeab7 --- /dev/null +++ b/app/OperatingSystemApp.py @@ -0,0 +1,628 @@ +#! /usr/bin/python3 +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import sys +import os.path +import re +import time +import datetime +# see snake_install.sh +import snakeboxx + + +import base.Const +import base.FileHelper +import base.TextProcessor +import app.BaseApp +from base import TextProcessor + + +class OperatingSystemApp(app.BaseApp.BaseApp): + '''Services for operating systems. + ''' + + def __init__(self, args): + '''Constructor. + @param args: the program arguments, e.g. ['test', 'a@bc.de'] + ''' + app.BaseApp.BaseApp.__init__( + self, 'OperatingSystemApp', args, progName='osboxx') + self._textProcessor = base.TextProcessor.TextProcessor(self._logger) + self._textProcessor.readFile('/etc/os-release') + self._osName = self._textProcessor.searchByGroup(r'ID=\W*([a-z]+)', 1) + self._osCodeName = self._textProcessor.searchByGroup( + r'VERSION_CODENAME=\W*([a-z]+)', 1) + self._ubuntuVersion = None + self._debianVersion = None + version = int(self._textProcessor.searchByGroup( + r'VERSION_ID=\D*(\d+)', 1)) + if self._osName == 'ubuntu': + self._ubuntuVersion = version + elif self._osName == 'debian': + self._debianVersion = version + + def authKeys(self): + '''Handles the command authKeys: put public keys into a 'authorized-keys' file. + ''' + def makeMap(lines, regExpr): + rc = {} + for line in lines: + matcher = regExpr.search(line) + if matcher is not None: + groupNo = len(matcher.groups()) + key = matcher.group(groupNo) + rc[key] = line + return rc + user = self.shiftProgramArgument() + fnKeys = self.shiftProgramArgument('/tmp/public.keys') + home = '' if user is None else self.createPath('/home', user) + if user is None: + self.argumentError('missing ') + elif not os.path.exists(fnKeys): + self.argumentError(' not found: ' + fnKeys) + elif not os.path.isdir(home): + self.argumentError(f'missing home: {home}') + elif self.handleOptions(): + regFilter = self._optionProcessor.valueOf('filter') + publicLines = base.StringUtils.fromFile(fnKeys, '\n') + baseSSH = home + '/.ssh' + base.FileHelper.ensureDirectory(baseSSH, 0o700, base.LinuxUtils.userId(user, -1), + base.LinuxUtils.groupId(user, -1)) + fnAuth = baseSSH + '/authorized_keys' + authLines = [] if not os.path.exists( + fnAuth) else base.StringUtils.fromFile(fnAuth, '\n') + regExprKey = re.compile(r'ssh-rsa (\S+) ') + mapAuthKeys = makeMap(authLines, regExprKey) + filterLines = publicLines if regFilter is None else [ + x for x in makeMap(publicLines, regFilter).values()] + filterKeys = makeMap(filterLines, regExprKey) + regExprLabel = re.compile(r' \S+@\S+') + for key in filterKeys: + if key not in mapAuthKeys: + line = filterKeys[key] + label = (regExprLabel.search(line).group(0) + if regExprLabel.search(line) is not None else key[1:8] + '...' + key[-12:]) + self._logger.log('adding ' + label, + base.Const.LEVEL_SUMMARY) + authLines.append(line) + base.StringUtils.toFile(fnAuth, authLines, '\n', fileMode=0o700, + user=base.LinuxUtils.userId(user, -1), group=base.LinuxUtils.groupId(user, -1)) + + def buildConfig(self): + '''Creates an useful configuration example. + ''' + content = '''# created by OperatingSystemApp +logfile=/var/log/local/{}.log +php.upload_max_filesize=1G +php.max_file_uploads=102 +php.post_max_size=512M +php.max_execution_time=900 +php.max_input_time=630 +'''.format(self._programName) + self.buildStandardConfig(content) + + def buildUsage(self): + '''Builds the usage message. + ''' + def installOne(mode, subMode, description): + self._usageInfo.addSubMode(mode, subMode, f'''{mode} {subMode} + {description} +''', '''APP_NAME {mode} {subMode} +''') + + self._usageInfo.appendDescription('''APP-NAME [] + Offers service round about the operating system. +''') + self._usageInfo.addMode('auth-keys', '''auth-keys [] + put some public keys taken from a file into the .ssh/authorized-keys. + : the username: for that user the auth-keys will be modified + : a text file with the known public keys. Default: /tmp/public.keys +''', '''APP-NAME -v3 auth-keys bupsupply /home/data/my.public.keys -f(^ext|@caribou) +APP-NAME -v3 auth-keys bupsrv --pattern=@caribou +''') + self._usageInfo.addMode('create-user', '''create-user + creates one or more standard user (for administration). + : bupsrv bupsupply bupwiki extbup extcave extcloud extdata exttmp extcmd + std: does the job for bupsupply extbup and exttmp +''', '''APP-NAME create-user exttmp +APP-NAME create-user std +''') + + self._usageInfo.addMode('php-reconfigure', '''php-reconfigure [] + Sets some values in the configuration file php.ini. + : the PHP version to reconfigure e.g. "7.3". Default: all installed versions +''', '''APP-NAME create-user exttmp +APP-NAME create-user std +''') + mode = 'init' + self._usageInfo.initializeSubModes(mode, '''init + Initializes a "sub system" defined as +''') + installOne( + mode, 'cms', 'installs packages relevant for content management systems: imagemagick redis-server...') + installOne( + mode, 'etc', 'prepares configuration data in /etc for usage as linuxserver') + installOne(mode, 'dirs', 'creates needed directories') + installOne(mode, 'linuxserver', + 'prepares the system to be a standard linux server') + installOne(mode, 'local-bin', + 'initializes the directory /usr/local/bin with standard scripts') + self._usageInfo.addSubMode(mode, 'php', '''init php [] + initializes the PHP programming environment +''', '''APP_INFO init php 7.4 +''') + installOne(mode, 'nginx', 'initializes the NGINX webserver') + installOne(mode, 'mariadb', 'initializes the mariadb database engine') + + def buildUsageOptions(self, mode=None, subMode=None): + '''Adds the options for a given mode. + @param mode: None or the mode for which the option is added + @param subMode: None or the submode for which the option is added + ''' + def add(mode, opt): + self._usageInfo.addModeOption(mode, opt) + + def add2(mode, subMode, opt): + self._usageInfo.addSubModeOption(mode, subMode, opt) + + if mode is None: + mode = self._mainMode + if mode == 'auth-keys': + add(mode, base.UsageInfo.Option('filter', 'f', + ''''only lines with a label in ''' + + ''' matching this regular expression will be respected +the label of the line is the last part with the form @''', 'regexpr')) + elif mode == 'create-user': + self._usageInfo.addModeOption(mode) + elif mode == 'init': + if subMode == 'cms': + add2(mode, subMode, None) + elif subMode == 'apache': + add2(mode, subMode, base.UsageInfo.Option('ports', 'p', + '''the ports for listening. Default: 80,443 +''', 'string', '80,443')) + + def createUser(self): + '''Create a special os user. + ''' + user = self.shiftProgramArgument() + uid = None if user is None else base.LinuxUtils.userId(user) + if not self._isRoot and not app.BaseApp.BaseApp.__underTest: + self.abort('be root!') + elif user is None: + self.argumentError('too few arguments: ') + elif uid is not None: + self._logger.log('user {} already exists: uid={}'.format( + user, uid), base.Const.LEVEL_SUMMARY) + elif self.handleOptions(): + users = [user] + if user == 'std': + users = ['bupsupply', 'extbup', 'exttmp'] + elif not re.match(r'bupsrv|bupsupply|bupwiki|extbup|extcave|extcloud|extdata|exttmp', user): + self.argumentError('unknown ') + else: + ids = {'bupsrv': 201, 'bupsupply': 203, 'bupwiki': 205, 'extbup': 211, 'extcave': 212, + 'extcloud': 213, 'extdata': 214, 'exttmp': 215, 'extcmd': 216 } + for user in users: + uid = str(ids[user]) + gid = uid + self._logger.log('creating user...', + base.Const.LEVEL_SUMMARY) + self._processHelper.execute( + ['/usr/sbin/groupadd', '-g', gid, user], True) + self._logger.log('creating user...', + base.Const.LEVEL_SUMMARY) + self._processHelper.execute(['/usr/sbin/useradd', '-s', '/usr/sbin/nologin', + '-d', '/home/' + user, '-u', uid, '-g', gid, user], True) + base.FileHelper.ensureDirectory('/home/' + user, 0o750) + base.FileHelper.ensureDirectory( + '/home/{}/.ssh'.format(user), 0o700) + if user.startswith('ext'): + self._logger.log( + 'Please leave the password empty...', base.Const.LEVEL_SUMMARY) + self._processHelper.execute(['/usr/bin/ssh-keygen', '-t', 'rsa', '-b', '4096', + '-f', '/home/{}/.ssh/id_rsa'.format(user), '-N', ''], True) + self._processHelper.execute( + ['/usr/bin/chown', '-R', '{}.{}'.format(user, user), '/home/' + user], True) + + def init(self, what=None): + '''Handles the mode 'init'. + @param what: None: use sub mode Otherwise: the sub mode + ''' + if what is None: + what = self.getSubMode() + if what == 'etc': + self._logger.log('setting vm.swappiness...') + fn = self.createPath('/etc', 'sysctl.conf') + self._textProcessor.simpleInsertOrReplace( + fn, '^vm.swappiness', 'vm.swappiness = 20', 'swappiness') + elif what == 'dirs': + base.FileHelper.ensureDirectory(self.createPath('/media', 'tmp')) + base.FileHelper.ensureDirectory(self.createPath('/media', 'trg')) + base.FileHelper.ensureDirectory(self.createPath('/media', 'src')) + base.FileHelper.ensureDirectory( + self.createPath('/var/log', 'local')) + base.FileHelper.ensureDirectory( + self.createPath('/var/cache', 'local')) + elif what == 'linuxserver': + self.initLinuxServer() + elif what == 'local-bin': + self.initLocalBin() + elif what == 'apache': + self.initApache() + elif what == 'nginx': + self.initNginx() + elif what == 'php': + self.initPhp() + elif what == 'mariadb': + self.initMariaDb() + elif what == 'cms': + self.updatePackages( + 'imagemagick php-redis redis-tools redis-server') + elif what == 'letsencrypt': + self.initLetsencrypt() + elif what == 'grub': + self.initGrub() + else: + self.abort(f'unknown : {what}') + + def initApache(self): + '''Initializes the OS for usage of the Apache webserver.' + ''' + raise NotImplementedError('init apache') +# ports = self._optionProcessor.valueOf('ports') +# if re.match(r'^\d+,\d+$', ports) is None: +# self.abort(f'not 2 comma separated ports in: {ports}') +# else: +# parts = ports.split(',') +# ports = [int(parts[0]), int(parts[1])] +# if self._isRoot: +# self.updatePackages('apache2 libapache2-mod-php libapache2-mod-php5.6 libapache2-mod-php7.0 libapache2-mod-php7.1 libapache2-mod-php7.2 libapache2-mod-php7.3') +# service = 'apache2' +# fn = self.createPath('/etc/apache2', 'ports.conf') +# self._textProcessor.readFile(fn, True) +# (start, end) = self._textProcessor.findRegion('^', True, '^<', False) +# if start >= 0: +# self._textProcessor.currentReplace( +# r'Listen\s', 'Listen {}'.format(ports[0]), None, False, start, end) +# (start, end) = self._textProcessor.findRegion( +# '', True, '= 0: +# self._textProcessor.currentReplace( +# r'\s*Listen\s', '\tListen {}'.format(ports[1]), None, False, start, end) +# (start, end) = self._textProcessor.findRegion( +# '', True, '= 0: +# self._textProcessor.currentReplace( +# r'\s*Listen\s', '\tListen {}'.format(ports[1]), None, False, start, end) +# self._textProcessor.writeFile() +# if self._isRoot: +# self._processHelper.execute(['a2enmod', 'rewrite', service], True) +# self._processHelper.execute(['systemctl', 'enable', service], True) +# self._processHelper.execute(['systemctl', 'start', service], True) +# self._processHelper.execute(['systemctl', 'status', service], True) + + def initGrub(self): + '''Initializes the OS for usage of the grub boot system.' + ''' + raise NotImplementedError('init grub') + + def initLetsencrypt(self): + '''Initializes the OS for usage of some services.' + ''' + raise NotImplementedError('init letsencrypt') + + def initLinuxServer(self): + '''Initializes the OS for usage of some typical services.' + ''' + self.init('etc') + self.init('dirs') + self.updatePackages('htop iotop curl tmux git etckeeper bzip2 zip unzip nfs-common nfs-kernel-server' + + ' nmap rsync sudo net-tools ntp btrfs-compsize wget') + self.createSystemDScript('bootboxx', 'bootboxx', 'root', 'root', + 'Starts shell scripts from /etc/snakeboxx/boot.d at boot time.') + bootDir = self.createPath('/etc/snakeboxx', 'boot.d') + base.FileHelper.ensureDirectory(bootDir) + script = self.createPath('/usr/local/bin', 'bootboxx') + base.StringUtils.toFile(script, f'''#! /bin/bash +DIR={bootDir} +LOG=/var/log/local/boot.log +export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin +cd $DIR +for script in *.sh; do + if [ "$script" != "*.sh" ]; then + date "+%Y.%m.%d-%H:%M:%S $script" >>$LOG + ./$script $* + fi +done +date "+%Y.%m.%d-%H:%M:%S" >>$LOG +systemctl stop bootboxx +exit 0 +''', fileMode=0o755) + self.enableAndStartService('bootboxx') + currentZone = base.StringUtils.fromFile('/etc/timezone').strip() + if currentZone != 'Europe/Berlin': + self._logger.log( + f'=== Curious timezone: {currentZone} Please execute: dpkg-reconfigure tzdata', base.Const.LEVEL_SUMMARY) + + def initLocalBin(self): + '''Initializes /usr/local/bin directory.' + ''' + tarNode = 'local_bin.tgz' + tar = '/tmp/' + tarNode + url = self._configuration.getString( + 'url.download', 'https://public.hamatoma.de') + self._processHelper.execute( + ['/usr/bin/wget', '-O', tar, url + '/' + tarNode], True) + self.restoreDirectoryByTar( + tar, '/usr/local/bin', None, True, None, False) + + def initNginx(self): + '''Initializes the OS for usage of the NGINX webserver.' + ''' + # if opt.startswith('--well-known='): + self.updatePackages('nginx-full ssl-cert ca-certificates') + fn = self.createPath('/etc/conf.d', 'gzip.conf') + if os.path.exists(fn): + self._logger.log(f'already exists: {fn}') + else: + base.StringUtils.toFile(fn, '''http { + gzip on; + gzip_disable "msie6"; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_min_length 256; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; +} +''') + fn = self.createPath('/etc/snippets', 'letsencrypt.conf') + if os.path.exists(fn): + self._logger.log(f'already exists: {fn}') + else: + base.StringUtils.toFile(fn, '''location ^~ /.well-known/acme-challenge { + default_type "text/plain"; + # Do NOT use alias, use root! Target directory is located here: + # /var/www/common/letsencrypt/.well-known/acme-challenge/ + root /var/www/letsencrypt; +} +# Hide /acme-challenge subdirectory and return 404 on all requests. +# It is somewhat more secure than letting Nginx return 403. +# Ending slash is important! +location = /.well-known/acme-challenge/ { + return 404; +} +''') + self.enableAndStartService('nginx') + + def initMariaDb(self): + '''Initializes the OS for usage of the mariadb database.' + ''' + self.updatePackages('mariadb-common mariadb-server mariadb-client') + baseDir = '/var/lib/mysql' + if not os.path.isdir(baseDir): + self._logger.log(f'is not a directory: {baseDir}') + else: + mountDir, fsType = base.FileHelper.mountPointOf(baseDir) + self._logger.log( + f'mountpoint/type of {baseDir}: {mountDir}/{fsType}', base.Const.LEVEL_DETAIL) + if fsType == 'btrfs': + self._processHelper.execute( + ['/bin/systemctl', 'stop', 'mysqld'], True) + if base.FileHelper.extendedAttributesOf(baseDir).find('C') >= 0: + self._logger.log(f'already no copy on write: {baseDir}') + else: + self._logger.log( + f'changing to no copy on write: {baseDir}...') + dir2 = f'{baseDir}.nowcow.{time.time()}' + base.FileHelper.ensureDirectory( + dir2, 0o755, 'mysql', 'mysql') + base.FileHelper.changeExtendedAttributes( + dir2, toAdd='C', toDelete='c') + base.FileHelper.copyDirectory(baseDir, dir2) + dir3 = f'{baseDir}.org.{time.time()}' + self._logger.log( + f'renaming {baseDir} -> {dir3}', base.Const.LEVEL_DETAIL) + os.rename(baseDir, dir3) + self._logger.log( + f'renaming {dir2} -> {baseDir}', base.Const.LEVEL_DETAIL) + os.rename(dir2, baseDir) + self.enableAndStartService('mysql') + + def initPhp(self): + '''Initializes the OS for usage of a PHP server.' + ''' + phpVersion = self.shiftProgramArgument() + packets = 'php-fpm php-cli php-json php-curl php-imap php-gd php-mysql php-xml php-zip php-intl php-imagick ' + packets += 'php-mbstring php-memcached memcached php-xdebug php-igbinary php-msgpack php-dom php-readline php-sqlite3' + if phpVersion != None: + if self._ubuntuVersion is not None: + self._processHelper.execute( + ['add-apt-repository', '-y', 'ppa:ondrej/php'], True) + self.updateApt(True) + elif self._debianVersion is not None: + if not os.path.exists('/etc/apt/sources.list.d/php.list'): + self._processHelper.executeScript(f'''#! /bin/bash +wget -q https://packages.sury.org/php/apt.gpg -O- | sudo apt-key add - +echo "deb https://packages.sury.org/php/ {self._osCodeName} main" | sudo tee /etc/apt/sources.list.d/php.list +''') + self.updateApt(True) + else: + self._logger.error( + f'not DEBIAN, not Ubuntu. I am confused: {self._osCodeName}') + return + packets = packets.replace('-', str(phpVersion) + '-') + if phpVersion == '5.6': + packets += ' php5.6-mcrypt php5.6-opcache php5.6-sqlite3' + args = ['/usr/bin/apt-get', '-y', 'install'] + packets.split(' ') + self._processHelper.execute(args, True) + service = 'php{}-fpm'.format('' if phpVersion == None else phpVersion) + self.enableAndStartService(service) + + def logFile(self, filename, messagePattern, start): + '''Writes metadata file size and runtime into the log. + @param filename: file to log + @param messagePattern: log message with makros %f (filename) %s (filesize) %t (filetime) and %r (runtime) + @param start: None or the start of the operation (for calculating runtime) + @return: the message (already written to log) + ''' + stat = os.stat(filename) + size = base.StringUtils.formatSize(stat.st_size) + fdate = datetime.datetime.fromtimestamp(stat.st_mtime) + dateString = fdate.strftime("%Y.%m.%d %H:%M:%S") + runtime = '?' if start == None else '{:d} sec'.format( + int(time.time() - start)) + msg = messagePattern.replace('%f', filename).replace( + '%s', size).replace('%t', dateString).replace('%r', runtime) + self._logger.log(msg) + return msg + + def phpReconfigure(self): + '''Reconfigures PHP configuration. + Sets some values in the php.ini. + ''' + version = self.shiftProgramArgument() + if not self._isRoot and not app.BaseApp.BaseApp.__underTest: + self.abort('be root!') + else: + source = '/etc/php/{}/php.ini'.format(version) + textProcessor = TextProcessor(self._logger) + textProcessor.readFile(source) + variables = self._textTool.findVariables('php.', self._configuration) + if version == None or version == 'all': + versions = os.listdir('/etc/php') + versions.sort() + for version in versions: + self.phpReconfigureOne(version) + else: + if not re.match(r'^\d+\.\d+$', version): + self.usage('invalid version: ' + version) + else: + self.phpReconfigureOne(version) + + def phpReconfigureOne(self, version): + fnConfig = '/etc/php/{}/fpm/php.ini'.format(version) + if not os.path.exists(fnConfig): + self.usage('missing {}: is version {} installed?'.format(fnConfig, version)) + else: + nodes = os.listdir('/etc/php/{}'.format(version)) + for node in nodes: + fnConfig = '/etc/php/{}/{}/php.ini'.format(version, node) + if os.path.exists(fnConfig): + missingDebug = False + configuration = base.StringUtils.fromFile(fnConfig).split('\n') + if not base.StringUtils.arrayContains(configuration, 'xdebug.remote_enabled'): + missingDebug = True + configuration += ''' +; Ist normalerweise in xdebug-spezifischer Konfiguration, z.B. mods.d/20-xdebug +;zend_extension="/usr/lib/php/20160303/xdebug.so" +xdebug.remote_port=9000 +xdebug.remote_enable=Off +xdebug.remote_handler=dbgp +xdebug.remote_host=127.0.0.1 +;xdebug.remote_connect_back=On +;xdebug.remote_log=/var/log/xdebug.log +xdebug.remote_autostart=1 +'''.split('\n') + content = '\n'.join(self._textTool.adaptVariables(variables, configuration)) + if missingDebug or self._textTool._hits > 0: + self.createBackup(fnConfig) + base.StringUtils.toFile(fnConfig, content) + if self._verboseLevel >= 2: + self._logger.log('{}: {} variable(s) changed{}'.format( + fnConfig, self._textTool._hits, + '' if not missingDebug else '\nxdebug setup added')) + + def restoreDirectoryByTar(self, archive, target, opts=None, tempDir=False, subdir=None, clearTarget=True): + '''Restores a directory from a tar archive. + @param archive: archive name + @param target: target directory + @param opt: None or an array of options like '-exclude=' + @param tempDir: True: usage of a temporary directory + @param subdir: None or a subdirectory in the TAR archive + @param clearTarget: True: the target directory will be cleared before extracting + ''' + if not os.path.exists(archive): + self._logger.error('missing tar archive {}'.format(archive)) + elif not os.path.exists(target): + self._logger.error('missing target dir: ' + target) + elif not os.path.isdir(target): + self._logger.error('not a directory: ' + target) + else: + start = time.time() + baseDir = '/tmp/restoretool' + tempBase = base.FileHelper.ensureDirectory(baseDir) + if clearTarget: + self.clearDirectory(target) + trg = target if not tempBase else os.path.join(baseDir, 'trg') + if tempDir: + base.FileHelper.ensureDirectory(trg) + argv = ['/bin/tar', 'xzf', archive, '--directory=' + trg] + if subdir != None: + argv.append('./' + subdir) + if opts != None: + argv += opts + self._processHelper.execute( + argv, self._logger._verboseLevel >= base.Const.LEVEL_LOOP) + # os.chdir(oldDir) + if tempDir: + if subdir != None: + trg += os.sep + subdir + self._processHelper.execute( + ['/usr/bin/rsync', '-a', trg + '/', target], self._logger._verboseLevel >= base.Const.LEVEL_LOOP) + self.logFile(archive, '%f: %s %t restored in %r', start) + + def run(self): + '''Implements the tasks of the application + ''' + if self._mainMode == 'create-user': + self.createUser() + elif self._mainMode == 'auth-keys': + self.authKeys() + elif self._mainMode == 'php-reconfigure': + self.phpReconfigure() + elif self._mainMode == 'init': + self.init() + else: + self.abort( + f'unknown mode: {self._mainMode}' if self._mainMode is not None else 'missing mode') + + def updateApt(self, force=False): + '''Tests whether the last "apt update" command is younger than one day. + If not the command is executed. + @param force: True: the marker file will be removed: apt-get update is executed always + ''' + fnLastUpdate = base.FileHelper.tempFile('.last.apt.update') + if force or not os.path.exists(fnLastUpdate) or os.stat(fnLastUpdate).st_mtime < time.time() - 86400: + base.StringUtils.toFile(fnLastUpdate, '') + self._processHelper.execute( + ['/usr/bin/apt-get', 'update'], True) + + def updatePackages(self, packages): + '''Installs or updates debian packages. + @param packages: a blank separated list of package names, e.g. 'nginx apache2' + ''' + if not self._isRoot: + self._logger.log(f'+++ not root: ignoring update of {packages}') + else: + self.updateApt() + packets = packages.split(' ') + self._processHelper.execute( + ['/usr/bin/apt-get', '-y', 'install'] + packets, True) + + +def main(args): + '''Main function. + @param args: the program arguments + ''' + snakeboxx.startApplication() + application = OperatingSystemApp(args) + application.main() + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/app/SatelliteApp.py b/app/SatelliteApp.py new file mode 100755 index 0000000..e0c417e --- /dev/null +++ b/app/SatelliteApp.py @@ -0,0 +1,534 @@ +#! /usr/bin/python3 +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import sys +import time +import os.path +import re +# see snake_install.sh +import snakeboxx + +import base.Const +import base.Scheduler +import base.FileHelper +import net.HttpClient +import app.BaseApp + + +class CloudTaskInfo(base.Scheduler.TaskInfo): + '''Executes a task for the satellite. + ''' + + def __init__(self, cloud, application): + '''Constructor. + @param cloud: the full path of the cloud + @param application: the calling parent + ''' + self._application = application + self._cloud = cloud + + def process(self, sliceInfo): + '''Executes the task. + @param sliceInfo: the entry of the scheduler list + @return True: success + ''' + base.StringUtils.avoidWarning(sliceInfo) + self._application.sendCloudInfo(self._cloud) + + +class FilesystemTaskInfo(base.Scheduler.TaskInfo): + '''Executes a task for the satellite. + ''' + + def __init__(self, filesystem, application): + '''Constructor. + @param filesystem: the mount path of the filesystem + @param application: the calling parent + ''' + self._application = application + self._filesystem = filesystem + + def process(self, sliceInfo): + '''Executes the task. + @param sliceInfo: the entry of the scheduler list + @return True: success + ''' + base.StringUtils.avoidWarning(sliceInfo) + self._application.sendFilesystemInfo(self._filesystem) + + +class SatelliteApp(app.BaseApp.BaseApp): + '''REST client for WebDashFiller. + ''' + + def __init__(self, args): + '''Constructor. + @param args: the program arguments, e.g. ['test', 'a@bc.de'] + ''' + app.BaseApp.BaseApp.__init__( + self, 'SatelliteApp', args, isService=True, progName='satboxx') + self._clouds = None + self._client = None + self._scheduler = None + self._webDashServer = None + self._countClouds = 0 + self._filesystems = None + self._countFilesystems = 0 + self._mapFilesystems = None + self._rexprExcludedFilesystem = None + self._rexprExcludedClouds = None + self._stopAtOnce = False + self._hostname = None + + def buildConfig(self): + '''Creates an useful configuration example. + ''' + content = '''# created by SatelliteApp +logfile=/var/log/local/satboxx.log +wdfiller.active=true +wdfiller.url=http://localhost:8080 +# separator ' ': possible modes cloud filesystem stress +wdfiller.kinds=cloud +# interval between 2 send actions in seconds +wdfiller.cloud.interval=600 +wdfiller.cloud.data.directory=/opt/clouds +wdfiller.cloud.main.directory=/opt/clouds +wdfiller.cloud.excluded=cloud.test|cloud.huber.de +wdfiller.filesystem.interval=600 +wdfiller.filesystem.excluded=/mnt/|/media/usb +#wdfiller.filesystem.map=/:root,/tmp/fs.system:fs.system +#wdfiller.filesystem.map= +wdfiller.stress.interval=120 +hostname=caribou +''' + self.buildStandardConfig(content) + + def buildUsage(self): + '''Builds the usage message. + ''' + self._usageInfo.appendDescription('''APP-NAME [] + Sends data via REST to some servers. Possible servers are WebDashFiller and Monitor. +''') + self._usageInfo.addMode('test', '''test [ []] + : 'cloud', 'filesystem' or 'stress' + : number of send actions, default: 1 + : time between two send actions in seconds, default: 1 +''', '''APP-NAME test cloud 3 5 +''') + del self._usageInfo._descriptions['daemon'] + self._usageInfo.addMode('daemon', '''daemon [] + : the name of the SystemD service +''', '''APP-NAME daemon satbutt --count=1 --interval=2 +''') + + def buildUsageOptions(self, mode=None, subMode=None): + '''Adds the options for a given mode. + @param mode: None or the mode for which the option is added + ''' + def add(mode, opt): + self._usageInfo.addModeOption(mode, opt) + + base.StringUtils.avoidWarning(subMode) + if mode is None: + mode = self._mainMode + if mode == 'test': + self._usageInfo.addModeOption(mode) + elif mode == 'daemon': + add(mode, base.UsageInfo.Option('count', 'c', + 'number of rounds: in one round all cloud state infos will be sent default: forever', 'int')) + add(mode, base.UsageInfo.Option('interval', 'i', + 'time (seconds) between two send rounds: default: see configuration file', 'int')) + + def cloudInit(self, startAtOnce=False, count=None, interval=None): + '''Initializes the cloud sending actions. + For each cloud: an task to send the cloud info will be inserted into the scheduler list. + @param startAtOnce: True: the cloud infos will be sent at once too + @param count: None or the number of rounds + @param interval: the time of a round in seconds + ''' + self.prepareClouds() + interval = self._configuration.getInt( + 'wdfiller.cloud.interval', 600) if interval is None else interval + self._logger.log('cloudInit: interval: {}'.format( + interval), base.Const.LEVEL_SUMMARY) + no = 0 + while True: + no += 1 + cloud = self.nextCloud() + if cloud is None: + break + taskInfo = CloudTaskInfo(cloud, self) + sliceInfo = base.Scheduler.SliceInfo( + taskInfo, self._scheduler, count, interval, 0.1) + self._scheduler.insertSlice(sliceInfo, int( + interval * no / self._countClouds), 0.0) + if startAtOnce: + taskInfo.process(sliceInfo) + timestamp = time.strftime( + '%H:%M:%S', time.localtime(sliceInfo._nextCall)) + self._logger.log('cloud: {} start: {}'.format( + cloud, timestamp), base.Const.LEVEL_LOOP) + + def filesystemInit(self, startAtOnce=False, count=None, interval=None): + '''Initializes the filesystem sending actions. + For each cloud: an task to send the cloud info will be inserted into the scheduler list. + @param startAtOnce: True: the cloud infos will be sent at once too + @param count: None or the number of rounds + @param interval: the time of a round in seconds + ''' + self.prepareFilesystems() + interval = self._configuration.getInt( + 'wdfiller.filesystem.interval', 600) if interval is None else interval + self._logger.log('filesystemInit: interval: {}'.format( + interval), base.Const.LEVEL_SUMMARY) + no = 0 + while True: + no += 1 + fs = self.nextFilesystem() + if fs is None: + break + taskInfo = FilesystemTaskInfo(fs, self) + sliceInfo = base.Scheduler.SliceInfo( + taskInfo, self._scheduler, count, interval, 0.1) + self._scheduler.insertSlice(sliceInfo, int( + interval * no / self._countFilesystems), 0.0) + if startAtOnce: + taskInfo.process(sliceInfo) + timestamp = time.strftime( + '%H:%M:%S', time.localtime(sliceInfo._nextCall)) + self._logger.log('filesystem: {} start: {}'.format( + fs, timestamp), base.Const.LEVEL_LOOP) + + def daemon(self): + '''Runs the service, a never ending loop, processing the time controlled scheduler. + ''' + serviceName = self.shiftProgramArgument() + if serviceName is None: + self.abort('missing ') + elif self.handleOptions(): + base.FileHelper.ensureDirectory( + os.path.dirname(self.reloadRequestFile(serviceName))) + self._scheduler = base.Scheduler.Scheduler(self._logger) + self._webDashServer = self._configuration.getString( + 'wdfiller.url', 'http://localhost') + self._client = net.HttpClient.HttpClient(self._logger) + count = self._optionProcessor.valueOf('count') + interval = self._optionProcessor.valueOf('interval') + kinds = self._configuration.getString('wdfiller.kinds', '') + if kinds.find('cloud') >= 0: + self.cloudInit(True, count, interval) + if kinds.find('filesystem') >= 0: + self.filesystemInit(True, count, interval) + self._stopAtOnce = False + fileReloadRequest = self.reloadRequestFile(serviceName) + while not self._stopAtOnce: + hasRequest = fileReloadRequest is not None and os.path.exists( + fileReloadRequest) + if hasRequest: + if not self.handleReloadRequest(fileReloadRequest): + fileReloadRequest = None + self._scheduler.checkAndProcess() + if not self._scheduler._slices: + self._logger.log( + 'daemon: empty list: stopped', base.Const.LEVEL_SUMMARY) + self._stopAtOnce = True + time.sleep(1) + self._logger.log('daemon regulary stopped', + base.Const.LEVEL_SUMMARY) + + def infoOfCloud(self, path): + '''Collects the state data of a cloud given by the path. + @param path: the base directory of the cloud + @return: the state info as a JSON map + ''' + curDir = os.curdir + os.chdir(path) + nodes = os.listdir('data') + users = '' + count = 0 + for node in nodes: + if (os.path.isdir('data/' + node) and not node.startswith('appdata_') + and not node.startswith('updater.') and not node.startswith('updater-') + and node != 'files_external'): + users += ' ' + node + count += 1 + logs = '' + fnLog = 'data/nextcloud.log' + if os.path.exists(fnLog): + lines = [] + maxCount = 5 + with open(fnLog, 'r') as fp: + lineNo = 0 + for line in fp: + lineNo += 1 + if len(lines) >= maxCount: + del lines[0] + lines.append(line) + for ix in range(len(lines)): + line = lines[len(lines) - ix - 1].strip().replace('"', "'") + logs += '{}: '.format(lineNo - ix) + line + '\\n\\n' + users = '[{}]:'.format(count) + users + self._processHelper.execute( + ['hmdu', '.', 'data', 'files_trashbin'], False, True) + duInfo = self._processHelper._output + # = files: 100 / 6 with 0.002999 MB / 0.000234 MB dirs: 25 / 3 ignored: 0/0 in cloud_1 + # = youngest: 2020.04.01-00:16:45 cloud_1/data/admin/dir_1/file1.txt + # = oldest: 2020.04.01-00:11:27 cloud_1/data/nextcloud.log + # = largest: 0.000068 MB cloud_1/data/nextcloud.log + # = trash: + # = youngest: 2020.04.01-00:14:35 cloud_1/data/admin/files_trashbin/f2.txt + # = oldest: 2020.04.01-00:14:28 cloud_1/data/admin/files_trashbin/f1.txt + # = largest: 0.000039 MB cloud_1/data/admin/files_trashbin/f1.txt + info = '\\n'.join(duInfo) + date = time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(time.time())) + content = base.StringUtils.fromFile(path + os.sep + '.fs.size') + total = int('64' if content == '' else content.strip() + [0:-1]) * 1024 * 1024 * 1024 + # ............................1...1...2...2......3......3......4......4..........5...5...6...6 + matcher = re.match( + r'= files: (\d+) / (\d+) with ([\d.]+) MB / ([\d.]+) MB dirs: (\d+) / (\d+)', duInfo[0]) + used = int(float(matcher.group(3)) * 1E6) + free = int(float(matcher.group(4)) * 1E6) + fnConfig = (self._configuration.getString('wdfiller.cloud.main.directory') + os.sep + + os.path.basename(path) + '/config/config.php') + version = '' + if os.path.exists(fnConfig): + version1 = base.StringUtils.grepInFile(fnConfig, re.compile( + re.compile(r"^\s*'version'\s*=>\s*'([\d.]+)")), 1, 1) + if version1: + version = version1[0] + rc = '''{} +"host": "{}", +"name": "{}", +"date": "{}", +"total": {}, +"used": {}, +"free": {}, +"trash": {}, +"trashDirs": {}, +"trashFiles": {}, +"users": "{}", +"info": "{}", +"log": "{}", +"version": "{}" +{} +'''.format('{', self._hostname, os.path.basename(path), date, total, used, total - used, free, + int(matcher.group(6)), int(matcher.group(2)), users, info, logs, version, '}') + os.chdir(curDir) + return rc + + def infoOfFilesystem(self, path): + '''Collects the state data of a filesystem given by the path. + @param path: the base directory of the cloud + @return: the state info as a JSON map + ''' + info = base.LinuxUtils.diskInfo(path) + if info is None: + rc = None + else: + name = info[0] + if path in self._mapFilesystems: + name = self._mapFilesystems[path] if self._mapFilesystems[path] != '' else name + elif path.startswith('/media/'): + name = path[7:] + elif path == '/': + name = 'root' + # info:..path, total, free, available + total, available = info[1], info[3] + used = total - available + rc = '{} "date": "{}", "host": "{}", "name": "{}", "total": {}, "used": {}, "free": {} {}'.format( + '{', time.strftime('%Y-%m-%dT%H:%M:%S', + time.localtime(time.time())), + self._hostname, name, total, used, available, '}') + return rc + + def install(self): + '''Installs the application and the related service. + ''' + app.BaseApp.BaseApp.install(self) + self.createSystemDScript('satboxx', 'satboxx', 'satboxx', + 'satboxx', 'satboxx: sends data via REST to servers') + self.installAsService('satboxx', True) + + def nextCloud(self): + '''Returns the info of the next cloud. + @return: None: no cloud available otherwise: the info about the next cloud + ''' + if self._clouds is None: + base1 = self._configuration.getString( + 'wdfiller.cloud.data.directory') + self._clouds = [] + if base1 is not None: + nodes = os.listdir(base1) + for node in nodes: + full = base1 + os.sep + node + if os.path.isdir(full) and (self._rexprExcludedClouds is None or not self._rexprExcludedClouds.search(node)): + self._clouds.append(full) + self._countClouds = len(self._clouds) + rc = None + if self._clouds: + rc = self._clouds[0] + del self._clouds[0] + return rc + + def nextFilesystem(self): + '''Returns the info of the next filesystem. + @return: None: no forther filesystem is available otherwise: the info about the filesystem + ''' + if self._filesystems is None: + self._filesystems = [] + fsList = base.LinuxUtils.disksMounted(self._logger) + for fsInfo in fsList: + mount = fsInfo + if self._rexprExcludedFilesystem is None or self._rexprExcludedFilesystem.search(mount): + continue + self._filesystems.append(fsInfo) + self._countFilesystems = len(self._filesystems) + rc = None + if self._filesystems: + rc = self._filesystems[0] + del self._filesystems[0] + return rc + + def prepareClouds(self): + '''Prepares some data taken from the configuration file. + ''' + excluded = self._configuration.getString('wdfiller.cloud.excluded') + if excluded is not None and excluded != '': + try: + self._logger.log('clouds: excluded: ' + + excluded, base.Const.LEVEL_LOOP) + self._rexprExcludedClouds = re.compile(excluded) + except Exception as exc: + self._logger.error('wrong regular expr in "wdfiller.cloud.excluded": {} [{}]'.format( + str(exc), str(type(exc)))) + + def prepareFilesystems(self): + '''Prepares some data taken from the configuration file. + ''' + self._mapFilesystems = {} + value = self._configuration.getString('wdfiller.filesystem.map', '') + if value != '': + fsMapping = value.split(',') + for item in fsMapping: + parts = item.split(':') + if len(parts) == 2: + self._mapFilesystems[parts[0]] = parts[1] + else: + self._logger.error( + 'wdfiller.filesystem.map has wrong syntax: ' + value) + excluded = self._configuration.getString( + 'wdfiller.filesystem.excluded') + if excluded is not None and excluded != '': + try: + self._logger.log('filesystems: excluded: ' + + excluded, base.Const.LEVEL_LOOP) + self._rexprExcludedFilesystem = re.compile(excluded) + except Exception as exc: + self._logger.error('wrong regular expr in "wdfiller.filesystem.excluded": {} [{}]'.format( + str(exc), str(type(exc)))) + + def run(self): + '''Implements the tasks of the application + ''' + self._hostname = self._configuration.getString('hostname', '') + if self._mainMode == 'test': + self.test() + elif self._mainMode == 'daemon': + self.daemon() + else: + self.abort('unknown mode: ' + self._mainMode) + + def sendCloudInfo(self, pathCloud): + '''Sends the cloud state info to the REST server. + @param pathCloud: the full path of the cloud directory + ''' + self._logger.log('sending cloud info to ' + + self._webDashServer, base.Const.LEVEL_LOOP) + self._client.putSimpleRest( + self._webDashServer, 'cloud', 'db', self.infoOfCloud(pathCloud)) + + def sendFilesystemInfo(self, pathFilesystem): + '''Sends the cloud state info to the REST server. + @param pathFilesystem: the mount path of the filesystem + ''' + self._logger.log('sending fs info to ' + + self._webDashServer, base.Const.LEVEL_LOOP) + info = self.infoOfFilesystem(pathFilesystem) + if info is None: + self._logger.log('info not available: ' + + pathFilesystem, base.Const.LEVEL_LOOP) + else: + self._client.putSimpleRest(self._webDashServer, 'fs', 'db', info) + + def test(self): + '''Tests a function. + ''' + kind = self.shiftProgramArgument() + count = base.StringUtils.asInt(self.shiftProgramArgument('1')) + interval = base.StringUtils.asInt(self.shiftProgramArgument('1')) + if kind is None: + self.abort('missing kind') + elif count is None: + self.abort(' is not an integer') + elif interval is None: + self.abort(' is not an integer') + elif self.handleOptions(): + for ix in range(count): + if kind == 'cloud': + self.testCloud(count, interval) + elif kind in ('filesystem', 'fs'): + self.testFilesystems(count, interval) + else: + self.abort('unknown kind: ' + kind) + break + time.sleep(interval) + base.StringUtils.avoidWarning(ix) + + def testCloud(self, count, interval): + '''Sends a limited count of cloud infos for test purposes. + ''' + self.prepareClouds() + self._webDashServer = self._configuration.getString( + 'wdfiller.url', 'http://localhost') + self._client = net.HttpClient.HttpClient(self._logger) + for ix in range(count): + cloud = self.nextCloud() + if cloud is None: + continue + else: + self.sendCloudInfo(cloud) + time.sleep(interval) + base.StringUtils.avoidWarning(ix) + + def testFilesystems(self, count, interval): + '''Sends a limited count of cloud infos for test purposes. + ''' + self.prepareFilesystems() + self._webDashServer = self._configuration.getString( + 'wdfiller.url', 'http://localhost') + self._client = net.HttpClient.HttpClient(self._logger) + for ix in range(count): + fs = self.nextFilesystem() + if fs is None: + continue + else: + self.sendFilesystemInfo(fs) + time.sleep(interval) + base.StringUtils.avoidWarning(ix) + + +def main(args): + '''Main function. + @param args: the program arguments + ''' + snakeboxx.startApplication() + application = SatelliteApp(args) + application.main() + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/app/SpecialApp.py b/app/SpecialApp.py new file mode 100755 index 0000000..56d6991 --- /dev/null +++ b/app/SpecialApp.py @@ -0,0 +1,369 @@ +#! /usr/bin/python3 +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import sys +import time +import shutil +import os.path +# see snake_install.sh +import snakeboxx + +import base.FileHelper +import app.BaseApp + + +class GeoAtBuilder: + '''Builds the import file for the table geo with Austrian data. + Glossar: + NUTS: Nomenclature des unités territoriales statistiques + Hierarchical structure of "political regions" with 3 Levels + ''' + + def __init__(self, logger): + '''Constructor. + @param logger: for logging + ''' + self._logger = logger + # + self._fnGemeinden = 'gemliste_knz.csv' + self._fnLaender = 'nuts_3.csv' + self._fnPolitischeBezirke = 'polbezirke.csv' + # id -> name, e.g. "1" -> "Burgenland" + self._mapLand = {} + # id => name, e.g. "101" -> "Eisenstadt(Stadt)" + self._mapPolitischerBezirk = {} + # gemeindenr (ags) => name, e.g. 10512 => 'Südburgenland' + self._mapTeilland = {} + # name => NUTS-3, e.g. "Nordburgenland" => "AT112" + self._mapNutsTeilland = {} + # ags => name, e.g. 10512 => 'Mühlgraben' + self._mapGemeinde = {} + + def readNuts(self): + '''Creates the maps _mapLand and _mapPolitischerBezierk from _fnPolitischeBezirke and _mapTeilland from _fnLaender + ''' + with open(self._fnLaender, 'r') as fp: + lineNo = 0 + expected = 'NUTS 3-Code;NUTS 3- Name;LAU 2 - Code Gemeinde- kennziffer' + for line in fp: + lineNo += 1 + if lineNo == 3 and not line.startswith(expected): + msg = 'wrong file format:\n{}\nexpected:\n{}'.format( + line, expected) + self._logger.error(msg) + raise ValueError(msg) + if lineNo < 4: + continue + if line.startswith(';;;') or line.startswith('Q: STATISTIK AUSTRIA'): + continue + # NUTS 3-Code;NUTS 3- + # Name;Gemeindekennziffer;Gemeindename;Fläche;Bevölkerungszahl + cols = line.strip().split(';') + if len(cols) < 5: + self._logger.error('unknown input: ' + line) + continue + ags = int(cols[2]) + self._mapTeilland[ags] = cols[1] + self._mapGemeinde[ags] = cols[3] + self._mapNutsTeilland[cols[1]] = cols[0] + with open(self._fnPolitischeBezirke, 'r') as fp: + lineNo = 0 + expected = 'Bundeslandkennziffer;Bundesland;Kennziffer pol. Bezirk' + for line in fp: + lineNo += 1 + if lineNo == 3 and not line.startswith(expected): + msg = 'wrong file format:\n{}\nexpected:\n{}'.format( + line, expected) + self._logger.error(msg) + raise ValueError(msg) + if lineNo < 4: + continue + cols = line.strip().split(';') + if len(cols) < 5: + if not line.startswith('Quelle: STATISTIK AUSTRIA'): + self._logger.error('unknown input: ' + line) + else: + idPolitischerBezirk = int(cols[4]) + self._mapPolitischerBezirk[idPolitischerBezirk] = cols[3] + if cols[2] != cols[4]: + self._logger.log('different codes: {} {} {}'.format( + cols[2], cols[4], cols[3])) + self._logger.log( + 'Land: {} Teilland: {} Pol. Bezirk: {}'.format(len(self._mapLand.keys()), + len(self._mapTeilland.keys( + )), + len(self._mapPolitischerBezirk.keys()))) + + def createGeoAt(self): + '''Creates the geo_at.csv from _fnGemeinden + ''' + with open('geo_at.sql', 'w') as fpOut, open(self._fnGemeinden, 'r') as fpIn: + fpOut.write('''insert into geo (geo_id,geo_staat,geo_land,geo_landnuts,geo_bezirk,geo_bezirknuts,geo_kreis, + geo_kreisnuts,geo_gemeinde,geo_ort,geo_gemeindeags,geo_plz) VALUES +''') + lineNo = 0 + written = 0 + for line in fpIn: + lineNo += 1 + if lineNo == 3 and not line.startswith('Gemeindekennziffer;Gemeindename;'): + self._logger.error( + 'missing Gemeindekennziffer;Gemeindename: ' + line) + raise ValueError( + 'wrong file format: ' + self._fnGemeinden) + if lineNo < 4: + continue + # Gemeindekennziffer;Gemeindename;Gemeindecode;Status;PLZ + # desGem.Amtes;weitere Postleitzahlen + cols = line.strip().split(';') + if len(cols) < 6: + if not line.startswith('Quelle: STATISTIK AUSTRIA'): + self._logger.error('unknown input: ' + line) + else: + ags = int(cols[2]) + # NUTS level 2: "Bundesland" + idLand = ags // 10000 + id2Land = 1200 + idLand + # NUTS level 3: Teilland" + teillandName = self._mapTeilland[ags] if ags in self._mapTeilland else '' + if teillandName == '': + if ags in [61060, 61061]: + teillandName = self._mapTeilland[61059] + elif ags >= 90001: + teillandName = self._mapTeilland[90001] + else: + self._logger.error( + f'missing teilland in map: {ags}') + nutsTeilland = self._mapNutsTeilland[teillandName] if teillandName in self._mapNutsTeilland else '' + # Kreis: politischer Bezirk + idPolitischerBezirk = ags // 100 + namePolitischerBezirk = (self._mapPolitischerBezirk[idPolitischerBezirk] + if idPolitischerBezirk in self._mapPolitischerBezirk else '') + if namePolitischerBezirk == '': + self._logger.error( + f'missing PolitischerBezirk for {idPolitischerBezirk}') + info1 = (f",'at',{id2Land},'AT{idLand}','{teillandName}','{nutsTeilland}'," + + f"'{namePolitischerBezirk}','{idPolitischerBezirk}','{cols[1]}','{cols[1]}',{ags},") + info = info1 + f'{cols[4]})' + primary = 20000001 + ags * 100 + written += 1 + if written > 1: + fpOut.write(',\n') + fpOut.write('(' + str(primary) + info) + for aZip in cols[5].split(' '): + if aZip == '': + continue + primary += 1 + written += 1 + fpOut.write(',\n(' + str(primary) + info1 + aZip + ')') + fpOut.write('\n;') + self._logger.log(f'written: geo_at.sql: {written} recs') + fn = 'ImportGeo.sh' + base.StringUtils.toFile(fn, '''#! /bin/bash +if [ "$3" = "" ]; then + echo "Usage: ImportGeo.sh DB USER DB + echo "Example: ImportGeo.sh appcovidmars covidmars TopSecret" +else + test geo.csf && rm geo.csv + ln -s geo_at.csv geo.csv + mysqlimport --ignore-lines=1 --fields-terminated-by=, --local \ + --columns=geo_id,geo_staat,geo_land,geo_bezirk,geo_kreis,geo_gemeindeags,geo_gemeinde,geo_plz \ + -u $2 "-p$3" $1 geo.csv +fi +''', fileMode=0o777) + self._logger.log('written: {} usage: {} DB USER PW'.format(fn, fn)) + + def check(self): + '''Checks the preconditions. + @return: True: success + ''' + rc = True + if not os.path.exists(self._fnGemeinden): + rc = self._logger.error('missing file ' + self._fnGemeinden) + if not os.path.exists(self._fnLaender): + rc = self._logger.error('missing file ' + self._fnLaender) + if not os.path.exists(self._fnPolitischeBezirke): + rc = self._logger.error( + 'missing file ' + self._fnPolitischeBezirke) + return rc + + def buildImport(self): + '''Builds the CSV file for importing the geo data. + ''' + if self.check(): + self.readNuts() + self.createGeoAt() + + +class SpecialApp(app.BaseApp.BaseApp): + '''Special task solver. + ''' + + def __init__(self, args): + '''Constructor. + @param args: the program arguments, e.g. ['test', 'a@bc.de'] + ''' + app.BaseApp.BaseApp.__init__( + self, 'SpecialApp', args, None, 'specboxx') + self._hostname = None + + def buildConfig(self): + '''Creates an useful configuration example. + ''' + content = '''# created by SpecialApp +logfile=/var/log/local/specboxx.log +''' + self.buildStandardConfig(content) + + def buildUsage(self): + '''Builds the usage message. + ''' + self._usageInfo.appendDescription('''APP-NAME [] + Convert file(s) into other. +''') + self._usageInfo.addMode('init-project', '''init-project [] + Initializes a PHP project using "skeleton". + : the project name + : the base project. Default: skeleton + ''', '''APP-NAME init-project webmonitor +''') + self._usageInfo.addMode('geo-at', '''geo-at + Builds the geodb data of Austria. + ''', '''APP-NAME geo-at +''') + self._usageInfo.addMode('copy-firefox', '''copy-firefox + Copies the secrets and favorites of a firefox profile into another profile. + the source profile (a directory) + the target profile (a directory) + ''', '''APP-NAME copy-firefox /media/backup/.mozilla/firefox/yw98aaen.default-release x6ytqd3w.default-release-2 +''') + + def buildUsageOptions(self, mode=None, subMode=None): + '''Adds the options for a given mode. + @param mode: None or the mode for which the option is added + @param subMode: None or the submode for which the option is added + ''' + # def add(mode, opt): + # self._usageInfo.addModeOption(mode, opt) + + base.StringUtils.avoidWarning(subMode) + if mode is None: + mode = self._mainMode + if mode == 'init-project': + self._usageInfo.addModeOption(mode) + elif mode == 'geo-at': + self._usageInfo.addModeOption(mode) + elif mode == 'copy-firefox': + self._usageInfo.addModeOption(mode) + + def copyFirefox(self): + '''Copies secrets and bookmarks from one firefox profile to another. + ''' + def isMissing(names, source): + rc = False + for item in names: + rc2 = not os.path.exists(os.path.join(source, item)) + if rc2: + rc = False + self._logger.error(f'missing {source}/{item}') + return rc + def copy(aFile, source, target): + src = os.path.join(source, aFile) + trg = os.path.join(target, aFile) + if os.path.exists(trg): + trg2 = trg + f'.{int(time.time())}' + self._logger.log(f'saving {trg} as {trg2}...', base.Const.LEVEL_DETAIL) + os.rename(trg, trg2) + shutil.copy(src, trg) + source = self.shiftProgramArgument() + target = self.shiftProgramArgument() + names = 'key4.db places.sqlite favicons.sqlite logins.json'.split(' ') + if self.handleOptions(): + if target is None: + self.abort('missing arguments') + elif not os.path.isdir(source): + self.abort(f'source is not a directory: {source}') + elif not os.path.isdir(source): + self.abort(f'target is not a directory: {target}') + elif isMissing(names, source): + self.abort(f'not a true profile directory: {source}') + elif not os.path.exists(os.path.join(target, 'key4.db')): + self.abort(f'target {target} is not a profile: missing key4.db') + else: + for item in names: + copy(item, source, target) + + def geoAt(self): + '''Builds an import file for the db table geo with Austrian data. + ''' + if self.handleOptions(): + builder = GeoAtBuilder(self._logger) + builder.buildImport() + + def initProject(self): + '''Initializes a PHP project using "skeleton". + ''' + projName = self.shiftProgramArgument() + baseProject = self.shiftProgramArgument('skeleton') + if projName is None: + self.argumentError('missing ') + elif not os.path.isdir(baseProject): + self.argumentError( + 'project "{}" must be in the current directory'.format(baseProject)) + elif os.path.isdir(projName): + self.argumentError('project "{}" already exists'.format(projName)) + elif self.handleOptions(): + fnFileStructure = '{}/tools/templates/project.structure.txt'.format(baseProject) + if not os.path.exists(fnFileStructure): + self.abort('missing ' + fnFileStructure) + base.FileHelper.setLogger(self._logger) + structure = base.StringUtils.fromFile( + fnFileStructure).replace('${proj}', projName).split('\n') + base.FileHelper.copyByRules(structure, baseProject, projName) + msg = '''# run as root: +cat </etc/snakeboxx/webapps.d/PROJ.dev +db=appPROJ +user=PROJ +password=PROJ4PROJ +sql.file=PROJ_appPROJ +directory=/home/ws/php/PROJ +excluded= +EOS +# +dbboxx create-db-and-user appPROJ PROJ PROJ4PROJ +grep PROJ.dev /etc/hosts || echo >>/etc/hosts "127.0.0.10 PROJ.dev" +FN=/etc/nginx/sites-available/PROJ.dev +perl -p -e 's/%project%/PROJ/g;' skeleton/tools/templates/nginx.project.dev > $FN +ln -s "../sites-available/PROJ.dev" /etc/nginx/sites-enabled/PROJ.dev +head $FN +dbboxx import-webapp PROJ.dev skeleton/tools/templates/empty_db.sql.gz +'''.replace('PROJ', projName) + print(msg) + + def run(self): + '''Implements the tasks of the application + ''' + self._hostname = self._configuration.getString('hostname', '') + if self._mainMode == 'init-project': + self.initProject() + elif self._mainMode == 'geo-at': + self.geoAt() + elif self._mainMode == 'copy-firefox': + self.copyFirefox() + else: + self.abort('unknown mode: ' + self._mainMode) + + +def main(args): + '''Main function. + @param args: the program arguments + ''' + snakeboxx.startApplication() + application = SpecialApp(args) + application.main() + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/app/SunMonitor.py b/app/SunMonitor.py new file mode 100644 index 0000000..e5c87f0 --- /dev/null +++ b/app/SunMonitor.py @@ -0,0 +1,78 @@ +''' +Created on 26.06.2022 + +@author: wk +''' +import sys +import os.path +import time +import fnmatch +import snakeboxx + + +import base.DirTraverser +import base.FileHelper +import net.HttpClient +import app.BaseApp +from typing import Sequence + + +class SunMonitorApp(app.BaseApp.BaseApp): + def __init__(self, args: Sequence[str]): + '''Constructor. + @param args: the program arguments, e.g. ['daemon', 'a@bc.de'] + ''' + app.BaseApp.BaseApp.__init__(self, 'SunMonitorApp', args, None, 'sunboxx') + self._url = None + self._ + + + def buildConfig(self): + '''Creates an useful configuration example. + ''' + content = '''# created by EMailApp +url.sun=http://sun/rpc +db.name=appsunmonitor +db.user=Jonny +db.password=TopSecret +data.directory=/opt/sunmonitor +''' + self.buildStandardConfig(content) + + def buildUsage(self): + '''Builds the usage message. + ''' + self._usageInfo.appendDescription('''APP-NAME [] + Request and store the photovoltaics statistics in a loop (multiple times). +''') + self._usageInfo.addMode('daemon', '''statistic + Assemble the data and store it in files or db. + ''', '''APP-NAME daemon +APP-NAME daemon +''') + self._usageInfo.addMode('status', '''status + Show the current status + ''', '''APP-NAME daemon +APP-NAME status +''') + + def fetchStatus(self): + '''Fetch the status from the Shelly box. + ''' + client = net.HttpClient(self._logger) + client. + def run(self): + '''Implements the tasks of the application. + ''' + if self._mainMode == 'daemon': + self.daemon() + elif self._mainMode == 'status': + self.status() + else: + self.abort('unknown mode: ' + self._mainMode) + def status(self): + '''Shows the current status. + ''' + +if __name__ == '__main__': + pass \ No newline at end of file diff --git a/app/TextApp.py b/app/TextApp.py new file mode 100755 index 0000000..db8ccac --- /dev/null +++ b/app/TextApp.py @@ -0,0 +1,693 @@ +#! /usr/bin/python3 +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import sys +import os +import re +# see snake_install.sh +import snakeboxx + +import base.CsvProcessor +import base.TextProcessor +import base.DirTraverser +import app.BaseApp + + +class OptionsExecuteRules: + '''Options for the mode execute-rules. + ''' + + def __init__(self): + '''Constructor. + ''' + self.maxLoop = 1 + self.backup = None + + +class OptionsGrep: + '''Options for the mode grep + ''' + + def __init__(self): + '''Constructor. + ''' + self.wordOnly = False + self.group = None + self.lineNumber = False + self.ignoreCase = False + self.invertMatch = False + self.formatFile = None + self.formatLine = None + self.belowContext = None + self.aboveContext = None + self.belowChars = None + self.aboveChars = None + + +class OptionsInsertOrReplace: + '''Options for the mode insert-or-replace + ''' + + def __init__(self): + '''Constructor. + ''' + self.ignoreCase = False + self.anchor = None + self.above = False + self.backup = None + + +class OptionsReplace: + '''Options for the mode replace + ''' + + def __init__(self): + '''Constructor. + ''' + self.ignoreCase = False + self.backupExtensions = None + self.prefixBackref = None + self.wordOnly = None + self.rawString = False + self.escActive = False + + +class TextApp(app.BaseApp.BaseApp): + '''Performs some tasks with text files: searching, modification... + ''' + + def __init__(self, args): + '''Constructor. + @param args: the program arguments, e.g. ['test', 'a@bc.de'] + ''' + app.BaseApp.BaseApp.__init__(self, 'TextApp', args, None, 'textboxx') + self._processor = None + self._traverser = None + self._hostname = None + + def buildConfig(self): + '''Creates an useful configuration example. + ''' + content = '''# created by TextApp +logfile=/var/log/local/textboxx.log +''' + self.buildStandardConfig(content) + + def buildUsage(self): + '''Builds the usage message. + ''' + self._usageInfo.appendDescription('''APP-NAME [] + Searching and modifying in text files. +''') + self._usageInfo.addMode('describe-csv', '''describe-csv + Describes the syntax and the usage of the CSV mode. + ''', '''APP-NAME describe-csv +''') + self._usageInfo.addMode('describe-rules', '''describe-rules + Describes the syntax and the meaning of the rules + ''', '''APP-NAME describe-rules +''') + self._usageInfo.addMode('exec-rules', '''exec-rules [] + Executes the given which allows to show some parts of or modify the given the files. + : a string describing the actions to do. call "APP-NAME describe-rules" for more info + ''', '''APP-NAME exec-rules ">/hello/" "*.txt" --recursive --min-depth=1 --files-only +''') + self._usageInfo.addMode('csv-execute', '''csv-execute + Does some operations on a Comma Separated File. More info with csv-describe. + : specifies the file(s) to process. +''', '''APP-NAME csv-info "address.csv" --sorted --unique --index=3,4 --cols=*name*,*city* +''') + self._usageInfo.addMode('grep', '''grep + Searches the regular expression in files. + : specifies the file(s) to process. +''', r'''APP-NAME grep -n -g1 'Date:\s+(\d{4}\.\d\d\.\d\d)' "*.txt" --exclude-dirs=.git --max-depth=2 +APP-NAME grep "[\w.+-]+@[\w.+-]+" "*.addr" --format-file="=== EMail addresses in file %f:" --format-line=%l:%T%t *.addr +''') + + self._usageInfo.addMode('insert-or-replace', '''insert-or-replace + Searches . If found this line is replaced by . + If not found and option --anchor is given: is inserted at this position. + : the regular expression to identify the line to replace + : the line to replace or insert + : specifies the file(s) to process. +''', r'''APP-NAME insert-or-replace '^\s*memory_limit\s*=' "memory_limit=2048M" "/etc/php/7.3/fmt/php.ini" -aphp.net/memory-limit +''') + + self._usageInfo.addMode('replace', r'''replace + Searches the regular expression in files. + : the pattern to search, a regular expression only if --not-regexpr + : the will be replaced by this string. Can contain backreferences: see --prefix-backref + : + file name pattern, with wilcards *, ? [chars] and [!not chars]. +''', r'''APP-NAME replace "version: ([\d+.]+)" "V%1" "*.py" --prefix-backref=% -B.bak +''') + + self._usageInfo.addMode('many-replacements', r'''many-replacements + Searches the regular expression in and print the processed (replaced) string. + This is useful for complex scripts. + : a text file with lines "TAB" + : + file name pattern, with wilcards *, ? [chars] and [!not chars]. +''', r'''APP-NAME many-replacements changes.txt *.html --file-type=fl --max-size=100ki +''') + + self._usageInfo.addMode('string-replacement', r'''string-replacement + Searches the regular expression in and print the processed (replaced) string. + This is useful for complex scripts. + : the pattern to search, a regular expression only if --not-regexpr + : the will be replaced by this string. Can contain backreferences: see --prefix-backref + : + this string will be processed +''', r'''APP-NAME string-replacement "(files|dirs): (\d+)" "%2 %1" "files: 4 dirs: 9" --prefix-backref=% +''') + + def buildUsageOptions(self, mode=None, subMode=None): + '''Adds the options for a given mode. + @param mode: None or the mode for which the option is added + @param subMode: None or the submode for which the option is added + ''' + def add(mode, opt): + self._usageInfo.addModeOption(mode, opt) + + def addIgnore(mode): + add(mode, base.UsageInfo.Option('ignore-case', 'i', + 'ignore case while searching', 'bool')) + + def addIgnoreAndWord(mode): + addIgnore(mode) + add(mode, base.UsageInfo.Option('word-regexpr', 'w', + 'only whole words will be found', 'bool')) + + def addBackup(mode): + add(mode, base.UsageInfo.Option('backup', 'B', + '''if is starting with ".": the origin file will be renamed with this extension +if is a directory: the original file is moved to this directory +if not given no backup is done: the original file is modified''')) + + def addEsc(mode): + add(mode, base.UsageInfo.Option('esc-active', 'e', + r"esc sequences '\n', '\r', \t', '\xXX', '\uXXXX' and '\Uxxxxxxxx' " + + 'in replacement will be recognized', 'bool')) + + def addPrefixBackticks(mode): + add(mode, base.UsageInfo.Option('prefix-backref', 'b', + r'''if given will be replaced by the group +example: opt: -b% reg-expr: "version: ([\d+.]+)" replacement: "V%1" string: "version: 4.7" result: "V4.7"''')) + + def addRawString(mode): + add(mode, base.UsageInfo.Option('raw-string', 'R', + ' is a string, not a regular expression', 'bool')) + + def addReplace(mode, changeFile): + addIgnoreAndWord(mode) + addEsc(mode) + addPrefixBackticks(mode) + addRawString(mode) + if changeFile: + addBackup(mode) + + base.StringUtils.avoidWarning(subMode) + if mode is None: + mode = self._mainMode + if mode == 'describe-rules': + self._usageInfo.addModeOption(mode) + elif mode == 'exec-rules': + base.DirTraverser.addOptions(mode, self._usageInfo) + addBackup(mode) + add(mode, base.UsageInfo.Option('max-loops', 'l', + 'the process is stopped after * statements', 'int', 1)) + elif mode == 'csv-execute': + base.DirTraverser.addOptions(mode, self._usageInfo) + elif mode == 'grep': + base.DirTraverser.addOptions(mode, self._usageInfo) + addIgnoreAndWord(mode) + add(mode, base.UsageInfo.Option('above-chars', 'a', + 'displays characters above the hit. Sets implicitly --only-matching', 'int')) + add(mode, base.UsageInfo.Option('below-chars', 'b', + 'displays characters below the hit. Sets implicitly --only-matching', 'int')) + add(mode, base.UsageInfo.Option('above-context', 'A', + 'displays lines above the hit', 'int')) + add(mode, base.UsageInfo.Option('below-context', 'B', + 'displays lines below the hit', 'int')) + add(mode, base.UsageInfo.Option('binary-too', None, + 'the search is done in binary files too', 'bool')) + add(mode, base.UsageInfo.Option('context', 'C', + 'displays lines above and below the hit', 'int')) + add(mode, base.UsageInfo.Option('format-line', 'f', + '''defines the display format of a hit line. +Placeholders: %f: full filename %p: path %n: node %# line number +%t: line text %: group N (N in [0..9] %%: '%' %L: newline %T: tabulator''')) + add(mode, base.UsageInfo.Option('format-file', 'F', + '''defines the display format of the prefix of a file with hits. +Placeholders: %f: full filename %p: path %n: node %%: '%' %L: newline %T: tabulator''')) + add(mode, base.UsageInfo.Option('group', 'g', + 'displays only the -th group of the regular expression', 'int')) + add(mode, base.UsageInfo.Option('length-binary-test', None, + 'the test whether the file is binary is done with that number of bytes', 'int', 4096)) + add(mode, base.UsageInfo.Option('line-number', 'n', + 'the line number will be displayed for each hit', 'bool')) + add(mode, base.UsageInfo.Option('only-matching', 'o', + 'only the matching part of the line will be displayed (not the whole line)', 'bool')) + add(mode, base.UsageInfo.Option('invert-match', 'v', + 'all lines not containing the search expression is displayed', 'bool')) + elif mode == 'insert-or-replace': + base.DirTraverser.addOptions(mode, self._usageInfo) + addIgnore(mode) + add(mode, base.UsageInfo.Option('above', 'A', + 'the insertion point is above the anchor', 'bool')) + add(mode, base.UsageInfo.Option('anchor', 'a', + 'defines the insertion position if is not found')) + addBackup(mode) + elif mode == 'replace': + base.DirTraverser.addOptions(mode, self._usageInfo) + addReplace(mode, True) + elif mode == 'many-replacements': + base.DirTraverser.addOptions(mode, self._usageInfo) + addReplace(mode, True) + elif mode == 'string-replacement': + addReplace(mode, False) + + def csvExecute(self): + '''Executes a sequence of commands on CSV files. + ''' + commands = self.shiftProgramArgument() + pattern = self.shiftProgramArgument() + wrong = self.shiftProgramArgument() + if wrong is not None: + self.abort('too many arguments: ' + wrong) + elif pattern is None: + self.abort('too few arguments') + elif self.handleOptions(): + self._processor = base.CsvProcessor.CsvProcessor(self._logger) + self._traverser = base.DirTraverser.buildFromOptions( + pattern, self._usageInfo, 'csv-execute') + self._traverser._findFiles = self._traverser._findLinks = True + self._traverser._findDirs = False + for filename in self._traverser.next(self._traverser._directory, 0): + self._processor.readFile(filename) + self._processor.execute(commands) + + def describeRules(self): + '''Displays the description of the rules. + ''' + item = base.SearchRuleList.SearchRuleList(self._logger) + item.describe() + + def execRules(self): + '''Executes a sequence of rules on specified files. + ''' + rules = self.shiftProgramArgument() + pattern = self.shiftProgramArgument() + wrong = self.shiftProgramArgument() + if wrong is not None: + self.abort('too many arguments') + elif pattern is None: + self.abort('missing ') + elif self.handleOptions(): + self._processor = base.TextProcessor.TextProcessor(self._logger) + options = OptionsExecuteRules() + options.backup = self._optionProcessor.valueOf('backup') + options.maxLoop = self._optionProcessor.valueOf('max-loops') + self._traverser = base.DirTraverser.buildFromOptions( + pattern, self._usageInfo, 'exec-rules') + self._traverser._findFiles = self._traverser._findLinks = True + self._traverser._findDirs = False + for filename in self._traverser.next(self._traverser._directory, 0): + base.StringUtils.avoidWarning(filename) + self.execRulesOneFile(rules, options) + + def execRulesOneFile(self, rules, options): + '''Executes rules for one file. + @param filename: file to process + @param rules: the rules to execute + @param options: the program options (instance of OptionsExecuteRules) + ''' + self._processor.readFile(self._traverser._fileFullName) + self._processor.executeRules(rules, options.maxLoop) + if self._processor._hasChanged: + self._processor.writeFile(None, options.backup) + + def grep(self): + '''Searches regular expressions in files. + ''' + self._resultLines = [] + what = self.shiftProgramArgument() + pattern = self.shiftProgramArgument() + if pattern is None: + self.abort('too few arguments') + elif self.handleOptions(): + options = self.grepOptions() + if self._optionProcessor.valueOf('max-size') is None: + self._optionProcessor.optionByName( + 'max-size')._value = 10 * 1000 * 1000 + if options.wordOnly: + what = r'\b' + what + r'\b' + regExpr = re.compile( + what, base.Const.IGNORE_CASE if options.ignoreCase else 0) + self._traverser = base.DirTraverser.buildFromOptions( + pattern, self._usageInfo, 'grep') + self._traverser._findFiles = self._traverser._findLinks = True + self._traverser._findDirs = False + for filename in self._traverser.next(self._traverser._directory, 0): + if not os.path.isdir(filename): + if not self.grepOneFile(filename, regExpr, options): + break + + @staticmethod + def grepFormat(theFormat, filename, text, lineNo, matcher): + '''Returns a format with expanded placeholders. + @param format: the format string with placeholders, e.g. '%f-%#: %t' + @param filename: [full, path, node], e.g. ['/etc/password', '/etc/', 'password'] + @param text: None or the line text with the hit + @param lineNo: None or the line number + @param matcher: None or the match object of the hit + @return: the format with expanded placeholders, e.g. '/etc/password-12:sync:x:4:65534:sync:/bin:/bin/sync' + ''' + last = 0 + rc = '' + lengthFormat = len(theFormat) + while True: + position = theFormat.find('%', last) + if position > 0: + rc += theFormat[last:position] + if position < 0: + rc += theFormat[last:] + break + if position == lengthFormat - 1: + rc += '%' + break + variable = theFormat[position + 1] + if variable == 'f': + rc += filename[0] + elif variable == 'p': + rc += filename[1] + elif variable == 'n': + rc += filename[2] + elif variable == 'T': + rc += '\t' + elif variable == 'L': + rc += '\n' + elif variable == '%': + rc += '%' + elif variable == 't': + if text is not None: + rc += text + elif variable == '#': + rc += str(lineNo) + elif '0' <= variable <= '9': + group = ord(variable) - ord('0') + if matcher is not None: + if matcher.lastindex is None: + rc += matcher.group(0) + elif matcher.lastindex <= group: + rc += matcher.group(group) + last = position + 2 + return rc + + def grepOneFile(self, filename, regExpr, options): + '''Searches the regular expression in one file. + @param filename: the name of the file to inspect + @param regExpr: the regular expression to search + @param options: the program options a OptionsGrep instance + @return True: success False: stop procession + ''' + def output(line): + if self._logger._verboseLevel > 0: + self._resultLines.append(line) + print(line) + + def outputContext(theFormat, fileNames, start, end, lines): + while start < end: + line2 = TextApp.grepFormat( + theFormat, fileNames, lines[start], start + 1, None) + output(line2) + start += 1 + return end - 1 + optProc = self._optionProcessor + lines = base.StringUtils.fileToText(filename, '\n', + binaryTestLength=optProc.valueOf( + 'length-binary-test'), + ignoreBinary=not optProc.valueOf( + 'binary-too'), + maxLength=optProc.valueOf('max-size')) + nameList = [filename, os.path.dirname( + filename), os.path.basename(filename)] + first = True + lastIx = -1 + missingInterval = [None, None] + for ix, line in enumerate(lines): + matcher = regExpr.search(line) + if matcher and first and options.formatFile is not None: + output(TextApp.grepFormat( + options.formatFile, nameList, None, None, None)) + first = False + if matcher and not options.invertMatch or matcher is None and options.invertMatch: + if missingInterval[0] is not None: + ix2 = min(ix, missingInterval[1]) + lastIx = outputContext( + options.formatLine, nameList, missingInterval[0], ix2, lines) + missingInterval[0] = None + if options.aboveContext is not None: + start = max(0, lastIx + 1, ix - options.aboveContext) + if start < ix: + lastIx = outputContext( + options.formatLine, nameList, start, ix, lines) + if options.group is None and options.aboveChars is None and options.belowChars is None: + output(TextApp.grepFormat(options.formatLine, + nameList, line, ix + 1, matcher)) + else: + self.grepOneLine(options, nameList, line, ix + 1, regExpr) + lastIx = ix + if options.belowContext is not None: + missingInterval = [ix + 1, ix + 1 + options.belowContext] + if missingInterval[0] is not None: + lastIx = min(len(lines), missingInterval[1]) + outputContext(options.formatLine, nameList, + missingInterval[0], lastIx, lines) + return True + + def grepOneLine(self, options, nameList, line, lineNo, regExpr): + '''Handles the multiple hits in one line. + @precondition: only the matching pattern should be displayed (not the whole line). + @param options: the program options + @param namelist: variants of the filename: [, , ] + @param line: the line to inspect + @param lineNo: the line number of line + @param regExpr: the regular expression to search + ''' + lineLength = len(line) + group = 0 if options.group is None else options.group + for matcher in regExpr.finditer(line): + start, end = matcher.span(group) + if options.aboveChars is not None: + start = max(0, start - options.aboveChars) + if options.belowChars is not None: + end = min(lineLength, end + options.belowChars) + info = line[start:end] + info2 = TextApp.grepFormat( + options.formatLine, nameList, info, lineNo, matcher) + if self._logger._verboseLevel > 0: + self._resultLines.append(info2) + print(info2) + + def grepOptions(self): + '''Evaluates the grep options. + @return: the options stored in a OptionsGrep instance + ''' + options = OptionsGrep() + options.ignoreCase = self._optionProcessor.valueOf('ignore-case') + options.wordOnly = self._optionProcessor.valueOf('word-regexpr') + if self._optionProcessor.valueOf('group') is not None: + options.group = self._optionProcessor.valueOf('group') + elif self._optionProcessor.valueOf('only-matching'): + options.group = 0 + options.lineNumber = self._optionProcessor.valueOf('line-number') + options.invertMatch = self._optionProcessor.valueOf('invert-match') + options.formatFile = self._optionProcessor.valueOf('format-file') + options.formatLine = self._optionProcessor.valueOf('format-line') + # if options.formatLine is None: + # info = '%t' if options.group is None else f'%{options.group}' + # options.formatLine = f'%f-%#:{info}' if options.lineNumber else + # f'%f:{info}' + options.belowContext = self._optionProcessor.valueOf( + 'below-context') + options.aboveContext = self._optionProcessor.valueOf( + 'above-context') + if options.belowContext is None and options.aboveContext is None: + options.belowContext = options.aboveContext = options.aboveContext = self._optionProcessor.valueOf( + 'context') + options.belowChars = self._optionProcessor.valueOf('below-chars') + options.aboveChars = self._optionProcessor.valueOf('above-chars') + if options.formatLine is None: + info = '%t' if options.group is None else f'%{options.group}' + options.formatLine = f'%f-%#:{info}' if options.lineNumber else f'%f:{info}' + return options + + def insertOrReplace(self): + '''Searches regular expressions in files. + ''' + self._resultLines = [] + key = self.shiftProgramArgument() + line = self.shiftProgramArgument() + pattern = self.shiftProgramArgument() + if pattern is None: + self.abort('too few arguments') + elif self.handleOptions(): + options = OptionsInsertOrReplace() + options.ignoreCase = self._optionProcessor.valueOf('ignore-case') + options.anchor = self._optionProcessor.valueOf('anchor') + options.above = self._optionProcessor.valueOf('above') + options.backup = self._optionProcessor.valueOf('backup') + if options.ignoreCase and options.anchor is not None: + options.anchor = re.compile( + options.anchor.pattern, base.Const.IGNORE_CASE) + self._traverser = base.DirTraverser.buildFromOptions( + pattern, self._usageInfo, 'insert-or-replace') + self._processor = base.TextProcessor.TextProcessor(self._logger) + self._traverser._findFiles = self._traverser._findLinks = True + self._traverser._findDirs = False + for filename in self._traverser.next(self._traverser._directory, 0): + self._processor.readFile(filename) + self._processor.insertOrReplace( + key, line, options.anchor, options.above) + if self._processor._hasChanged: + self._processor.writeFile(filename, options.backup) + + def replace(self): + '''Replaces a regular expression in files by a replacement. + ''' + self._resultLines = [] + what = self.shiftProgramArgument() + replacement = self.shiftProgramArgument() + pattern = self.shiftProgramArgument() + if pattern is None: + self.abort('too few arguments') + elif self.handleOptions(): + self._processor = base.TextProcessor.TextProcessor(self._logger) + options = self.replaceOptions(True) + if options is not None: + self._traverser = base.DirTraverser.buildFromOptions( + pattern, self._usageInfo, 'replace') + self._traverser._findFiles = self._traverser._findLinks = True + self._traverser._findDirs = False + for filename in self._traverser.next(self._traverser._directory, 0): + self._processor.readFile(filename) + hits = self._processor.replace(what, replacement, options.prefixBackref, + options.rawString, True, options.wordOnly, options.ignoreCase, + options.escActive) + if hits > 0: + self._processor.writeFile( + filename, options.backupExtensions) + + def replaceMany(self): + '''Replaces strings by replacements given in a file in files. + ''' + self._resultLines = [] + dataFile = self.shiftProgramArgument() + pattern = self.shiftProgramArgument() + if pattern is None: + self.abort('too few arguments') + elif not os.path.exists(dataFile): + self.abort('data file does not exists: ' + dataFile) + elif self.handleOptions(): + what = [] + replacements = [] + lines = base.StringUtils.fromFile(dataFile, '\n') + for line in lines: + parts = line.split('\t') + if len(parts) == 2: + what.append(parts[0]) + replacements.append(parts[1]) + self._processor = base.TextProcessor.TextProcessor(self._logger) + options = self.replaceOptions(True) + if options is not None: + self._traverser = base.DirTraverser.buildFromOptions( + pattern, self._usageInfo, 'many-replacements') + self._traverser._findFiles = self._traverser._findLinks = True + self._traverser._findDirs = False + for filename in self._traverser.next(self._traverser._directory, 0): + self._processor.readFile(filename) + hits = self._processor.replaceMany(what, replacements) + if hits > 0: + self._processor.writeFile( + filename, options.backupExtensions) + + def replaceOptions(self, fileOptions): + '''Evaluates the options for the mode "replace", "string-replacement" and "many-replacements". + @param fileOptions: the mode is "replace" or "many-replacements": operates on files + @return: None: error occurred otherwise: the OptionsReplace instance + ''' + options = OptionsReplace() + options.ignoreCase = self._optionProcessor.valueOf('ignore-case') + options.wordOnly = self._optionProcessor.valueOf('word-regexpr') + options.prefixBackref = self._optionProcessor.valueOf('prefix-backref') + options.rawString = self._optionProcessor.valueOf('raw-string') + options.escActive = self._optionProcessor.valueOf('esc-active') + if fileOptions: + options.backupExtensions = self._optionProcessor.valueOf('backup') + if options.prefixBackref is not None and len(options.prefixBackref) != 1: + self.argumentError( + 'prefix-backref must have length 1, not ' + options.prefixBackref) + options = None + return options + + def replaceString(self): + '''Replaces a regular expression in a given string by a replacement and display it. + ''' + self._resultLines = [] + what = self.shiftProgramArgument() + replacement = self.shiftProgramArgument() + inputString = self.shiftProgramArgument() + if inputString is None: + self.abort('too few arguments') + elif self.handleOptions(): + self._processor = base.TextProcessor.TextProcessor(self._logger) + options = self.replaceOptions(False) + if options is not None: + self._processor.setContent(inputString) + self._processor.replace(what, replacement, options.prefixBackref, + options.rawString, True, options.wordOnly, options.ignoreCase, options.escActive) + info = '\n'.join(self._processor._lines) + self._resultText = info + print(info) + + def run(self): + '''Implements the tasks of the application + ''' + self._hostname = self._configuration.getString('hostname', '') + if self._mainMode == 'exec-rules': + self.execRules() + elif self._mainMode == 'describe-rules': + self.describeRules() + elif self._mainMode == 'csv-execute': + self.csvExecute() + elif self._mainMode == 'describe-csv': + base.CsvProcessor.CsvProcessor.describe() + elif self._mainMode == 'grep': + self.grep() + elif self._mainMode == 'replace': + self.replace() + elif self._mainMode == 'many-replacements': + self.replaceMany() + elif self._mainMode == 'string-replacement': + self.replaceString() + elif self._mainMode == 'insert-or-replace': + self.insertOrReplace() + else: + self.abort('unknown mode: ' + self._mainMode) + + +def main(args): + '''Main function. + @param args: the program arguments + ''' + snakeboxx.startApplication() + application = TextApp(args) + application.main() + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/base/BaseLogger.py b/base/BaseLogger.py new file mode 100644 index 0000000..af8c831 --- /dev/null +++ b/base/BaseLogger.py @@ -0,0 +1,109 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import base.Const + +class BaseLogger: + '''Base class of the loggers. + The derived class must implement the method log(message) + ''' + + def __init__(self, verboseLevel): + '''Constructor. + @param verboseLevel: logging is done only if minLevel < verboseLevel. minLevel is a parameter of log() + ''' + self._verboseLevel = verboseLevel + self._logDebug = True + self._logInfo = True + self._errors = 0 + self._maxErrors = 20 + self._firstErrors = [] + self._errorFilter = None + self._mirrorLogger = None + self._inUse = False + + def debug(self, message): + '''Logs a debugging message. + @param message: the message to log + @return: True + ''' + if self._mirrorLogger is not None: + self._mirrorLogger.debug(message) + if self._logDebug: + self._inUse = True + self.log(message) + self._inUse = False + return True + + def error(self, message): + '''Logs a message. + @param message: the error message to log + @return: False + ''' + if self._mirrorLogger is not None: + self._mirrorLogger.error(message) + filtered = self._errorFilter is not None + if filtered: + if isinstance(self._errorFilter, str): + filtered = message.find(self._errorFilter) >= 0 + else: + filtered = self._errorFilter.search(message) is not None + if not filtered: + self._inUse = True + self.log('+++ ' + message) + self._errors += 1 + if self._errors < self._maxErrors: + self._firstErrors.append(message) + self._inUse = False + return False + + def info(self, message): + '''Logs an info message. + @param message: the message to log + @return: True + ''' + if self._mirrorLogger is not None: + self._mirrorLogger.info(message) + if self._logInfo: + self._inUse = True + self.log(message) + self._inUse = False + return True + + def log(self, message=None, minLevel=base.Const.LEVEL_SUMMARY): + '''Logs a message. + @param message: the string to log + @param level: logging will be done only if level >= self._verboseLevel + ''' + raise NotImplementedError('BaseLogger.log(): must be overriden') + + def setMirror(self, logger): + '''Sets a "mirror" logger: all messages are logged to the mirror too + @param logger: the mirror logger + ''' + if self._mirrorLogger is not None: + logger.setLogger(self._mirrorLogger) + self._mirrorLogger = logger + + def setErrorFilter(self, excluded, mirrorsToo=True): + '''Sets the error filter: if the pattern matches the error is ignored (not logged) + @param excluded: string: a substring of the ignored error + re.RegExpression: a compiled regular expression of the ignored errors + @param mirrorsToo: True: the filter is used for the mirror loggers too + ''' + self._errorFilter = excluded + if mirrorsToo and self._mirrorLogger is not None: + self._mirrorLogger.setErrorFilter(excluded) + + def transferErrors(self, logger): + '''Transfers the error from another logger. + @param logger: the source of the errors to transfer + ''' + self._errors += logger._errors + self._firstErrors += logger._firstErrors + + +if __name__ == '__main__': + pass diff --git a/base/BaseRandom.py b/base/BaseRandom.py new file mode 100644 index 0000000..91935c4 --- /dev/null +++ b/base/BaseRandom.py @@ -0,0 +1,119 @@ +''' +Created on 2023 Feb 26 + +@author: wk +''' +import random +import datetime +import random + +class BaseRandom(object): + ''' + A base class for random generators. + ''' + _maxInt = 0x7ffffffe + def __init__(self): + ''' + Constructor + ''' + self._lastSeed = 0 + + def nextBool(self) -> bool: + '''Returns a random bool value. + @return a random bool value + ''' + rc = self.nextInt(BaseRandom._maxInt) >= BaseRandom._maxInt/2 + return rc + + def nextFloat(self) -> float: + '''Returns a random float number from [0..1.0]. + ''' + rc = self.nextInt(BaseRandom._maxInt) / BaseRandom._maxInt + return rc + + def nextInt(self, value1: int=0x7ffffffe, value2: int=None) -> int: + '''Returns a random integer from [0..maxValue] + @param value1: if value2 is None this is the largest return value (inclusive) + Otherwise this is the lowest return value + @param value2: None or the largest return value (inclusive) + @return a random number + ''' + if value2 is None: + rc = random.randint(0, min(value1, BaseRandom._maxInt)) + else: + if value1 > value2: + value1, value2 = (value2, value1) + range = value2 - value1 + if range <= 0 or range > BaseRandom._maxInt: + raise ValueError(f'BaseRandom::nextInt(): range too large: {range} / {BaseRandom._maxInt}') + rc = value1 + self.nextInt(range) + return rc + + @staticmethod + def hashString(text: str) -> int: + '''A portable hash function for strings. + This algorithm can be implemented in any programming environment: + Only 31 bit integer operations will be done and multiplications are + done in the 31 bit range. + @param text: the text to hash + @return: a hash value + ''' + p = 31 + m = 0x7fffffff + maxFactor = 0x10000 + hash = 0x7eadbeef + powerValue = 1 + for ix in range(len(text)): + cc = ord(text[ix]) + hash += cc + hash = (hash % maxFactor * powerValue + (hash / maxFactor) \ + * maxFactor) % m + powerValue = powerValue * p % maxFactor + return hash % m + + def setPhrase(self, text1: str, text2: str=None, text3: str=None): + '''Sets the generator state with until three secret phrases. + @param text1: the first secret + @param text2: the second secret + @param text3: the third secret + ''' + seed2 = 19891211 if text2 is None else BaseRandom.hashString(text2) + seed3 = 19910420 if text3 is None else BaseRandom.hashString(text2) + self.setSeed(BaseRandom.hashString(text1), seed2, seed3) + + def setSeed(self, seed1: int, seed2: int = 0x7654321, seed3: int = 0x3adf001): + '''Sets the generator state with until three secret integers. + @param seed1: the first secret + @param seed2: the second secret + @param seed3: the third secret + ''' + self._lastSeed = seed1 + seed2 + seed3 + random.seed(self._lastSeed) + + @staticmethod + def testDistribution(random, maxInt: int, loops: int=5000): + '''Shows the distribution of the random generator as ASCII graphic. + @param random: the random generator to test + @param maxInt: the largest generated value + @param loops: the number of random calls + ''' + parts = 20 + divider = maxInt / parts + data = [] + for ix in range(parts): + data.append(0) + start = datetime.datetime.now() + for ix in range(loops): + value = int(random.nextInt(maxInt) / divider) + data[value] += 1 + end = datetime.datetime.now() + minimum = min(data) + maximum = max(data) + for ix in range(len(data)): + value = 50 * data[ix] // maximum + print(f'{ix:02d}: {data[ix]:5d} {"*"*value}') + diff = end - start + seconds = diff.seconds + diff.microseconds * 1E-6 + ratio = loops / seconds * 1E-6 + print(f'= maxInt: {maxInt} min: {minimum} max: {maximum} calls/microseconds: {ratio:.3}') + diff --git a/base/Const.py b/base/Const.py new file mode 100644 index 0000000..bbf91b3 --- /dev/null +++ b/base/Const.py @@ -0,0 +1,24 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +# replaces re.I: +IGNORE_CASE = 2 +RE_UNICODE = 32 +RE_TEMPLATE = 1 # template mode (disable backtracking) +RE_IGNORECASE = 2 # case insensitive +RE_LOCALE = 4 # honour system locale +RE_MULTILINE = 8 # treat target as multiline string +RE_DOTALL = 16 # treat target as a single string +RE_UNICODE = 32 # use unicode "locale" +RE_VERBOSE = 64 # ignore whitespace and comments +RE_DEBUG = 128 # debugging +RE_ASCII = 256 # use ascii "locale" + +# base.Logger levels: +LEVEL_SUMMARY = 1 +LEVEL_DETAIL = 2 +LEVEL_LOOP = 3 +LEVEL_FINE = 4 +LEVEL_DEBUG = 5 diff --git a/base/CryptoEngine.py b/base/CryptoEngine.py new file mode 100644 index 0000000..6a4faa8 --- /dev/null +++ b/base/CryptoEngine.py @@ -0,0 +1,408 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import random +import base64 +import math +import time + + +class CryptoEngine: + '''Implements a Pseudo Random Generator with the KISS algorithm. + We want an algorithm which can be implemented in any programming language, e.g. in JavaScript or Java. + JavaScript (at this moment) only contains floating point calculation. + Java knows only signed integers or floating point numbers. + Therefore we use IEEE 754 (64 bit floating point). + ''' + + def __init__(self, logger): + '''Constructor. + @param logger: the logger + ''' + self._counter = 0 + self._base64Trailer = '!#$%&()*' + self._uBoundBase64Tail = '*' + self._x = 372194.0 + # @cond _y != 0 + self._y = 339219.0 + # @cond z | c != 0 + self._z = 470811222.0 + self._c = 1.0 + self._logger = logger + + def bytesToString(self, aBytes): + '''Converts a string into a byte array without encoding. + @param aBytes: byte array to convert + @return a string + ''' + try: + rc = aBytes.decode('ascii') + except UnicodeDecodeError as exc: + rc = -1 + raise exc + return rc + + def decode(self, string, charSet): + '''Decodes a string encoded by encode(). + Format of the string: version salt encrypted + '0' (version string) + 4 characters salt + rest: the encrypted string + @param string: string to encode + @param charSet: the character set of the string and the result, e.g. 'word' + @return: the decoded string (clear text) + ''' + self._counter += 1 + aSet = self.getCharSet(charSet) + aSize = len(aSet) + rc = '' + if string.startswith('0'): + prefix = string[1:5] + string = string[5:] + aHash = self.hash(prefix) + self.setSeed(aHash, 0x20111958, 0x4711, 1) + length = len(string) + for ix in range(length): + ix3 = aSet.find(string[ix]) + ix2 = (aSize + ix3 - self.nextInt(aSize - 1)) % aSize + rc += aSet[ix2] + return rc + + def decodeBinary(self, string): + '''Decodes a string encrypted by encryptBinary(). + @param string: string to decode + @return: the decoded string (clear text) + ''' + aSet = self.getCharSet('base64') + aSize = len(aSet) + rc = '' + if string.startswith('0'): + prefix = string[1:5] + string = string[5:] + aHash = self.hash(prefix) + self.setSeed(aHash, 0x20111958, 0x4711, 1) + aLen = len(string) + buffer = '' + # replace the trailing '=' "randomly" with a char outside the + # character set: + if aLen > 0 and string[aLen - 1] == '=': + string[aLen - 1] = self._base64Trailer[self._counter * 7 % + len(self._base64Trailer)] + if aLen > 1 and string[aLen - 2] == '=': + string[aLen - 2] = self._base64Trailer[self._counter * 13 % + len(self._base64Trailer)] + for ix in range(aLen): + ix3 = aSet.find(string[ix]) + ix2 = (aSize + ix3 - self.nextInt(aSize - 1)) % aSize + buffer += aSet[ix2] + binBuffer = self.stringToBytes(buffer + '\n') + try: + binBuffer2 = base64.decodebytes(binBuffer) + except Exception as exc: + if str(exc) == 'Incorrect padding': + try: + binBuffer = binBuffer[0:-1] + binBuffer2 = base64.decodebytes(binBuffer) + except Exception: + binBuffer = binBuffer[0:-1] + binBuffer2 = base64.decodebytes(binBuffer) + ix = binBuffer2.find(b'\n') + if ix >= 0: + binBuffer2 = binBuffer2[0:ix] + rc = self.bytesToString(binBuffer2) + return rc + + def encode(self, string, charSet): + '''Encodes a string with a randomly generated salt. + Format of the string: version salt encoded + '0' (version string) + 4 characters salt + rest: the encoded string + @param string: string to encode + @param charSet: the character set of the string and the result, e.g. 'word' + @return: the encrypted string + ''' + self._counter += 1 + self.setSeedRandomly() + rc = self.nextString(4, charSet) + aSet = self.getCharSet(charSet) + aSize = len(aSet) + aHash = self.hash(rc) + self.setSeed(aHash, 0x20111958, 0x4711, 1) + length = len(string) + for ix in range(length): + ix3 = aSet.find(string[ix]) + ix2 = (ix3 + self.nextInt(aSize - 1)) % aSize + rc += aSet[ix2] + return '0' + rc + + def encodeBinary(self, string): + '''Encrypts a string with a randomly generated salt. + The string can be based on any char set. It will be base64 encoded before encryption. + Format of the result: version salt encrypted + '0' (version string) + 4 characters salt + rest: the encrypted string + @param string: the string or bytes to encrypt + @return: the encoded string + ''' + self.setSeedRandomly() + if isinstance(string, str): + string = self.stringToBytes(string) + # convert it to a ascii usable string + string += b'\n' + buffer = base64.encodebytes(string) + string = self.bytesToString(buffer).rstrip() + rc = self.nextString(4, 'base64') + aSet = self.getCharSet('base64') + aSize = len(aSet) + aHash = self.hash(rc) + self.setSeed(aHash, 0x20111958, 0x4711, 1) + length = len(string) + for ix in range(length): + ix3 = aSet.find(string[ix]) + ix2 = (ix3 + self.nextInt(aSize - 1)) % aSize + rc += aSet[ix2] + return '0' + rc + + def getCharSet(self, name): + '''Returns a string with all characters of the charset given by name. + @param name: the name of the charset + @return: None: unknown charset + otherwise: the charset as string + ''' + if name == 'dec': + rc = '0123456789' + elif name == 'hex': + rc = '0123456789abcdef' + elif name == 'upper': + rc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + elif name == 'lower': + rc = 'abcdefghijklmnopqrstuvwxyz' + elif name == 'alfa': + rc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + elif name == 'word': + rc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_' + elif name == 'ascii94': + rc = r'''!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''' + elif name == 'ascii95': + rc = r''' !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''' + elif name == 'ascii': + rc = r''' !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''' + chr( + 127) + elif name == 'base64': + rc = r'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + else: + self._logger.error('unknown character set: ' + name) + rc = '' + + return rc + + def getCharSetNames(self): + '''Returns the list of the known charset names. + @return the list of the known charset names + ''' + rc = [ + 'dec', + 'hex', + 'upper', + 'lower', + 'alfa', + 'word', + 'ascii94', + 'ascii95', + 'ascii', + 'base64'] + return rc + + def hash(self, string): + '''Converts a string into an integer. + @param string: the string to convert + @return: the hash value + ''' + rc = len(string) + count = rc + for ix in range(count): + rc = (rc * (ix + 1) + + (ord(string[ix]) << (ix % 4 * 7))) & 0x7fffffff + return rc + + def nextChar(self, charSet='ascii'): + '''Returns a pseudo random character. + @param charSet: the result is a character from this string + @return: a pseudo random character + ''' + aSet = self.getCharSet(charSet) + ix = self.nextInt(0, len(aSet) - 1) + rc = aSet[ix] + return rc + + def nextInt(self, maxValue=0x7fffffff, minValue=0): + '''Returns a pseudo random 31 bit integer. + @param maxValue: the maximal return value (inclusive) + @param minValue: the minimal return value (inclusive) + @return: a number from [minValue..maxValue] + ''' + if maxValue == minValue: + rc = minValue + else: + if minValue > maxValue: + maxValue, minValue = minValue, maxValue + rc = self.nextSeed() + rc = rc % (maxValue - minValue) + minValue + return rc + + def nextString(self, length, charSet): + '''Returns a pseudo random string. + @param length: the length of the result + @param charSet: all characters of the result are from this string + @return: a pseudo random string with the given charset and length + ''' + aSet = self.getCharSet(charSet) + aSize = len(aSet) + rc = '' + aRandom = None + for ix in range(length): + if ix % 4 == 0: + aRandom = self.nextSeed() + else: + aRandom >>= 8 + rc += aSet[aRandom % aSize] + return rc + + def nextSeed(self): + '''Sets the next seed and returns a 32 bit random value. + @return: a pseudo random number with 0 <= rc <= 0xffffffff + ''' + # linear congruential generator (LCG): + self._x = math.fmod(69069.0 * self._x + 473219.0, 4294967296) + # Xorshift + #self._y ^= int(self._y) << 13 + self._y = math.fmod(int(self._y) ^ int(self._y) << 13, 4294967296) + #self._y ^= self._y >> 17 + self._y = math.fmod(int(self._y) ^ int(self._y) >> 17, 4294967296) + #self._y ^= self._y << 5 + self._y = math.fmod(int(self._y) ^ int(self._y) << 5, 4294967296) + # multiply with carry: + t = 698769069.0 * self._z + self._c + #self._c = math.fmod(t >> 32, 2) + self._c = math.fmod(int(t) >> 32, 2) + self._z = math.fmod(t, 4294967296) + return int(math.fmod(self._x + self._y + self._z, 4294967296)) + + def oneTimePad(self, user, data): + '''Builds a one time pad. + @param user: the user id + @param data: None or additional data: allowed char set: word + @return: char set: word + ''' + if data is not None and self.testCharSet(data, 'word') >= 0: + rc = '' + else: + padData = '{:08x}{:04x}'.format( + int(round(time.time())), user) + data + rc = self.encode(padData, 'word') + return rc + + def restoreSeed(self, seed): + '''Returns the current seed as string. + @return the seed as string + ''' + parts = seed.split(':') + self.setSeed(float(parts[0]), float(parts[1]), + float(parts[2]), float(parts[3])) + + def saveSeed(self): + '''Returns the current seed as string. + @return the seed as string + ''' + rc = '{}:{}:{}:{}'.format(repr(self._x), repr( + self._y), repr(self._z), repr(self._c)) + return rc + + def setSeed(self, x, y, z, c): + '''Sets the parameter of the KISS algorithm. + @param x: + @param y: + @param z: + @param c: + ''' + self._x = math.fmod(x, 4294967296) + self._y = 1234321.0 if y == 0 else math.fmod(y, 4294967296) + if z == 0 and c == 0: + c = 1.0 + self._c = math.fmod(c, 2) + self._z = math.fmod(z, 4294967296) + + def setSeedFromString(self, seedString): + '''Converts a string, e.g. a password, into a seed. + @param seedString: the string value to convert + ''' + if seedString == '': + seedString = 'Big-Brother2.0IsWatching!You' + while len(seedString) < 8: + seedString += seedString + x = self.hash(seedString[0:len(seedString) - 3]) + y = self.hash(seedString[1:8]) + z = self.hash(seedString[3:5]) + c = self.hash(seedString[1:]) + self.setSeed(x, y, z, c) + + def setSeedRandomly(self): + '''Brings "true" random to the seed + ''' + utime = time.time() + rand1 = int(math.fmod(1000 * 1000 * utime, 1000000000.0)) + rand2 = int(math.fmod(utime * 1000, 1000000000.0)) + self.setSeed(rand1, rand2, int(random.random() * 0x7fffffff), 1) + + def stringToBytes(self, string): + '''Converts a string into a byte array without encoding. + @param string: string to convert + @return a bytes array + ''' + rc = string.encode('ascii') + return rc + + def testCharSet(self, string, charSet): + '''Tests whether all char of a string belong to a given charSet. + @param string: string to test + @param charSet: the char set to test + @return: -1: success + otherwise: the index of the first invalid char + ''' + aSet = self.getCharSet(charSet) + rc = -1 + for ix, item in enumerate(string): + if aSet.find(item) < 0: + rc = ix + break + return rc + + def unpackOneTimePad(self, pad, maxDiff=60): + '''Decodes a one time pad. + @param pad: the encoded one time pad + @param maxDiff: maximal difference (in seconds) between time of the pad and now + @return: None: invalid pad + otherwise: a tuple (time, user, data) + ''' + padData = self.decode(pad, 'word') + length = len(padData) + if length < 12 or self.testCharSet(padData[0:12], 'hex') >= 0 or self.testCharSet(padData[12:], 'word') >= 0: + rc = None + else: + padTime = int(padData[0:8], 16) + now = time.time() + if abs(now - padTime) >= maxDiff: + rc = None + else: + user = int(padData[8:12], 16) + data = None if len(padData) == 12 else padData[12:] + rc = (padTime, user, data) + return rc + + +if __name__ == '__main__': + pass diff --git a/base/CsvProcessor.py b/base/CsvProcessor.py new file mode 100644 index 0000000..9233183 --- /dev/null +++ b/base/CsvProcessor.py @@ -0,0 +1,393 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import re +import os.path +import csv +import datetime +import fnmatch + +import base.Const +import base.StringUtils + +class CsvProcessor: + '''A processor for finding/modifying text. + ''' + + def __init__(self, logger): + self._filename = None + self._lines = None + self._logger = logger + self._colNames = None + # data type of one row, e.g. [str, int, None] + self._dataTypes = [] + # bool flags of one row: True: any row has null (empty value) + self._hasEmpty = [] + self._rows = [] + self._indexes = None + self._minCols = 0x7fffffff + self._rowMinCols = None + self._maxCols = 0 + self._rowMaxCols = None + self._columnOrder = None + self._dialect = None + + def addColumn(self, header, colIndex, value1, value2): + '''Adds a CSV column to the internal structure. + @param header: '' or the column header, e.g. 'name' + @param colIndex: the index of the column after inserting. '0' means: the new column is the first + @param value1: the column value in row[0] + @param value2: None: all column values are set to value1 + Otherwise: the column value of the 2nd row. all other values will be interpolated + ''' + index = base.StringUtils.asInt(colIndex) + if index is None: + self._logger.error(' is not an integer: ' + colIndex) + else: + index = index if index < len(self._rows[0]) else len(self._rows[0]) + value = value1 + step = None + if value2 is not None: + v1 = base.StringUtils.asInt(value1) + v2 = base.StringUtils.asInt(value2) + if v1 is None or v2 is None: + self._logger.error( + 'cannot interpolate {}..{}'.format(value1, value2)) + index = None + else: + step = v2 - v1 + value = v1 + if index is not None and header == '' and self._colNames is not None: + self._logger.error( + 'missing
in add-column command: may not be empty') + index = None + if index is not None: + if self._colNames is not None: + self._colNames.insert(index, header) + self._rows[0].insert(index, value) + for ixRow in range(1, len(self._rows)): + if step is not None: + value += step + self._rows[ixRow].insert(index, value) + @staticmethod + def dataType(string): + '''Returns the data type of the given string. + @param string: string to inspect + @return the data type: None, str int + ''' + if string is None or string == '': + rc = None + elif base.StringUtils.asInt(string) is not None: + rc = int + else: + rc = str + return rc + + __description = '''= Commands of the CsvProcessor: +commands: [...] +add-column:
,,[,] +
: name of the column + : 0..N: 0 means the new column is the first column + : the value of the first row (below the header) + : if it does not exists: all other rows are set to + if it exists: the second row is set to this value and the next values are interpolated +info:[,...] + Note: only filtered column will be respected + : summary | min | max | sorted | unique | multiple +set-filter:[,[,[,[,[,]] + If is empty, the current filename is taken. + : the delimiter between two columns: comma | tab | semicolon + If empty the current is taken. + : if given and a already exists: it is renamed + with this extension. May contain placeholders %date%, %datetime% or %seconds%. +Example: +set-filter:name,prename info:summary,unique set-order:nam*,*pren* write:names.csv,tab,.%date% +''' + @staticmethod + def describe(): + '''Prints a description of the commands. + ''' + print(CsvProcessor.__description) + + def execute(self, commands): + '''Executes a sequence of commands. + @see describe() + @param commands: the command sequence as string + ''' + for command in commands.split(): + name, delim, args = command.partition(':') + if name == '' or name.startswith('#'): + continue + if delim != ':': + self._logger.error('missing ":" in ' + command) + break + arguments = args.split(',') + if name == 'add-column': + if len(arguments) < 3: + self._logger.error( + 'missing arguments for add-column:
,, expected') + else: + self.addColumn(arguments[0], arguments[1], arguments[2], None if len( + arguments) < 4 else arguments[3]) + elif name == 'info': + self.info(args) + elif name == 'set-filter': + if base.StringUtils.asInt(arguments[0]) is not None: + self.setFilterIndexes(arguments) + else: + self.setFilterCols(arguments) + elif name == 'set-order': + self.setColumnOrder(arguments) + elif name == 'write': + ext = None if len(arguments) < 3 else arguments[2] + delim = None if len(arguments) < 2 else arguments[1] + if delim == 'comma': + delim = ',' + elif delim == 'semicolon': + delim = ';' + elif delim == 'tab': + delim = '\t' + fn = None if arguments[0] == '' else arguments[0] + self.writeFile(fn, ext, delim) + + def info(self, what): + '''Prints some infos about the CSV file. + Note: only columns listed in self._indexes are inspected. + @param what: a comma separated list of requests, e.g. 'unique,sorted,summary,min,max' + ''' + def prefix(col): + return '{}{}: '.format(col, (' "' + self._colNames[col] + '"') if col < len(self._colNames) else '') + + if re.search(r'unique|sorted|multiple|min|max|max-length', what) is not None: + unique = what.find('unique') >= 0 + multiple = what.find('multiple') >= 0 + for index in self._indexes: + cols = [] + for row in self._rows: + value = row[index] + if self._dataTypes == int: + cols.append(int(value)) + else: + cols.append(value) + cols.sort() + self._logger.log('== {}'.format(prefix(index))) + if what.find('min') >= 0: + self._logger.log('minimum: {}'.format(cols[0])) + if re.search(r'(max[^-])|(max$)', what) is not None: + self._logger.log('maximum: {}'.format(cols[-1])) + if what.find('max-length') >= 0: + maxLength = 0 + maxString = None + for row in self._rows: + current = len(row[index]) + if current > maxLength: + maxLength = current + maxString = row[index] + if maxString is not None: + self._logger.log( + 'max-length: {} "{}"'.format(maxLength, maxString)) + if unique or what.find('sorted') >= 0: + lastValue = None + for col in cols: + if unique and col == lastValue: + continue + self._logger.log(col) + lastValue = col + elif multiple: + lastValue = None + lastCount = 0 + for col in cols: + if col == lastValue: + lastCount += 1 + else: + if lastValue is not None and lastCount > 0: + self._logger.log('{}: {}'.format( + lastValue, lastCount + 1)) + lastValue = col + if lastCount > 0: + self._logger.log('{}: {}'.format( + lastValue, lastCount + 1)) + if what.find('summary') >= 0: + info = '== summary:\nRows: {}\nCols: {}\nHeaders: {} line(s)'.format( + len(self._rows), len(self._rows[0]), 1 if self._colNames else 0) + info += (f'\ndelimiter: {self._dialect.delimiter}\ndoublequote: {self._dialect.doublequote}' + + f'\nescapechar: {self._dialect.escapechar}\nquotechar: {self._dialect.quotechar}') + self._logger.log(info) + for col in range(len(self._rows[0])): + info = '{} {} {}'.format(prefix(col), + str(self._dataTypes[col]), 'hasEmpty' if self._hasEmpty[col] else '') + self._logger.log(info) + + def readFile(self, filename, mustExists=True): + '''Reads a file into the internal buffer. + @param filename: the file to read + @param mustExists: True: errros will be logged + @return True: success False: cannot read + ''' + self._filename = filename + rc = os.path.exists(filename) + if not rc: + if mustExists: + self._logger.error('{} does not exists'.format(filename)) + else: + with open(filename, newline='') as csvfile: + sniffer = csv.Sniffer() + buffer = csvfile.read(16000) + self._dialect = sniffer.sniff(buffer) + hasHeaders = sniffer.has_header(buffer) + csvfile.seek(0) + reader = csv.reader(csvfile, self._dialect) + self._colNames = None + ix = -1 + for row in reader: + ix += 1 + if ix == 0 and hasHeaders: + self._colNames = row + else: + self._rows.append(row) + currentLength = len(row) + if currentLength < self._minCols: + self._minCols = currentLength + self._rowMinCols = reader.line_num + if currentLength > self._maxCols: + self._maxCols = currentLength + self._rowMaxCols = reader.line_num + if not self._dataTypes: + for col in row: + currentType = CsvProcessor.dataType(col) + self._dataTypes.append(currentType) + self._hasEmpty.append(currentType is None) + else: + for ix, col in enumerate(row): + currentType = self.dataType(col) + if ix >= len(self._dataTypes): + self._dataTypes.append(currentType) + self._hasEmpty.append(currentType is None) + else: + if currentType is None: + self._hasEmpty[ix] = True + if self._dataTypes[ix] == int and currentType is not None and currentType != int: + self._dataTypes[ix] = str + return rc + + def setColumnOrder(self, patterns): + '''Sets the filter indexes by column name patterns. + @param patterns: a list of column name patterns, e.g. ['*name*', 'ag*'] + ''' + self._columnOrder = [] + for pattern in patterns: + found = False + ix = -1 + for name in self._colNames: + ix += 1 + if fnmatch.fnmatch(name, pattern): + found = True + self._columnOrder.append(ix) + self._logger.log('pattern {} found as column {} at index {}'.format( + pattern, name, ix), base.Const.LEVEL_FINE) + break + if not found: + self._logger.error('pattern {} not found') + + def setFilterIndexes(self, indexes): + '''Sets the filter indexes by indexes. + @param indexes: a list of indexes. May be strings or integers like ['0', 2] + ''' + self._indexes = [] + for ix in indexes: + self._indexes.append(int(ix)) + self._indexes.sort() + + def setFilterCols(self, patterns): + '''Sets the filter indexes by column name patterns. + @param patterns: a list of column name patterns, e.g. ['*name*', 'ag*'] + ''' + self._indexes = [] + for pattern in patterns: + found = False + ix = -1 + for name in self._colNames: + ix += 1 + if fnmatch.fnmatch(name, pattern): + found = True + self._indexes.append(ix) + self._logger.log('pattern {} found as column {} at index {}'.format( + pattern, name, ix), base.Const.LEVEL_FINE) + break + if not found: + self._logger.error('col name pattern {} not found in [{}]'.format( + pattern, ','.join(self._colNames))) + break + self._indexes.sort() + + def quoteString(self, string): + '''Quotes the given string if necessary. + @param string: the string to quote + @return: string or the quoted string + ''' + rc = string + quote = self._dialect.quotechar + delim = self._dialect.delimiter + forceQuoting = self._dialect.quoting == csv.QUOTE_ALL or ( + self._dialect.quoting == csv.QUOTE_NONNUMERIC and base.StringUtils.asInt(string) is None) + if forceQuoting or string.find(delim) >= 0: + if self._dialect.doublequote: + string = string.replace(delim, delim + delim) + else: + esc = self._dialect.escapechar + string = string.replace( + esc, esc + esc).replace(quote, esc + quote) + rc = quote + string + quote + return rc + + def writeFile(self, filename=None, backupExtension=None, delimiter=None): + '''Writes the internal buffer as a file. + @param filename: the file to write: if None _filename is taken + @param backupExtension: None or: if the file already exists it will be renamed with this extension + macros: '%date%' replace with the current date %datetime%: replace with the date and time + '%seconds%' replace with the seconds from epoc + ''' + filename = self._filename if filename is None else filename + delimiter = self._dialect.delimiter if delimiter is None else delimiter + if os.path.exists(filename) and backupExtension is not None: + if backupExtension.find('%') >= 0: + now = datetime.datetime.now() + backupExtension = backupExtension.replace( + '%date%', now.strftime('%Y.%m.%d')) + backupExtension = backupExtension.replace( + '%datetime%', now.strftime('%Y.%m.%d-%H_%M_%S')) + backupExtension = backupExtension.replace( + '%seconds%', now.strftime('%a')) + if not backupExtension.startswith('.'): + backupExtension = '.' + backupExtension + parts = base.FileHelper.splitFilename(filename) + parts['ext'] = backupExtension + newNode = parts['fn'] + backupExtension + base.FileHelper.deepRename(filename, newNode, deleteExisting=True) + with open(filename, "w") as fp: + indexes = self._columnOrder if self._columnOrder is not None else [ + ix for ix in range(len(self._rows[0]))] + if self._colNames is not None: + line = '' + for ix, item in enumerate(indexes): + if ix > 0: + line += delimiter + line += self.quoteString(self._colNames[item]) + fp.write(line + self._dialect.lineterminator) + for row in self._rows: + line = '' + for ix, item in enumerate(indexes): + if ix > 0: + line += delimiter + value = base.StringUtils.toString(row[item]) + line += self.quoteString(value) + fp.write(line + self._dialect.lineterminator) + + +if __name__ == '__main__': + pass diff --git a/base/DirTraverser.py b/base/DirTraverser.py new file mode 100644 index 0000000..fcb8b92 --- /dev/null +++ b/base/DirTraverser.py @@ -0,0 +1,267 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import fnmatch +import os.path +import stat +import re +import datetime + +#import base.FileHelper +import base.Const +import base.StringUtils + + +class DirTraverser: + '''Traverse a directory tree and return specified filenames. + ''' + + def __init__(self, directory, filePattern='*', dirPattern='*', + reFileExcludes=None, reDirExcludes=None, fileType=None, + minDepth=0, maxDepth=None, fileMustReadable=False, fileMustWritable=False, + dirMustWritable=False, maxYields=None, youngerThan=None, olderThan=None, + minSize=0, maxSize=None): + '''Constructor. + @param directory: the start directory + @param filePattern: the shell pattern of the files to find + @param dirPattern: the shell pattern of the dirs to find. Note: selects only the returned dirs, not the processed + @param reFileExcludes: None or the regular expression for files to exclude + @param reDirExcludes: None or the regular expression for directories to exclude + @param nodeType: a string with the filetypes to find: d(irectory) f(ile) l(ink), e.g. 'dfl' + @param minDepth: the minimum depth of the processed files: 1 means only files in subdirectories will be found + @param maxDepth: the maximum depth of the processed files: 0 means only the base directory is scanned + @param fileMustReadable: True: a file will only yielded if it is readable + @param fileMustWritable: True: a file will only yielded if it is writable + @param dirMustWritable: True: a directory will only processed if it is writable + @param maxYields: None or after this amount of yields the iteration stops + @param youngerThan: None or the modification file time must be greater or equal this + @param olderThan: None or the modification file time must be lower or equal this + @param minSize: None or only files larger than that will be found + @param maxSize: None or only files smaller than that will be found + ''' + self._directory = directory if directory != '' else '.' + # +1: the preceeding slash + self._lengthDirectory = 0 if directory == os.sep else len( + self._directory) + 1 + if fileType is None: + fileType = 'dfl' + self._findFiles = fileType.find('f') >= 0 + self._findDirs = fileType.find('d') >= 0 + self._findLinks = fileType.find('l') >= 0 + self._filePattern = filePattern + self._dirPattern = dirPattern + self._reDirExcludes = reDirExcludes + self._reDirExcludes = None if reDirExcludes is None else ( + re.compile(reDirExcludes, base.Const.IGNORE_CASE) if isinstance(reDirExcludes, str) else + reDirExcludes) + self._reFileExcludes = None if reFileExcludes is None else ( + re.compile(reFileExcludes, base.Const.IGNORE_CASE) if isinstance(reFileExcludes, str) else + reFileExcludes) + self._fileMustReadable = fileMustReadable + self._fileMustWritable = fileMustWritable + self._dirMustWritable = dirMustWritable + self._minSize = minSize if minSize is not None else 0 + # 2**63-1: + self._maxSize = maxSize + # the stat info about the current file + self._fileInfo = None + self._fileFullName = None + self._fileNode = None + self._fileRelativeName = None + # the stat info about the current directory + self._dirInfo = None + self._dirFullName = None + self._dirRelativeName = None + self._dirNode = None + self._minDepth = minDepth if minDepth is not None else 0 + self._maxDepth = maxDepth if maxDepth is not None else 900 + self._youngerThan = None if youngerThan is None else youngerThan.timestamp() + self._olderThan = None if olderThan is None else olderThan.timestamp() + # 2.14*10E9 + self._maxYields = maxYields if maxYields is not None else 0x7fffffff + self._yields = 0 + self._countFiles = 0 + self._bytesFiles = 0 + self._countDirs = 0 + self._ignoredDirs = 0 + self._ignoredFiles = 0 + self._euid = os.geteuid() + self._egid = os.getegid() + self._statInfo = None + self._node = None + self._isDir = False + + def asList(self): + '''Returns a list of all found files (filenames with relative path). + @return: []: nothing found otherwise: the list of all filenames + ''' + rc = [] + for item in self.next(self._directory, 0): + rc.append(item) + return rc + + def next(self, directory, depth): + '''Implements a generator which returns the next specified file. + Note: this method is recursive (for each directory in depth) + The traversal mode: yields all files and directories and than enters the recursivly the directories + @param directory: the directory to process + @param depth: the current file tree depth + @return: the next specified file (the filename with path relative to _directory) + ''' + self._countDirs += 1 + dirs = [] + directory2 = '' if directory == os.sep else directory + for node in os.listdir(directory): + full = directory2 + os.sep + node + self._statInfo = statInfo = os.lstat(full) + self._isDir = stat.S_ISDIR(statInfo.st_mode) + if self._isDir: + self._ignoredDirs += 1 + if (self._reDirExcludes is not None and self._reDirExcludes.search(node)): + continue + if not base.LinuxUtils.isReadable(statInfo, self._euid, self._egid): + continue + elif self._dirMustWritable and not base.LinuxUtils.isReadable(statInfo, self._euid, self._egid): + continue + elif depth < self._maxDepth: + dirs.append(node) + if depth < self._minDepth: + continue + if depth < self._maxDepth: + self._ignoredDirs -= 1 + if self._dirPattern is None: + continue + if not self._findDirs: + continue + if self._dirPattern == '*' or fnmatch.fnmatch(node, self._dirPattern): + self._dirInfo = statInfo + self._dirNode = node + self._dirFullName = full + self._dirRelativeName = full[self._lengthDirectory:] + yield self._dirRelativeName + self._yields += 1 + if self._yields >= self._maxYields: + return + else: + # tentative: because of "continue" + self._ignoredFiles += 1 + if os.path.islink(full): + if not self._findLinks: + continue + elif not self._findFiles: + continue + if statInfo.st_size < self._minSize or (self._maxSize is not None and statInfo.st_size > self._maxSize): + continue + if self._youngerThan is not None and statInfo.st_mtime < self._youngerThan: + continue + if self._olderThan is not None and statInfo.st_mtime > self._olderThan: + continue + if self._filePattern != '*' and not fnmatch.fnmatch(node, self._filePattern): + continue + if self._reFileExcludes is not None and self._reFileExcludes.search(node): + continue + if self._fileMustReadable and not base.LinuxUtils.isReadable(statInfo, self._euid, self._egid): + continue + if self._fileMustWritable and not base.LinuxUtils.isWritable(statInfo, self._euid, self._egid): + continue + if depth < self._minDepth: + continue + self._ignoredFiles -= 1 + self._statInfo = statInfo + self._countFiles += 1 + self._bytesFiles += self._statInfo.st_size + self._fileFullName = full + self._node = node + self._fileRelativeName = full[self._lengthDirectory:] + yield full + self._yields += 1 + if self._yields >= self._maxYields: + return + for node in dirs: + full = directory2 + os.sep + node + yield from self.next(full, depth + 1) + + def summary(self): + '''Returns the info about the traverse process: count of files... + @return: the infotext + ''' + rc = 'dir(s): {} file(s): {} / {}\nignored: dir(s): {} file(s): {}'.format( + self._countDirs, self._countFiles, + base.StringUtils.formatSize(self._bytesFiles), + self._ignoredDirs, self._ignoredFiles) + return rc + + +def addOptions(mode, usageInfo): + '''Adds the options for controlling the DirTraverser instance to a UsageInfo instance. + @param: the options will be assigned to this mode + @param: usageInfo: the options will be added to that + ''' + option = base.UsageInfo.Option('dirs-pattern', 'f', 'if a directory matches this regular expression it will be not processed', + 'regexpr') + usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option('dirs-excluded', 'X', 'if a directory matches this regular expression it will be not processed', + 'regexpr') + usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option( + 'file-type', 't', 'only files with this filetype will be found: d(irectory) f(ile) l(link)') + usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option( + 'min-depth', 'm', 'only subdirectories with that (or higher) depth will be processed', 'int', 0) + usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option( + 'min-depth', 'm', 'only subdirectories with that (or lower) depth will be processed', 'int') + option = base.UsageInfo.Option( + 'max-yields', 'Y', 'only that number of matching files will be found', 'int') + usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option( + 'min-size', 's', 'only larger files than that will be found, e.g. --min-size=2Gi', 'size', 0) + usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option( + 'max-size', 'S', 'only smaller files than that will be found, e.g. --max-size=32kByte', 'size') + usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option( + 'older-than', 'o', 'only files older than that will be found, e.g. --older-than=2020.7.3-4:32', 'datetime') + usageInfo.addModeOption(mode, option) + option = base.UsageInfo.Option( + 'younger-than', 'y', 'only files younger than that will be found, e.g. --younger-than=2020.7.3-4:32', 'datetime') + usageInfo.addModeOption(mode, option) + + +def buildFromOptions(pattern, usageInfo, mode): + '''Returns a DirTraverser instance initialized by given options. + @param pattern: a file pattern with path, e.g. "*.txt" or "/etc/*.conf" + @param usageInfo: contains the options built by addOptions() + @param mode: specifies the storage in usageInfo + @return a DirTravers instance initiatialized by values from the options + ''' + def v(name): + return usageInfo._optionProcessors[mode].valueOf(name) + baseDir = None + if os.path.isdir(pattern): + baseDir = pattern + pattern = '*' + else: + baseDir = os.path.dirname(pattern) + pattern = os.path.basename(pattern) + fileMustReadable = fileMustWritable = dirMustWritable = None + rc = DirTraverser(baseDir, pattern, v('dir-pattern'), v('files-excluded'), v('dirs-excluded'), + v('file-type'), v('min-depth'), v('max-depth'), + fileMustReadable, fileMustWritable, dirMustWritable, + v('max-yields'), v('younger-than'), v('older-than'), v('min-size'), v('max-size')) + return rc + + +def _stringToDate(string, errors): + rc = datetime.datetime.strptime(string, '%Y.%m.%d-%H:%M:%S') + if rc is None: + rc = datetime.datetime.strptime(string, '%Y.%m.%d') + if rc is None: + errors.append('wrong datetime syntax: {}'.format(string)) + return rc + + +if __name__ == '__main__': + pass diff --git a/base/FileHelper.py b/base/FileHelper.py new file mode 100644 index 0000000..44c7684 --- /dev/null +++ b/base/FileHelper.py @@ -0,0 +1,1201 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' + +import os +import stat +import datetime +import time +import shutil +import re +import tarfile +import zipfile +import tempfile +import fnmatch + +import base.Const +import base.StringUtils +import base.LinuxUtils +import base.TextProcessor +import base.ProcessHelper + +REG_EXPR_WILDCARDS = re.compile(r'[*?\[\]]') +GLOBAL_LOGGER = None +GLOBAL_UNIT_TEST_MODE = None +CURRDIR_PREFIX = '.' + os.sep + + +class DirInfo: + '''Stores the directory info + ''' + + def __init__(self, maxYoungest=5, maxLargest=5, maxOldest=5, maxSmallest=5, minSize=1, + dirsOnly=False, filesOnly=False, trace=0): + '''Constructor. + @param maxYoungest: the maximal number of entries in self._youngest + @param maxLargest: the maximal number of entries in self._largest + @param maxOldest: the maximal number of entries in self._oldest + @param maxLargest: the maximal number of entries in self._smallest + @param minSize: the minimum size of the entries in self._smallest + @param dirsOnly: True: only directories will be processed + @param filesOnly: True: only files (not dirs) will be processed + @param trace: if > 0: after processing this amount of nodes a statistic is logged + ''' + self._fileCount = 0 + self._fileSizes = 0 + self._dirCount = 0 + self._dirPattern = None + self._filePattern = None + self._ignoredDirs = 0 + self._ignoredFiles = 0 + self._youngest = [] + self._largest = [] + self._smallest = [] + self._oldest = [] + self._maxYoungest = maxYoungest + self._maxLargest = maxLargest + self._maxLargest = maxOldest + self._maxSmallest = maxSmallest + self._minSize = minSize + self._timeYoungest = 0 + self._timeOldest = 0 + self._sizeLargest = 0 + self._dirsOnly = dirsOnly + self._filesOnly = filesOnly + self._trace = trace + self._nextTracePoint = trace + self._maxOldest = None + self._maxDepth = None + + +def _error(message): + '''Prints an error message. + @param message: error message + @return False: for chaining + ''' + global GLOBAL_LOGGER + if GLOBAL_LOGGER is None: + print('+++ ' + message) + else: + GLOBAL_LOGGER.error(message) + return False + + +def _log(message, level=base.Const.LEVEL_SUMMARY): + '''Prints a message. + @param message: error message + ''' + global GLOBAL_LOGGER + if GLOBAL_LOGGER is None: + print(message) + else: + GLOBAL_LOGGER.log(message, level) + + +def changeExtendedAttributes(path, toAdd=None, toDelete=None): + '''Changes the attributes of a file (using from /usr/bin/chattr). + Important attributes: c(ompression) (no)C(OW) a(ppendOnly) (no)A(timeUpdates) (synchronous)D(irectoryUpdates) + I(mmutable) (data)J(ournaling) S(ynchronousUpdates) u(ndeletable) + @param path: that file will be changed + @param toAdd: None or a list of attributes to add to the file, e.g. "cC" + @param toDelete: None or a list of attributes to delete from the file, e.g. "cC" + ''' + if toAdd is None and toDelete is None: + _error(f'changeExtendedAttributes(): missing attributes to add/delete for {path}') + else: + helper = base.ProcessHelper.ProcessHelper(GLOBAL_LOGGER) + argv = ['/usr/bin/chattr'] + if toAdd: + argv.append(f'+{toAdd}') + if toDelete: + argv.append(f'-{toDelete}') + argv.append(path) + helper.execute(argv, True) + + +def clearDirectory(path): + '''Deletes (recursivly) all files and subdirectories of a given path. + Note: if the path is not a directory (or it does not exists) it will not be handled as an error + @param path: the directory to clear + ''' + if os.path.exists(path): + global GLOBAL_LOGGER + for node in os.listdir(path): + full = path + os.sep + node + if os.path.isdir(full): + shutil.rmtree(full, True) + else: + os.unlink(full) + if os.path.exists(full) and GLOBAL_LOGGER is not None: + _error('cannot remove: ' + full) + + +def createBackup(source, target=None, extension=None, expandPlaceholders=True, checkEqualNames=True): + '''Save the source as target to save a file as backup. + @param source: the file to backup + @param target: the "safe place" of the file + @param expandPlaceholders: True: the following placeholders in target will be expanded: + @see expandPathPlaceholders() for more info + @param checkEqualNames: True: the test is done whether source != target + ''' + if target is None: + if extension.find('%') >= 0: + extension = expandPlaceholders(extension, source) + if not extension.startswith('.'): + extension = '.' + extension + deepRename(source, extension, deleteExisting=True) + else: + if expandPlaceholders and target.find('%') >= 0: + target = expandPlaceholders(target, source) + if os.path.isdir(target): + target += os.sep + os.path.basename(source) + elif target.find(os.sep) < 0: + pass + if checkEqualNames and os.path.realpath(source) != os.path.realpath(target): + target += '~' + if target.os.path.islink(source): + pass + else: + moveFile(source, target) + + +def createFileTree(files, baseDirectory): + '''Creates a directory tree with files specified in a text: each line contains one dir/file + Specification: one file/dir per line (directories does not have a content) + filename[|content[|mode[|date]]] + If content starts with '->' a link is created + If filename ends with / it is a directory. + Examples: + dir1/ + dir1/file1|this is in file|664|2020-01-22 02:44:32 + main.dir|->dir1 + @param files: the text describing the dirs/files + @parma baseDirectory: the tree begins with this directory + ''' + lines = files.split('\n') + for line in lines: + if line.strip() == '': + continue + parts = line.split('|') + full = baseDirectory + os.sep + parts[0] + if parts[0].endswith('/'): + full = full[0:-1] + mode = int(parts[1], 8) if len(parts) > 1 else 0o777 + if not os.path.exists(full): + os.makedirs(full, mode) + os.chmod(full, mode) + if len(parts) > 2: + date = datetime.datetime.strptime( + parts[2], '%Y-%m-%d %H:%M:%S') + setModified(full, None, date) + else: + parent = os.path.dirname(full) + if not os.path.isdir(parent): + os.makedirs(parent) + content = parts[1] if len(parts) > 1 else '' + mode = int(parts[2], 8) if len(parts) > 2 else 0o666 + if content.startswith('->'): + if os.path.exists(full): + os.unlink(full) + os.symlink(content[2:], full) + else: + base.StringUtils.toFile(full, content, fileMode=mode) + if len(parts) > 3: + date = datetime.datetime.strptime( + parts[3], '%Y-%m-%d %H:%M:%S') + setModified(full, None, date) + + +def copyDirectory(source, target, option=None, verboseLevel=0): + '''Copies all files (and dirs) from source to target directory. + @param source: the base source directory + @param target: the base target directoy() + @param option: None, 'clear' or 'update' + 'clear': all files (and subdirs) of target will be deleted + 'update': only younger or not existing files will be copied False: all files will be copied + ''' + if option == 'clear': + if verboseLevel >= base.Const.LEVEL_DETAIL: + _log('clearing ' + target, verboseLevel) + clearDirectory(target) + for node in os.listdir(source): + src = source + os.sep + node + trg = target + os.sep + node + if os.path.islink(src): + if not option == 'update' or not os.path.exists(trg): + ref = os.readlink(src) + if verboseLevel >= base.Const.LEVEL_DETAIL: + _log('symlink: {} [{}]'.format(trg, ref), verboseLevel) + try: + os.symlink(ref, trg) + except OSError as exc: + _error('cannot create a symlink: {} -> {}'.format(ref, trg)) + elif os.path.isdir(src): + if option != 'update' or not os.path.exists(trg): + if verboseLevel >= base.Const.LEVEL_DETAIL: + _log('directory: {} -> {}'.format(src, trg), verboseLevel) + shutil.copytree(src, trg, True) + else: + copyDirectory(src, trg, option) + else: + if not os.path.exists(trg) or option == 'update' and os.path.getmtime(src) > os.path.getmtime(trg): + try: + if verboseLevel >= base.Const.LEVEL_DETAIL: + _log('{} -> {}'.format(src, trg), verboseLevel) + shutil.copy2(src, trg) + except OSError as exc: + _error('cannot copy {}: {}'.format(trg, str(exc))) + + +def copyByRules(rules, baseSource, baseTarget): + '''Copies directories/files from a given directory tree controlled by a list of rules. + The rules is a list of lines. + Each line contains a copy rule: a source file/file pattern followed by a target name and options. + Separator in the line is the ':', in options ',' + Examples: + public/index.php + copy public/index.php to public/index.php + public/icons:* + copy public/icons with all subdirectories and files into public/icons + app/*:*:symlink,dirsonly,except local|test + creates symbolic links from all directories in the subdir app/ except "common" + tools/run.template:tools/run.sh + copies the file tools/run.template and change the name to run.sh + : symlink filesonly dirsonly "except " "replacewhatwith" + @param rules: a list of rules for copying + @param baseSource the directory tree to copy + @param baseTarget the target directory. Will be created if it does not exist + ''' + if os.path.dirname(baseSource) != os.path.dirname(baseTarget): + _error('source and target does not have the same parent. Not supported') + elif not os.path.isdir(baseSource): + _error('not a directory: ' + baseSource) + else: + ensureDirectory(baseTarget) + clearDirectory(baseTarget) + lineNo = 0 + for rule in rules: + lineNo += 1 + rule = rule.strip() + if rule == '' or rule.startswith('#'): + continue + if rule.startswith(':'): + target = rule[1:] + full = baseTarget + os.sep + target + _log('create ' + full, base.Const.LEVEL_DETAIL) + os.makedirs(full) + continue + parts = rule.split(':') + source = parts[0].lstrip(os.sep) + opts = {} + full = baseSource + os.sep + source.lstrip(os.sep) + toTest = os.path.dirname(full) if hasWildcards(source) else full + if toTest != '' and not os.path.exists(toTest): + _error('line {}: source not found: {}'.format(lineNo, toTest)) + continue + if len(parts) == 1: + target = source + elif len(parts) == 2: + target = parts[1] if parts[1].lstrip(os.sep) != '*' else source + elif len(parts) == 3: + target = parts[1] if parts[1].lstrip(os.sep) != '*' else source + for opt in parts[2].split(','): + optParts = opt.split(' ', 2) + opts[optParts[0]] = None if len( + optParts) < 2 else optParts[1] + if re.match(r'^dirsonly|except|filesonly|recursive|replace|symlink$', optParts[0]) is None: + _error('unknown option: ' + opt) + if optParts[0] == 'replace': + value = optParts[1] + if value == '' or value.count(value[0]) != 3: + _error('wrong syntax in replace option: ' + value) + del opts['replace'] + else: + _error('line {}: too many ":" in: {}'.format(lineNo, rule)) + continue + copyByRule(full, baseTarget + os.sep + + target, opts, source.count(os.sep)) + + +def copyByRule(fnSource, fnTarget, options, depthRelPath): + '''Execute a rule from copyFileTree. + @param fnSource: the source file/directory + @param fnTarget: the target + @param options: a dictionary with (option_name, option_value) pairs + @param depthRelPath: the depth of the tree from the base directory + ''' + parentSource = os.path.dirname(fnSource) + pathSource = parentSource + os.sep if parentSource != '' else '' + parentTarget = os.path.dirname(fnTarget) + pathTarget = parentTarget + os.sep if parentTarget != '' else '' + if parentTarget != '': + ensureDirectory(parentTarget) + nodeSource = os.path.basename(fnSource) + if hasWildcards(nodeSource): + reExcept = None if 'except' not in options else re.compile( + options['except']) + for node in os.listdir('.' if parentSource == '' else parentSource): + full = pathSource + node + if not fnmatch.fnmatch(node, nodeSource): + _log('ignoring {}'.format(full), base.Const.LEVEL_DETAIL) + continue + if reExcept is not None and reExcept.match(node) is not None: + _log('ignoring {}'.format(full), base.Const.LEVEL_DETAIL) + continue + isDir = os.path.isdir(full) + if 'dirsonly' in options and not isDir: + _log('ignoring non directory {}'.format( + full), base.Const.LEVEL_DETAIL) + continue + if 'filesonly' in options and isDir: + _log('ignoring directory {}'.format( + full), base.Const.LEVEL_DETAIL) + continue + copyByRule(pathSource + node, pathTarget + + node, options, depthRelPath) + else: + if 'symlink' in options: + depth = fnSource.count(os.sep) + 1 + partsSource = fnSource.split(os.sep) + relPath = os.sep.join(partsSource[depth - depthRelPath - 1:]) + linkSource = '../' * \ + (1 + depthRelPath) + \ + partsSource[depth - depthRelPath - 2] + os.sep + relPath + os.symlink(linkSource, fnTarget) + else: + if 'dirsonly' in options and not os.path.isdir(fnSource): + _log('ignoring non directory {}'.format( + fnSource), base.Const.LEVEL_DETAIL) + elif 'filesonly' in options and os.path.isdir(fnSource): + _log('ignoring directory {}'.format( + fnSource), base.Const.LEVEL_DETAIL) + else: + if os.path.isdir(fnSource): + if 'recursive' in options: + shutil.copytree(fnSource, fnTarget) + else: + _log('creating ' + fnTarget, base.Const.LEVEL_DETAIL) + os.makedirs(fnTarget) + shutil.copystat(fnSource, fnTarget) + elif 'replace' in options: + global GLOBAL_LOGGER + processor = base.TextProcessor.TextProcessor(GLOBAL_LOGGER) + processor.readFile(fnSource, mustExists=True) + value = options['replace'] + if value == '' or value.count(value[0]) != 3: + _error('wrong syntax in replace option: ' + value) + else: + parts = value[1:].split(value[0], 2) + hits = processor.replace( + parts[0], parts[1], noRegExpr=True, countHits=True) + _log('{} replacement(s) [{}] in {} => {}'.format(hits, value, fnSource, + fnTarget), base.Const.LEVEL_DETAIL) + processor.writeFile(fnTarget) + else: + _log('copying {} => {}'.format( + fnSource, fnTarget), base.Const.LEVEL_DETAIL) + shutil.copy2(fnSource, fnTarget, follow_symlinks=False) + + +def copyIfExists(source, target): + '''Copies all files (and dirs) from source to target directory. + @param source: the base source directory + @param target: the base target directoy() + @param verboseLevel: True: do logging + ''' + if os.path.exists(source): + _log('copying {} => {} ...'.format( + source, target), 2, base.Const.LEVEL_DETAIL) + shutil.copy2(source, target) + + +def deepRename(oldName, newNode, deleteExisting=False): + '''Renames a file or symbolic link. + Not symbolic links: renaming "normally". + Symbolic links: the link target will be renamed. This is useful for backing up files: + not the symbolic link is the subject to save but the link target. + @param oldName: the file to rename (with path) + @param newNode: the new filename (without path) + @return True: success + ''' + rc = True + if not os.path.exists(oldName): + rc = _error('cannot rename (old name does not exist): ' + oldName) + elif os.path.islink(oldName): + source = endOfLinkChain(oldName) + nodeOld = os.path.basename(source) + if nodeOld == newNode: + rc = _error('cannot rename (link target has the same name {}): {}'.format( + nodeOld, oldName)) + else: + rc = deepRename(source, newNode) + else: + newName = os.path.join(os.path.dirname(oldName), newNode) + if os.path.exists(newName): + if not deleteExisting: + rc = _error('cannot rename (new name exists): ' + newName) + else: + _log('deleting ' + newName, base.Const.LEVEL_LOOP) + global GLOBAL_UNIT_TEST_MODE + if GLOBAL_UNIT_TEST_MODE == 'deepRename-no-unlink': + _log('suppressing delete of {} ({})'.format( + newName, GLOBAL_UNIT_TEST_MODE), base.Const.LEVEL_DETAIL) + else: + os.unlink(newName) + rc = not os.path.exists(newName) + if not rc: + _error('cannot remove new name: ' + newName) + if rc: + _log('renaming {} => {}'.format( + oldName, newName), base.Const.LEVEL_LOOP) + os.rename(oldName, newName) + return rc + + +def directoryInfo(path, filePattern=None, dirPattern=None, maxDepth=-1, fileInfo=None, + maxYoungest=5, maxLargest=5, maxOldest=5, maxSmallest=5, minSize=1, dirsOnly=False, + filesOnly=False, trace=0): + '''Returns the directory info of the given path. + @param path: the full path of the directory to inspect + @param filePattern: None or a regular expression (as text) describing the file names to inspect + @param dirPattern: None or a regular expression (as text) describing the directory names to inspect + @param maxDepth: maximal depth of recursion. < 0: unlimited 0: only the start directory + @param dirInfo: None or a DirectoryInfo instance which will be completed + @param maxYoungest: the maximal number of entries in DirInfo._youngest[] + @param maxLargest: the maximal number of entries in DirInfo._largest[] + @param maxSmallest: the maximal number of entries in DirInfo._smallest[] + @param minSize: the minimum size of the entries in DirInfo._smallest[] + @param dirsOnly: only directories will be part of the result + @param filessOnly: only files (not directories) will be part of the result + @param trace: if > 0: a statistic is printed if this amount of nodes (files or nodes) is processed + @return: a DirInfo instance + ''' + def infoOneDir(path, depth, fileInfo): + def showStatistic(info): + print('{}: dirs: {} files: {} ignored dirs: {} ignored files: {}'.format( + path, info._dirCount, info._fileCount, info._ignoredDirs, info._ignoredFiles)) + info._nextTracePoint += info._trace + if not isinstance(fileInfo, DirInfo): + depth = 0 + fileInfo._dirCount += 1 + if (fileInfo._trace > 0 and fileInfo._dirCount + fileInfo._fileCount + fileInfo._ignoredDirs + + fileInfo._ignoredFiles > fileInfo._nextTracePoint): + showStatistic(fileInfo) + try: + nodes = os.listdir(path) + except PermissionError: + fileInfo._ignoredDirs += 1 + return + if (fileInfo._trace > 0 and not nodes + and fileInfo._dirCount + fileInfo._fileCount + fileInfo._ignoredDirs + + fileInfo._ignoredFiles % fileInfo._trace == 0): + showStatistic(fileInfo) + for node in nodes: + if (fileInfo._trace > 0 and fileInfo._dirCount + fileInfo._fileCount + + fileInfo._ignoredDirs + fileInfo._ignoredFiles > fileInfo._nextTracePoint): + showStatistic(fileInfo) + full = path + os.sep + node + stats = os.lstat(full) + isDir = stat.S_ISDIR(stats.st_mode) + if isDir: + if not fileInfo._filesOnly: + length = len(fileInfo._youngest) + if (fileInfo._maxYoungest > 0 + and (length < fileInfo._maxYoungest + or stats.st_mtime > fileInfo._timeYoungest)): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxYoungest: + del fileInfo._youngest[0] + fileInfo._youngest.append( + str(stats.st_mtime) + ':' + path + os.sep + node) + fileInfo._youngest.sort( + key=lambda x: float(x.split(':')[0])) + fileInfo._timeYoungest = float( + fileInfo._youngest[0].split(':')[0]) + length = len(fileInfo._oldest) + if fileInfo._maxOldest > 0 and (length < fileInfo._maxOldest or stats.st_mtime < fileInfo._timeOldest): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxOldest: + del fileInfo._oldest[-1] + fileInfo._oldest.insert( + 0, str(stats.st_mtime) + ':' + path + os.sep + node) + fileInfo._oldest.sort( + key=lambda x: float(x.split(':')[0])) + fileInfo._timeOldest = float( + fileInfo._oldest[-1].split(':')[0]) + if ((fileInfo._dirPattern is None or fileInfo._dirPattern.match(node) is None) + and (maxDepth is None or maxDepth < 0 or depth < maxDepth)): + infoOneDir(path + os.sep + node, depth + 1, fileInfo) + else: + fileInfo._ignoredDirs += 1 + else: # not isDir + if fileInfo._dirsOnly: + fileInfo._ignoredFiles += 1 + continue + if (fileInfo._filePattern is None or fileInfo._filePattern.match(node) is not None): + fileInfo._fileSizes += stats.st_size + fileInfo._fileCount += 1 + length = len(fileInfo._largest) + if (fileInfo._maxLargest > 0 and (length < fileInfo._maxLargest + or stats.st_size > fileInfo._sizeLargest)): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxLargest: + del fileInfo._largest[0] + fileInfo._largest.append( + str(stats.st_size) + ':' + path + os.sep + node) + fileInfo._largest.sort( + key=lambda x: float(x.split(':')[0])) + fileInfo._sizeLargest = float( + fileInfo._largest[-1].split(':')[0]) + length = len(fileInfo._smallest) + if (fileInfo._maxSmallest > 0 and (stats.st_size >= fileInfo._minSize + and (length < fileInfo._maxSmallest + or stats.st_size > fileInfo._sizeSmallest))): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxSmallest: + del fileInfo._smallest[-1] + fileInfo._smallest.insert(0, + str(stats.st_size) + ':' + path + os.sep + node) + fileInfo._smallest.sort( + key=lambda x: float(x.split(':')[0])) + fileInfo._sizeSmallest = float( + fileInfo._smallest[-1].split(':')[0]) + length = len(fileInfo._youngest) + if (fileInfo._maxYoungest > 0 and (length < fileInfo._maxYoungest + or stats.st_mtime > fileInfo._timeYoungest)): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxYoungest: + del fileInfo._youngest[0] + fileInfo._youngest.append( + str(stats.st_mtime) + ':' + path + os.sep + node) + fileInfo._youngest.sort( + key=lambda x: float(x.split(':')[0])) + fileInfo._timeYoungest = float( + fileInfo._youngest[0].split(':')[0]) + length = len(fileInfo._oldest) + if (fileInfo._maxOldest > 0 and (length < fileInfo._maxOldest + or stats.st_mtime < fileInfo._timeOldest)): + if not base.LinuxUtils.isReadable(stats, euid, egid): + fileInfo._ignoredFiles += 1 + else: + if length >= fileInfo._maxOldest: + del fileInfo._oldest[-1] + fileInfo._oldest.insert( + 0, str(stats.st_mtime) + ':' + path + os.sep + node) + fileInfo._oldest.sort( + key=lambda x: float(x.split(':')[0])) + fileInfo._timeOldest = float( + fileInfo._oldest[-1].split(':')[0]) + else: + fileInfo._ignoredFiles += 1 + continue + # end of infoOneDir() + if fileInfo is None: + fileInfo = DirInfo(maxYoungest, maxLargest, maxOldest, + maxSmallest, minSize, dirsOnly, filesOnly, trace) + if filePattern is not None: + fileInfo._filePattern = base.StringUtils.regExprCompile( + filePattern, 'file pattern') + if dirPattern is not None: + fileInfo._filePattern = base.StringUtils.regExprCompile( + dirPattern, 'dir pattern') + + euid = os.geteuid() + egid = os.getegid() + fileInfo._maxYoungest = maxYoungest + fileInfo._maxLargest = maxLargest + fileInfo._maxOldest = maxOldest + fileInfo._maxDepth = maxDepth + infoOneDir(path, 0, fileInfo) + return fileInfo + + +def distinctPaths(path1, path2): + '''Tests whether two paths are not part of each other. + @param path1: first path to test + @param path2: 2nd path to test + @return: True: path1 is not parent of path2 and path2 is not parent of path1 + ''' + dir1 = os.path.realpath(path1) + dir2 = os.path.realpath(path2) + return not dir1.startswith(dir2) and not dir2.startswith(dir1) + + +def endOfLinkChain(filename): + '''Returns the last entry of a symbolic link chain or None. + @param filename: the first entry of a symbol link chain + @return: None: not existing nodes in the link chain Otherwise: the last element of the chain + ''' + rc = os.path.realpath(filename) + if not os.path.lexists(rc) or os.path.islink(rc): + _error('invalid entry {} in the symbolic link chain: {}'.format(rc, filename)) + rc = None + return rc + + +def ensureDirectory(directory, mode=0o777, user=None, group=None): + '''Ensures that the given directory exists. + @param directory: the complete name + @return: None: could not create the directory + otherwise: the directory's name + ''' + if not os.path.isdir(directory): + try: + os.lstat(directory) + os.unlink(directory) + except FileNotFoundError: + pass + _log('creating {}{} ...'.format( + directory, os.sep), base.Const.LEVEL_SUMMARY) + try: + os.makedirs(directory, mode) + os.chmod(directory, mode) + except OSError as exc: + _error('cannot create dir {}: {}'.format(directory, str(exc))) + if not os.path.isdir(directory): + directory = None + elif user is not None or group is not None: + os.chown(directory, base.LinuxUtils.userId( + user), base.LinuxUtils.groupId(group)) + return directory + + +def ensureFileDoesNotExist(filename): + '''Ensures that a file does not exist. + @param filename: the file to delete if it exists. + ''' + if os.path.lexists(filename): + try: + try: + if os.path.isdir(filename): + _log('removing {}{} ...'.format( + filename, os.sep), base.Const.LEVEL_DETAIL) + shutil.rmtree(filename, False) + else: + _log('removing {} ...'.format( + filename), base.Const.LEVEL_DETAIL) + os.unlink(filename) + except OSError as exp: + _error('cannot delete {:s}: {:s}'.format(filename, str(exp))) + except FileNotFoundError: + pass + + +def ensureFileExists(filename, content=''): + '''Ensures that a file does not exist. + @param filename: the file to create if it does not exist + @param content: this text will be stored for a new created file + ''' + try: + if os.path.exists(filename): + if os.path.isdir(filename): + _log('is a directory: {}'.format( + filename), base.Const.LEVEL_DETAIL) + else: + _log('creating {} ...'.format(filename), base.Const.LEVEL_DETAIL) + base.StringUtils.toFile(filename, content) + except OSError as exc: + _error('problems with {}: {}'.format(filename, str(exc))) + + +def ensureSymbolicLink(source, target, createTarget=True): + '''Ensures that a directory exists. + @param source: the full name of the link source, e.g. '../sibling' + @param target: full name of the file of type 'link' + @param createTarget: creates the target if it does not exist + @return: True: the link exists + ''' + info = None + try: + info = os.lstat(target) + except FileNotFoundError: + pass + if info is not None: + if os.path.islink(target): + oldLink = os.readlink(target) + if oldLink != source: + _log('changing link from {} to {}'.format( + oldLink, source), base.Const.LEVEL_DETAIL) + os.unlink(target) + elif os.path.isdir(target): + _error('target {} is already a directory (not a link)'.format(target)) + else: + _log('removing the non link file ' + + target, base.Const.LEVEL_DETAIL) + os.unlink(target) + if not os.path.exists(target): + baseDir = os.path.dirname(target) + if not os.path.isdir(baseDir) and createTarget: + ensureDirectory(baseDir) + hasParent = os.path.isdir(baseDir) + if not hasParent: + _error('parent of target is not a directory: ' + baseDir) + realPath = os.path.join(target, source) + absSource = os.path.normpath(realPath) + if not os.path.exists(absSource): + _error('missing source {} [= {}]'.format(source, absSource)) + elif hasParent: + _log('creating symbol link {} -> {}'.format(source, target), + base.Const.LEVEL_DETAIL) + os.symlink(source, target) + rc = os.path.islink(target) and os.readlink(target) == source + return rc + + +def extendedAttributesOf(path): + '''Returns the attributes of a file (returned from /usr/bin/lsattr). + Important attributes: c(ompression) (no)C(OW) a(ppendOnly) (no)A(timeUpdates) (synchronous)D(irectoryUpdates) + I(mmutable) (data)J(ournaling) S(ynchronousUpdates) u(ndeletable) + @param path: the file to inspect + @return: a list of attributes, e.g. 'cC' or '' + ''' + helper = base.ProcessHelper.ProcessHelper(GLOBAL_LOGGER) + helper.execute(['/usr/bin/lsattr', path], False, storeOutput=True) + rc = ('' if not helper._output else helper._output[0]).split(' ')[0].replace('-', '') + return rc + + +def expandPathPlaceholders(pattern, filename): + '''Expands placeholders in a pattern with the current date time or parts of a related filename. + @param pattern: a string with placeholders: + %date%: the current date %datetime%: the current date and time + %seconds%: the current date time as seconds after the epoche + %path%: the path of source %node%: the node of source + %name%: the name of source (without extension) %ext%: the extension of source + @param filename: the placeholders can be parts of this filename + @return: pattern with expanded placeholders + ''' + parts = splitFilename(filename) + now = datetime.datetime.now() + for matcher in re.finditer(r'%(date(time)?|seconds|path|node|name|ext)%', pattern): + name = matcher.group(1) + macro = matcher.group(0) + if name == 'date': + value = now.strftime('%Y.%m.%d') + elif name == 'datetime': + value = now.strftime('%Y.%m.%d-%H_%M_%S') + elif name == 'seconds': + value = now.strftime('%a') + elif name == 'path': + value = parts['path'] + elif name == 'node': + value = parts['node'] + elif name == 'name': + value = parts['fn'] + elif name == 'ext': + value = parts['ext'] + pattern = pattern.replace(macro, value) + return pattern + + +def fileClass(path): + '''Returns the file class of the file. + @param path: the full filename + @return: a tuple (class, subclass): class: 'container', 'text', 'binary', 'unknown' + subclass of 'container': 'dir', 'tar', 'tgz', 'zip' + subclass of 'text': 'xml', 'shell' + ''' + def isBinaryByte(bb): + rc = bb < 0x09 or (0x0d < bb < 0x20) + return rc + + def isBinary(byteArray): + found = 0 + rc = False + # for ix in range(len(byteArray)): + # bb = byteArray[ix] + for bb in byteArray: + if bb == b'\x00': + rc = True + break + elif isBinaryByte(bb): + found += 1 + if found > 100 or found > len(byteArray) / 10: + rc = True + break + return rc + + def isNullString(byteArray): + '''Tests whether the byteArray is a text delimited with 0. + @param byteArray: array to test + @return True: only text and '\0' is part of byteArray + ''' + ix = 0 + rc = True + hasNull = False + while ix < len(byteArray): + if byteArray[ix] == 0: + hasNull = True + elif isBinaryByte(byteArray[ix]): + rc = False + break + ix += 1 + return rc and hasNull + + def isNullNumber(byteArray): + '''Tests whether the byteArray are digits delimited with 0. + @param byteArray: array to test + @return True: only decimal digits and '\0' is part of byteArray + ''' + ix = 0 + rc = True + hasNull = False + while ix < len(byteArray): + if byteArray[ix] == 0: + hasNull = True + elif not (byteArray[ix] >= 0x30 and byteArray[ix] <= 0x39): # TAB + rc = False + break + ix += 1 + return rc and hasNull + if os.path.isdir(path): + (theClass, subClass) = ('container', 'dir') + else: + with open(path, 'rb') as fp: + start = fp.read(4096) + if start.startswith(b'\x1f\x8b\x08'): + (theClass, subClass) = ('container', 'tar') + elif start.startswith(b'BZ') and isBinary(start[8:80]): + (theClass, subClass) = ('container', 'tar') + elif start.startswith(b'PK') and isBinary(start[2:32]): + (theClass, subClass) = ('container', 'zip') + elif isNullString(start[0:100]) and isNullNumber(start[100:0x98]): + (theClass, subClass) = ('container', 'tar') + elif (start[0:100].lower().find(b'') >= 0 or start[0:100].lower().find(b'= 0) and not isBinary(start): + (theClass, subClass) = ('text', 'xml') + elif len(start) > 5 and start.startswith(b'#!') and not isBinary(start): + (theClass, subClass) = ('text', 'shell') + elif isBinary(start): + (theClass, subClass) = ('binary', 'binary') + else: + (theClass, subClass) = ('text', 'text') + return (theClass, subClass) + + +def fileType(path): + '''Returns the file type: 'file', 'dir', 'link', 'block' + @param path: the full filename + @return: the filetype: 'file', 'dir', 'link', 'block', 'char' + ''' + if os.path.islink(path): + rc = 'link' + elif os.path.isdir(path): + rc = 'dir' + else: + rc = 'file' + return rc + + +def fromBytes(line): + '''Converts a line with type bytes into type str. + @param line: line to convert + ''' + try: + rc = line.decode() + except UnicodeDecodeError: + try: + rc = line.decode('latin-1') + except UnicodeDecodeError: + rc = line.decode('ascii', 'ignore') + return rc + + +def hasWildcards(filename): + '''Tests whether a filename has wildcards. + @param filename: filename to test + @return: True: the filename contains wildcard like '*', '?' or '[...]' + ''' + global REG_EXPR_WILDCARDS + rc = REG_EXPR_WILDCARDS.search(filename) is not None + return rc + + +def joinFilename(parts): + '''Joins an array of parts into a filename. + This is the other part of splitFilename(). + @param parts: the array created by splitFilename + @return the filename decribed in parts + ''' + rc = parts['path'] + parts['fn'] + parts['ext'] + return rc + + +def joinRelativePath(relPath, start=None): + '''Joins a relative path and a start path to a non relative path. + Example: joinPath('../brother', '/parent/sister') is '/parent/brother' + @param relPath: the relative path, e.g. '../sister' + @param start: the start point for joining, e.g. 'family/sister'. If None: the current directory + @returns the non relative path, e.g. 'family/brother' + ''' + rc = None + relParts = relPath.split(os.sep) + if start is None: + start = os.curdir + startParts = start.split(os.sep) + if not relParts or relParts[0] != '..': + _error('not a relative path: ' + relPath) + else: + rc = '' + while relParts and relParts[0] == '..': + if not startParts: + _error('too many backsteps in relpath {} for start {}'.format( + relPath, start)) + rc = None + break + relParts = relParts[1:] + startParts = startParts[0:-1] + if rc is not None: + rc = os.sep.join(startParts) + if relParts: + if rc == '': + rc = os.sep.join(relParts) + else: + rc += os.sep + os.sep.join(relParts) + return rc + + +def listFile(statInfo, full, orderDateSize=True, humanReadable=True): + '''Builds the info for one file (or directory) + @param statInfo: the info returned by os.(l)stat() + @param full: the filename + @param orderDateSize: True: order is date left of size False: order is size leftof date + @param humanReadable: True: better for reading (matching unit), e.g. "10.7 GByte" or "3 kByte" + ''' + if full.startswith(CURRDIR_PREFIX): + full = full[2:] + if stat.S_ISDIR(statInfo.st_mode): + size = '' + elif stat.S_ISLNK(statInfo.st_mode): + size = '' + full += ' -> ' + os.readlink(full) + elif humanReadable: + size = "{:>8s}".format(base.StringUtils.formatSize(statInfo.st_size)) + else: + size = '{:13.6f} MB'.format(statInfo.st_size / 1000000) + fdate = datetime.datetime.fromtimestamp(statInfo.st_mtime) + dateString = fdate.strftime("%Y.%m.%d %H:%M:%S") + if orderDateSize: + rc = '{:s} {:>12s} {:s}'.format(dateString, size, full) + else: + rc = '{:>12s} {:s} {:s}'.format(size, dateString, full) + return rc + + +def mountPointOf(path, mountFile='/proc/mounts'): + '''Returns the mount point of the filesystem of a given directory. + @param path: the path to inspect + @return: (, ) the mount point of the filesystem containing the path and the filesystem name + ''' + rc = None + fsType = None + with open(mountFile, 'r') as fp: + for line in fp: + parts = line.split(' ') + if len(parts) > 3 and path.startswith(parts[1]) and (rc is None or len(rc) < len(parts[1])): + fsType = parts[2] + rc = parts[1] + return (rc, fsType) + + +def moveFile(source, target, removeAlways=True, createBaseDir=True): + '''Moves a file from one location to another. + If both parent directories are in the same filesytem rename is used. + Otherwise a copy is done and a deletion of the source. + @param source: the file to move + @param target: the target filename + @param removeAlways: True: the source will be deleted after copying + @param createBaseDir: True: the parent directory of target will be created if it does not exists + ''' + if createBaseDir: + baseDir = os.path.dirname(target) + ensureDirectory(baseDir) + try: + os.rename(source, target) + except OSError: + try: + shutil.copy2(source, target) + if removeAlways: + try: + os.unlink(source) + except OSError as exc2: + _error(f'cannot delete {source} after moving: {exc2}') + except OSError as exc: + _error(f'cannot copy {source} to {target}: {exc}') + + +def pathToNode(path): + '''Changed a path into a name which can be used as node (of a filename). + @param path: the path to convert + @return: path with replaced path separators + ''' + rc = path.replace(os.sep, '_').replace(':', '_') + return rc + + +def replaceExtension(filename, extension): + '''Replaces the extension of a filename. + @param filename: the filename to process + @param extension: the new extension + @return: the filename with the new extension + ''' + ix = filename.rfind('.') + ix2 = filename.rfind(os.sep) + if ix > ix2 + 1: + rc = filename[0:ix] + extension + else: + rc = filename + extension + return rc + +def splitFilename(full): + '''Splits a filename into its parts. + This is the other part of joinFilename(). + @param full: the filename with path + @return: a dictionary with the keys 'full', 'path', 'node', 'fn', 'ext' + example: { 'full': '/home/jonny.txt', 'path': '/home/', 'node' : 'jonny.txt', 'fn': 'jonny' , 'ext': '.txt' } + ''' + rc = dict() + rc['full'] = full + ix = full.rfind(os.sep) + if ix < 0: + rc['path'] = '' + node = rc['node'] = full + else: + rc['path'] = full[0:ix + 1] + node = rc['node'] = full[ix + 1:] + ix = node.rfind('.', 1) + if ix < 0: + rc['fn'] = node + rc['ext'] = '' + else: + rc['fn'] = node[0:ix] + rc['ext'] = node[ix:] + return rc + + +def setLogger(logger): + '''Sets the global logger. + @param logger: the global logger + ''' + global GLOBAL_LOGGER + GLOBAL_LOGGER = logger + + +def setUnitTestMode(mode): + '''Sets special behaviour for unit tests. + ''' + global GLOBAL_UNIT_TEST_MODE + GLOBAL_UNIT_TEST_MODE = mode + + +def setModified(path, timeUnix, date=None): + '''Sets the file modification time. + @precondition: exactly one of date and timeUnix must be None and the other not None + @param path: the full path of the file to modify + @param timeUnix: None or the time to set (unix timestamp since 1.1.1970) + @param date: None or the datetime to set (datetime.datetime instance) + @return: True: success False: precondition raised + ''' + dateModified = None + rc = True + if date is not None: + dateModified = time.mktime(date.timetuple()) + elif timeUnix is None: + rc = False + else: + dateModified = timeUnix + if dateModified is not None: + try: + os.utime(path, (int(dateModified), int(dateModified))) + except Exception as exc: + raise exc + return rc + + +def tail(filename, maxLines=1, withLineNumbers=False): + '''Returns the tail of a given file. + @param filename: the file to inspect + @param maxLines: the number of lines to return (or less) + @param withLineNumbers: True: add line numbers at the begin of line + @return: a list of lines from the end of the file + ''' + lines = [] + if maxLines < 1: + maxLines = 1 + with open(filename, "r") as fp: + lineNo = 0 + for line in fp: + lineNo += 1 + if len(lines) >= maxLines: + del lines[0] + lines.append(line) + if withLineNumbers: + lineNo -= len(lines) - 1 + for ix, line in enumerate(lines): + lines[ix] = '{}: {}'.format(lineNo, line) + lineNo += 1 + return lines + + +def tempFile(node, subDir=None): + '''Returns the name of a file laying in the temporary directory. + @param node: the filename without path + @param subdir: None or a subdirectory in the temp directory (may be created) + ''' + path = tempfile.gettempdir() + os.sep + if subDir is not None: + path += subDir + os.makedirs(path, 0o777, True) + path += os.sep + path += node + return path + + +def unpack(archive, target, clear=False): + '''Copies the content of an archive (tar, zip...) into a given directory. + @param archive: name of the archive, the extension defines the type: '.tgz': tar '.zip': zip + @param target: the directory which will be filled by the archive content. Will be created if needed + ''' + if not os.path.exists(target): + os.makedirs(target, 0o777, True) + elif not os.path.isdir(target): + _error('target is not a directory: ' + target) + archive = None + elif clear: + clearDirectory(target) + if archive is None: + pass + elif archive.endswith('.tgz'): + tar = tarfile.open(archive, 'r:gz') + tar.extractall(target) + elif archive.endswith('.zip'): + zipFile = zipfile.ZipFile(archive, 'r') + zipFile.extractall(target) + else: + _error('unknown file extend: ' + archive) + + +def main(): + '''The main function. + ''' + info1 = directoryInfo('/etc') + print('{}: file(s): {} / {:.3f} MB dir(s): {} ignored (files/dirs): {} / {}'.format( + '/etc', info1._fileCount, info1._fileSizes / 1024 / 1024.0, + info1._dirCount, info1._ignoredFiles, info1._ignoredDirs)) + lines1 = tail('/etc/fstab', 5, True) + print('{}:\n{}'.format('/etc/fstab', ''.join(lines1))) + + +if __name__ == '__main__': + main() diff --git a/base/JavaConfig.py b/base/JavaConfig.py new file mode 100644 index 0000000..5aaf05b --- /dev/null +++ b/base/JavaConfig.py @@ -0,0 +1,118 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import re +import os.path + + +class JavaConfig: + ''' + Handles a java style configuration file. + Format: + = + # comment + ''' + + def __init__(self, filename, logger, ignoreIniHeader=False): + ''' + Constructor. + @param filename: None: the instance remain "empty" otherwise: the filename with path + @param logger: the logger + @param ignoreIniHeader: True: '[
]' will be ignored + ''' + self._ignoreIniHeader = ignoreIniHeader + self._vars = dict() + self._logger = logger + if filename is not None: + self.readConfig(filename) + + def readConfig(self, filename): + '''Reads the configuration file and put the data into the instance. + @param filename: the name of the configuration file + ''' + self._filename = filename + self._vars = dict() + regExpr = re.compile(r'([\w.]+)\s*=\s*(.*)$') + if not os.path.exists(filename): + self._logger.error('missing ' + filename) + else: + with open(filename, "r") as fp: + lineNo = 0 + for line in fp: + lineNo += 1 + line = line.strip() + if line.startswith('#') or line == '': + continue + matcher = regExpr.match(line) + if matcher is not None: + self._vars[matcher.group(1)] = matcher.group(2) + elif self._ignoreIniHeader and line.startswith('['): + continue + else: + self._logger.error('{:s} line {:d}: unexpected syntax [expected: =]: {:s}'.format( + filename, lineNo, line)) + + def getBool(self, variable, defaultValue=None): + '''Returns the value of a given variable. + @param variable: name of the Variable + @param defaultValue: if variable does not exist this value is returned + @return: None: Variable not found or not a bool value + otherwise: the bool value + ''' + rc = defaultValue + if variable in self._vars: + value = self._vars[variable].lower() + if value in ('t', 'true', 'yes'): + rc = True + elif value in ('f', 'false', 'no'): + rc = False + else: + self._logger.error("{}: variable {} is not a boolean: {}".format( + self._filename, variable, value)) + rc = defaultValue + return rc + + def getInt(self, variable, defaultValue=None): + '''Returns the value of a given variable. + @param variable: name of the Variable + @param defaultValue: if variable does not exist this value is returned + @return: None: Variable not found or not an integer + otherwise: the int value + ''' + rc = defaultValue + if variable in self._vars: + value = self._vars[variable] + try: + rc = int(value) + except ValueError: + self._logger.error("{}: variable {} is not an integer: {}".format( + self._filename, variable, value)) + rc = defaultValue + return rc + + def getString(self, variable, defaultValue=None): + '''Returns the value of a given variable. + @param variable: name of the Variable + @param defaultValue: if variable does not exist this value is returned + @return: None: Variable not found otherwise: the value + ''' + rc = defaultValue if variable not in self._vars else self._vars[variable] + return rc + + def getKeys(self, regExpr=None): + r'''Returns an array of (filtered) keys. + @param regExpr: None or a regular expression to filter keys. regExpr can be an object or a text + example: re.compile(r'^\s*pattern.\d+$', re.I) + @return: the array of sorted keys matching the regExpr + ''' + if isinstance(regExpr, str): + regExpr = re.compile(regExpr) + keys = self._vars.keys() + rc = [] + for key in keys: + if regExpr is None or regExpr.search(key): + rc.append(key) + rc.sort() + return rc diff --git a/base/JobController.py b/base/JobController.py new file mode 100644 index 0000000..408d9fc --- /dev/null +++ b/base/JobController.py @@ -0,0 +1,123 @@ +''' +Wait for jobs written as files in a defined directory. + +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import time +import os.path +import random + +import base.StringUtils + + +class JobController: + '''Wait for jobs written as files in a defined directory. + Needed: overriding process() + Format of the job file: + ... + Example of a job file: + |email|a@bc.de|Greetings|Greetings from your family + ''' + + def __init__(self, jobDirectory, cleanInterval, logger): + '''Constructor. + @param jobDirectory: the jobs will be expected in this directory + @param cleanInterval: files older than this amount of seconds will be deleted + @param logger: for messages + ''' + self._jobDirectory = jobDirectory + if not os.path.exists(jobDirectory): + os.makedirs(jobDirectory, 0o777, True) + self._cleanInterval = cleanInterval + self._logger = logger + + def check(self): + '''Checks whether a new job is requested. + @return: True: a new job has been found and processed + ''' + files = os.listdir(self._jobDirectory) + now = time.time() + found = False + for file in files: + full = self._jobDirectory + os.sep + file + if not file.endswith('.job'): + date = os.path.getmtime(full) + if date < now - self._cleanInterval: + self._logger.log('cleaning too old file {}'.format( + file), base.Const.LEVEL_LOOP) + os.unlink(full) + if os.path.exists(full): + self._logger.error('cannot delete ' + full) + else: + content = base.StringUtils.fromFile(full) + if len(content) < 2: + self._logger.error('cleaning empty job file: ' + file) + os.unlink(full) + continue + separator = content[0] + parts = content[1:].split(separator) + self._logger.log( + 'processing ' + parts[0], base.Const.LEVEL_SUMMARY) + self.process(parts[0], parts[1:]) + self._logger.log('removing processed job file', + base.Const.LEVEL_FINE) + os.unlink(full) + found = True + break + return found + + def jobDirectory(self): + '''Returns the current job directory. + @return: the job directory + ''' + return self._jobDirectory + + def process(self, name, args): + '''Dummy method for processing a job. Must be overridden. + @param name: name of the job + @param args: the arguments as list + @return False: error + ''' + base.StringUtils.avoidWarning(name) + base.StringUtils.avoidWarning(args) + self._logger('missing overriding method process()') + return False + + @staticmethod + def writeJob(name, args, directory, logger): + '''Writes a job into a job file. + @param name: name of the job + @param args: arguments as a list + @param directory: the name of the job directory + @param logger: for messages + @return: True: success + ''' + rc = True + chars = name + ''.join(args) + separator = None + for item in ['|', '\t', '^', '°', '#', '~', '/', '\\', '`', '?', '$', '%']: + if chars.find(item) < 0: + separator = item + break + if separator is None: + logger.error('I am confused: no separator is possible') + rc = False + else: + fn = '{}{}t{:05d}{:x}.xxx'.format(directory, os.sep, int( + time.time()) % 86400, random.randint(0x1000, 0xffff)) + suffix = '' if args is None or not args else separator + \ + separator.join(args) + content = separator + name + suffix + base.StringUtils.toFile(fn, content) + fn2 = fn.replace('.xxx', '.job') + try: + os.rename(fn, fn2) + except OSError as exc: + logger.error(f'cannot rename {fn} to {fn2}') + return rc + + +if __name__ == '__main__': + pass diff --git a/base/LinuxUtils.py b/base/LinuxUtils.py new file mode 100644 index 0000000..5b74dc6 --- /dev/null +++ b/base/LinuxUtils.py @@ -0,0 +1,360 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' + +import os +import subprocess +import re +import stat +import pwd +import grp + +import base.StringUtils + + +def diskFree(verboseLevel=0, logger=None): + '''Returns an info about the mounted filesystems. + @return: a list of info entries: entry: [mountPath, totalBytes, freeBytes, availableBytesForNonPrivilegs] + ''' + if logger is not None and verboseLevel > base.Const.LEVEL_LOOP: + logger.log('taskFileSystem()...', verboseLevel) + + rc = [] + ignoredDevs = ['udev', 'devpts', 'tmpfs', 'securityfs', 'pstore', + 'cgroup', 'tracefs', 'mqueue', 'hugetlbfs', 'debugfs'] + with open('/proc/mounts', 'r') as f: + for line in f: + dev, path, fstype, rest = line.split(None, 3) + base.StringUtils.avoidWarning(rest) + if logger is not None and verboseLevel >= base.Const.LEVEL_FINE: + logger.log(line, verboseLevel) + if (fstype in ['sysfs', 'proc'] or dev in ignoredDevs): + continue + if (path.startswith('/proc/') or path.startswith('/sys/') or path.startswith('/run/') + or path.startswith('/dev/loop') or path.startswith('/snap/')): + continue + if not os.path.isdir(path): + continue + if logger is not None and verboseLevel >= base.Const.LEVEL_FINE: + logger.log(path + '...', verboseLevel) + info = diskInfo(path) + if info is not None: + rc.append(info) + return rc + + +def diskInfo(path): + '''Returns some infe about a mounted block device. + @param path: the mount path + @return: None: not available otherwise: [mountPath, totalBytes, freeBytes, availableBytesForNonPrivilegs] + ''' + rc = None + try: + stat1 = os.statvfs(path) + blocksize = stat1.f_bsize + # ....path, total, free, available + rc = [path, stat1.f_blocks * blocksize, stat1.f_bfree * + blocksize, stat1.f_bavail * blocksize] + except FileNotFoundError: + # if mounted by autofs: the path can not be found + pass + return rc + + +def disksMounted(logger=None): + '''Returns a list of mounted filesystems. + @return: a list of mounted filesystems, e.g. ['/', '/home'] + ''' + if logger is not None: + logger.log('taskFileSystem()...', base.Const.LEVEL_LOOP) + + rc = [] + ignoredDevs = ['udev', 'devpts', 'tmpfs', 'securityfs', 'pstore', + 'cgroup', 'tracefs', 'mqueue', 'hugetlbfs', 'debugfs'] + with open('/proc/mounts', 'r') as f: + found = [] + for line in f: + dev, path, fstype, rest = line.split(None, base.Const.LEVEL_LOOP) + base.StringUtils.avoidWarning(rest) + if dev in found: + continue + found.append(dev) + if logger is not None: + logger.log(line.rstrip(), base.Const.LEVEL_LOOP) + if fstype == 'sysfs' or fstype == 'proc' or dev in ignoredDevs: + continue + if (path.startswith('/proc/') + or path.startswith('/sys/') or path.startswith('/run/') + or path.startswith('/dev/loop') + or path.startswith('/snap/')): + continue + if not os.path.isdir(path): + continue + rc.append(path) + return rc + + +def diskIo(): + '''Returns a list of [diskname, countReads, countWrites, countDiscards] arrays. + Data are accumulated since last boot. + Note: sector size: 512 Byte + @see https://www.kernel.org/doc/Documentation/iostats.txt + @return: array of arrays [id, diskname, countReads, countWrites, countDiscards], e.g. [ ['8-0-sda', 299, 498, 22 ] ] + ''' + rc = [] + with open('/proc/diskstats', 'r') as fp: + for line in fp: + # 1......2.....3....4.............5...........6...........7.........8.............. + # mainid subid name readscomplete readsmerged readsectors readmsecs writescomplete + # 9............10...........11.........12.........13.....14.............15................ + # writesmerged writesectors writesmsec inprogress iomsec weightediomsec discardscompleted + # 16.............17...............18 + # discardsmerged discardssectors discardsmsec + # 8 0 sda 101755 2990 6113900 37622 69827 44895 1535408 41169 0 85216 2732 0 0 0 0 + # 8 1 sda1 82 0 6368 22 0 0 0 0 0 76 0 0 0 0 0 + parts = line.split() + rc.append(['{}-{}'.format(parts[0], parts[1]), + parts[2], parts[5], parts[9], parts[16]]) + return rc + + +def groupId(nameOrId, defaultValue=None): + '''Returns the group id of a given group name. + @param nameOrId: normally a group. If this is a number this will taken as result + @param defaultValue: the result value when the group name is unknown + @return: defaultValue: unknown group name otherwise: the group id + ''' + if isinstance(nameOrId, int): + rc = nameOrId + elif base.StringUtils.asInt(nameOrId) is not None: + rc = base.StringUtils.asInt(nameOrId) + else: + try: + rc = grp.getgrnam(nameOrId).gr_gid + except KeyError: + rc = defaultValue + return rc + + +def isExecutable(statInfo, euid, egid): + '''Tests whether the file or directory) is executable + @param statInfo: the result of os.stat() + @param euid: the effective UID of the current process. We can get it with os.geteuid() + @param egid: the the effective GID of the current process. We can get it with os.getegid() + @return: True: the file is executable + ''' + if statInfo.st_uid == euid: + # S_IXUSR S_IXGRP S_IXOTH + mask = (stat.S_IXUSR | stat.S_IXOTH) + elif statInfo.st_gid == egid: + mask = (stat.S_IXGRP | stat.S_IXOTH) + else: + mask = stat.S_IXOTH + return (statInfo.st_mode & mask) != 0 + + +def isReadable(statInfo, euid, egid): + '''Tests whether the file or directory) is readable. + @param statInfo: the result of os.stat() + @param euid: the effective UID of the current process. We can get it with os.geteuid() + @param egid: the the effective GID of the current process. We can get it with os.getegid() + @return: True: the file is readable + ''' + if statInfo.st_uid == euid: + # S_IXUSR S_IXGRP S_IXOTH + mask = (stat.S_IRUSR | stat.S_IROTH) + elif statInfo.st_gid == egid: + mask = (stat.S_IRGRP | stat.S_IROTH) + else: + mask = stat.S_IROTH + return (statInfo.st_mode & mask) != 0 + + +def isWritable(statInfo, euid, egid): + '''Tests whether the file or directory) is writable. + @param statInfo: the result of os.stat() + @param euid: the effective UID of the current process. We can get it with os.geteuid() + @param egid: the the effective GID of the current process. We can get it with os.getegid() + @return: True: the file is writable + ''' + if statInfo.st_uid == euid: + mask = (stat.S_IWUSR | stat.S_IWOTH) + elif statInfo.st_gid == egid: + mask = (stat.S_IWGRP | stat.S_IWOTH) + else: + mask = stat.S_IWOTH + return (statInfo.st_mode & mask) != 0 + + +def stress(patternDisks, patternInterface): + '''Returns the load data of a server. + Note: the byte data (ioReadBytes ... netWriteBytes) are summarized since boot time. + @param patternDisk: a regular expression of the disk devices used for the result (sum is built), e.g. 'sd[ab]' + @param patternInterface: a regular expression of the network interfaces used for the result (sum is built), e.g. 'eth0|wlan0' + @return: [ioReadBytes, ioWriteBytes, netReadBytes, netWriteBytes, load1Minute, memoryAvailable, swapAvailable] + ''' + readIO = 0 + writeIO = 0 + rexprDisks = base.StringUtils.regExprCompile(patternDisks, 'disk pattern') + with open('/proc/diskstats', 'r') as fp: + for line in fp: + # 1......2.....3....4.............5...........6...........7.........8.............. + # mainid subid name readscomplete readsmerged readsectors readmsecs writescomplete + # 9............10...........11.........12.........13.....14.............15................ + # writesmerged writesectors writesmsec inprogress iomsec weightediomsec discardscompleted + # 16.............17...............18 + # discardsmerged discardssectors discardsmsec + # 8 0 sda 101755 2990 6113900 37622 69827 44895 1535408 41169 0 85216 2732 0 0 0 0 + # 8 1 sda1 82 0 6368 22 0 0 0 0 0 76 0 0 0 0 0 + parts = line.split() + if rexprDisks.match(parts[2]) is not None: + readIO += int(parts[5]) + writeIO += int(parts[9]) + readIO *= 512 + writeIO *= 512 + readNet = 0 + writeNet = 0 + rexprNet = base.StringUtils.regExprCompile( + patternInterface, 'interface pattern') + with open('/proc/net/dev', 'r') as fp: + for line in fp: + # 1......2........3......4....5....6....7.....8..........9.........10....... + # Inter-| Receive | Transmit + # 11.....12....13...14...15....16......17 + # face |bytes packets errs drop fifo frame compressed multicast|bytes + # packets errs drop fifo colls carrier compressed + # lo: 33308 376 0 0 0 0 0 0 33308 + # 376 0 0 0 0 0 0 + parts = line.split() + # remove ':' from the first field: + if rexprNet.match(parts[0][0:-1]) is not None: + readNet += int(parts[1]) + writeNet += int(parts[9]) + with open('/proc/loadavg', 'rb') as fp: + loadMin1 = float(fp.read().decode().split()[0]) + #@return: [TOTAL_RAM, AVAILABLE_RAM, TOTAL_SWAP, FREE_SWAP, BUFFERS] + with open('/proc/meminfo', 'r') as fp: + lines = fp.read().split('\n') + freeRam = _getNumber(lines[2]) + freeSwap = _getNumber(lines[15]) + return [readIO, writeIO, readNet, writeNet, loadMin1, freeRam, freeSwap] + + +def userId(nameOrId, defaultValue=None): + '''Returns the user id of a given user name. + @param nameOrId: normally a username. If this is a number this will taken as result + @param defaultValue: the result value when the user name is unknown + @return: defaultValue: unknown user name otherwise: the user id + ''' + if isinstance(nameOrId, int): + rc = nameOrId + elif base.StringUtils.asInt(nameOrId) is not None: + rc = base.StringUtils.asInt(nameOrId) + else: + try: + rc = pwd.getpwnam(nameOrId).pw_uid + except KeyError: + rc = defaultValue + return rc + + +def users(): + '''Returns the users currently logged in. + @return: None: parser error. otherwise: tuple of entries (USERNAME, IP, LOGINSTART, LOGINDURATION, CPUTIME) + ''' + with subprocess.Popen('/usr/bin/w', stdout=subprocess.PIPE) as proc: + data = proc.stdout.read().decode() + lines = data.split('\n')[2:] + rc = [] + for line in lines: + if line == '': + break + # hm pts/0 88.67.239.209 21:17 1:32 m 6:37 0.04 s w + # hm pts/0 88.67.239.209 21:17 60s 0.00s 0.00s w + parts = line.split() + if len(parts) < 4: + rc = None + break + rc.append((parts[0], parts[2], parts[3], parts[4], + parts[5] if parts[5].find(':') > 0 else parts[6])) + return rc + + +def load(): + '''Returns average loads. + @return: [LOAD_1_MINUTE, LOAD_5_MINUTE, LOAD_10_MINUTE, RUNNING_PROCESSES, PROCESSES] + ''' + with open('/proc/loadavg', 'rb') as fp: + data = fp.read().decode() + matcher = re.match(r'(\S+)\s+(\S+)\s+(\S+)\s+(\d+)/(\d+)', data) + if matcher is None: + rc = None + else: + rc = [float(matcher.group(1)), float(matcher.group(2)), float(matcher.group(3)), + int(matcher.group(4)), int(matcher.group(5))] + return rc + + +def _getNumber(line): + parts = line.split() + return int(parts[1]) + + +def memoryInfo(): + '''Returns the memory usage. + @return: [TOTAL_RAM, AVAILABLE_RAM, TOTAL_SWAP, FREE_SWAP, BUFFERS] + ''' + with open('/proc/meminfo', 'rb') as fp: + lines = fp.read().decode().split('\n') + rc = [_getNumber(lines[0]), _getNumber(lines[2]), _getNumber( + lines[14]), _getNumber(lines[15]), _getNumber(lines[3])] + return rc + + +def mdadmInfo(filename='/proc/mdstat'): + '''Returns the info about the software raid systems. + @return: a list of array [name, type, members, blocks, status>, + e.g. [['md0', 'raid1', 'dm-12[0] dm-13[1]', 1234, 'OK'], ['md1', 'raid0', 'sda1[0] sdb1[1]', 1234, 'broken']] + status: 'OK', 'recovery', 'broken' + ''' + rc = [] + if os.path.exists(filename): + with open(filename, 'r') as fp: + # md2 : active raid1 sdc1[0] sdd1[1] + # md1 : active raid1 hda14[0] sda11[2](F) + rexpr1 = re.compile(r'^(\w+) : active (raid\d+) (.*)') + # 1953378368 blocks super 1.2 [2/2] [UU] + rexpr2 = re.compile(r'^\s+(\d+) blocks.*\[([_U]+)\]') + members = None + for line in fp: + matcher = rexpr1.match(line) + if matcher: + name = matcher.group(1) + aType = matcher.group(2) + members = matcher.group(3) + continue + matcher = rexpr2.match(line) + if matcher: + blocks = matcher.group(1) + status = matcher.group(2) + status2 = 'broken' if status.find( + '_') >= 0 or members is not None and members.find('(F)') > 0 else 'OK' + rc.append([name, aType, members, int(blocks), status2]) + continue + if line.find('recovery') > 0: + rc[len(rc) - 1][4] = 'recovery' + return rc + + +def main(): + '''main function. + ''' + infos = diskFree() + for info in infos: + print(base.StringUtils.join(' ', info)) + + +if __name__ == '__main__': + main() diff --git a/base/Logger.py b/base/Logger.py new file mode 100644 index 0000000..b87b538 --- /dev/null +++ b/base/Logger.py @@ -0,0 +1,56 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import os +import datetime +import base.BaseLogger +import base.Const + +class Logger(base.BaseLogger.BaseLogger): + '''A feature reach class for logging messages of different levels. + ''' + + def __init__(self, logfile, verboseLevel): + '''Constructor. + @param logfile: the file for logging + @param verboseLevel: > 0: logging to stdout too + ''' + base.BaseLogger.BaseLogger.__init__(self, verboseLevel) + self._logfile = logfile + # Test accessability: + try: + with open(self._logfile, 'a'): + pass + os.chmod(self._logfile, 0o666) + except OSError as exc: + msg = '+++ cannot open logfile {}: {}'.format( + self._logfile, str(exc)) + print(msg) + self.error(msg) + + def log(self, message, minLevel=base.Const.LEVEL_SUMMARY): + '''Logs a message. + @param message: the message to log + @param minLevel: the logging is done only if _verboseLevel >= minLevel + @return: true: OK false: error on log file writing + ''' + rc = False + try: + if not self._inUse and self._mirrorLogger is not None: + self._mirrorLogger.log(message) + now = datetime.datetime.now() + message = now.strftime('%Y.%m.%d %H:%M:%S ') + message + if self._verboseLevel >= minLevel: + print(message) + with open(self._logfile, 'a') as fp: + rc = True + fp.write(message + '\n') + except OSError as exc: + print(str(exc)) + return rc + + +if __name__ == '__main__': + pass diff --git a/base/MemoryLogger.py b/base/MemoryLogger.py new file mode 100644 index 0000000..49441ac --- /dev/null +++ b/base/MemoryLogger.py @@ -0,0 +1,102 @@ +''' +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import base.BaseLogger + + +class MemoryLogger(base.BaseLogger.BaseLogger): + '''Implements a logger storing the logging messages in an internal array. + ''' + + def __init__(self, verboseLevel=0, verboseListMinLevel=99): + '''Constructor. + @param verboseLevel: > 0: the messages will be printed (to stdout) + @param verboseListMinLevel: messages with a higher level will be stored + ''' + base.BaseLogger.BaseLogger.__init__(self, verboseLevel) + self._lines = [] + self._listMinLevel = verboseListMinLevel + + def clear(self): + '''Clears all messages and error messages. + ''' + self._lines = [] + self._firstErrors = [] + self._errors = 0 + + def contains(self, string, errorsToo=False): + '''Tests whether the log contains a given string. + @param string: string to search + @param errorsToo: the errors will inspected too + @return: True: the log (or the errors) contains the string + ''' + rc = False + for line in self._lines: + if not errorsToo and line.startswith('+++'): + continue + if line.find(string) >= 0: + rc = True + break + if not rc and errorsToo: + for line in self._firstErrors: + if line.find(string) >= 0: + rc = True + break + return rc + + def derive(self, logger, messagesToo=False): + '''Transfers error to another logger. + ''' + for item in self._firstErrors: + logger.error(item) + if messagesToo: + for item in self._lines: + logger.log(item, 4) + + def getMessages(self): + '''Returns the internal messages as array. + @return: array of messages + ''' + return self._lines + + def log(self, message, minLevel=0): + '''Logs a message. + @param message: the message to log + @param minLevel: the logging is done only if _verboseLevel >= minLevel + @return: True: OK + ''' + if self._verboseLevel >= minLevel: + print(message) + if self._listMinLevel >= minLevel: + self._lines.append(message) + return True + + def matches(self, pattern, flags=0, errorsToo=False): + r'''Tests whether the log contains a given regular expression. + @param pattern: reg expression to search, e.g. r'\d+' + @param flags: flags of the method re.compile(), e.g. re.I (for ignore case) + @param errorsToo: the errors will inspected too + @return: True: the log contains the string + ''' + rc = False + regExpr = base.StringUtils.regExprCompile( + pattern, 'memory logger pattern', None, flags == 0) + if regExpr is not None: + for line in self._lines: + if not errorsToo and line.startswith('+++'): + continue + if regExpr.search(line): + rc = True + break + if not rc and errorsToo: + for line in self._firstErrors: + if regExpr.search(line): + rc = True + break + return rc + + +if __name__ == '__main__': + pass diff --git a/base/PortableRandom.py b/base/PortableRandom.py new file mode 100644 index 0000000..d6bcfc4 --- /dev/null +++ b/base/PortableRandom.py @@ -0,0 +1,73 @@ +''' +Created on 2023 Feb 26 + +@author: wk +''' +import base.BaseRandom + +class PortableRandom(base.BaseRandom.BaseRandom): + ''' + classdocs + ''' + _offsets = [ 477417747081244, 11891569955439, + 1112953394633955, 914449391256858, 574570619325411, 140389009826762, + 608000936401952, 32991187017681, 158039946481952, 700357467346732, + 968941481455215, 25223739256570, 471061646629072, 551047729047099, + 907255805781784, 628387343904130, 893283886891000, 981075091853751, + 776506900100360, 434511969151373, 177432087675717, 491951583746060, + 299997671178038, 1110099901520936, 255708987865834, 578403292103942, + 1093220808197141, 643105169062152, 189554682132125, 947667916711915, + 599096422373408, 662711516075667, 112172498828425, 588478080163912, + 720064562609749, 520347612306989, 330303796102313, 191695345476469, + 419517186899079, 752629552906654, 18312389588488, 88428897947984, + 675611758740110, 1064495336311691 ] + _countNumbers = len(_offsets) + _factors = [ 33554393, 23554387, 33554383, 32559983 ] + _countFactors = len(_factors) + _maxNumber = 1125899906842624 # 2**50 + _maxFactor = 134217728 # 2**27 + def __init__(self): + ''' + Constructor + ''' + self._indexOffset = 0 + self._indexFactor = 0 + self._seed = 0 + self._seed2 = 0 + self.setSeed(0xfeadbeef, 0x1989211, 0x19910420) + + def nextInt(self, value1: int=0x7ffffffe, value2:int=None): + '''Returns a random integer from [0..maxValue] + @param value1: if value2 is None this is the largest return value (inclusive) + Otherwise this is the lowest return value + @param value2: None or the largest return value (inclusive) + @return a random number + ''' + if value2 is None: + self._seed = ((self._seed % PortableRandom._maxFactor) \ + * PortableRandom._factors[self._indexFactor] \ + + PortableRandom._offsets[self._indexOffset] + self._seed2) \ + % PortableRandom._maxNumber + self._indexFactor = (self._indexFactor + 1) % PortableRandom._countFactors + self._indexOffset = (self._indexOffset + 1) % PortableRandom._countNumbers + rc = self._seed % (value1 + 1) + else: + if value1 > value2: + value1, value2 = (value2, value1) + range = value2 - value1 + if range <= 0 or range > base.BaseRandom.BaseRandom._maxInt: + raise ValueError(f'PortableRandom::nextInt(): range too large: {range} / {self._maxInt}') + rc = value1 + self.nextInt(range) + return rc + + def setSeed(self, seed1: int, seed2: int = 0x7654321, seed3: int = 0x3adf001): + '''Sets the generator state with until three secret integers. + @param seed1: the first secret + @param seed2: the second secret + @param seed3: the third secret + ''' + self._seed = abs(seed1 + seed2 + seed3) + self._seed2 = seed2 + self._indexOffset = abs(seed1 + seed3) // 47 % (PortableRandom._countNumbers) + self._indexFactor = abs(seed1 + seed2) // 61 % (PortableRandom._countFactors) + diff --git a/base/ProcessHelper.py b/base/ProcessHelper.py new file mode 100644 index 0000000..8fbdc5d --- /dev/null +++ b/base/ProcessHelper.py @@ -0,0 +1,237 @@ +#! /usr/bin/python3 +''' +processhelper: starting external scripts/programs + +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import subprocess +import tempfile +import os +import time + +import base.StringUtils + + +class ProcessHelper: + '''Executes external processes. + ''' + + def __init__(self, logger): + '''Constructor: + @param logger: display output + ''' + self._logger = logger + self._output = None + self._rawOutput = None + self._error = None + + def execute(self, argv, logOutput, storeOutput=False, mode='!shell', timeout=None, currentDirectory=None): + '''Executes an external program with input from stdin. + @param argv: a list of arguments, starting with the program name + @param logOutput: True: the result of stdout is written to stdout via logger. + @param storeOutput: True: the raw output is available as self._output[] + @param timeout: None or the timeout of the external program + @return: None (logOutput==False) or array of strings + ''' + curDir = self.pushd(currentDirectory) + if argv is None: + self._logger.error('execute(): missing argv (is None)') + elif curDir != '': + self._logger.log('executing: ' + ' '.join(argv), + base.Const.LEVEL_LOOP) + shell = mode == 'shell' + proc = subprocess.Popen( + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell) + (out, err) = proc.communicate(None, timeout) + self._output = [] + self._error = [] + if logOutput or storeOutput: + for line in out.decode().split('\n'): + line2 = line.rstrip() + if len(line) > 1: + if storeOutput: + self._output.append(line2) + if logOutput: + self._logger.log(line2, base.Const.LEVEL_SUMMARY) + for line in err.decode().split('\n'): + msg = line.rstrip() + if msg != '': + self._error.append(msg) + self._logger.error(msg) + self.popd(curDir) + return None if not logOutput else self._output + + def executeCommunicate(self, process, inputString, logOutput, timeout): + '''Handles the output of subprocess calls. + @param process: the process to inspect + @param inputString: None or an input string + @param logOutput: True: output should be returned + @param timeout: the longest time a process should use + ''' + if inputString is None: + (out, err) = process.communicate(timeout=timeout) + else: + (out, err) = process.communicate(inputString.encode(), timeout) + self._rawOutput = out + if logOutput: + for line in out.decode().split('\n'): + if line != '': + self._output.append(line) + self._logger.log(line, base.Const.LEVEL_SUMMARY) + for line in err.decode().split('\n'): + if line != '': + self._error.append(line) + self._logger.error(line) + + def executeInput(self, argv, logOutput, inputString=None, mode='!shell', timeout=None): + '''Executes an external program with input from stdin. + @param argv: a list of arguments, starting with the program name + @param logOutput: True: the result of stdout is written to stdout via logger. + Note: the raw output is available as self._output[] + @param inputString: None or the input for the program as string + @param timeout: None or the timeout of the external program + ''' + self._output = [] + self._error = [] + if inputString is None: + inputString = '' + self._logger.log('executing: ' + ' '.join(argv), base.Const.LEVEL_LOOP) + if mode == 'not used and shell': + fn = tempfile.gettempdir() + '/dbtool.' + str(time.time()) + base.StringUtils.toFile(fn, inputString) + command = argv[0] + " '" + "' '".join(argv[1:]) + "' < " + fn + subprocess.run([command], check=True, shell=True) + os.unlink(fn) + else: + try: + proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=mode == 'shell') + self.executeCommunicate(proc, inputString, logOutput, timeout) + except OSError as exc: + msg = str(exc) + self._logger.error(msg) + self._error = msg.split('\n') + except Exception as exc2: + msg = str(exc2) + self._logger.error(msg) + self._error = msg.split('\n') + + def executeInputOutput(self, argv, inputString=None, logOutput=False, mode='!shell', timeout=None): + '''Executes an external program with input from stdin and returns the output. + @param argv: a list of arguments, starting with the program name + @param inputString: None or the input for the program as string + @param timeout: None or the timeout of the external program + @return: a list of lines (program output to stdout) + ''' + self.executeInput(argv, logOutput, inputString, mode, timeout) + rc = self._output + if (rc is None or not rc) and self._rawOutput is not None and self._rawOutput != '': + try: + rc = self._rawOutput.decode('utf-8').split('\n') + except UnicodeDecodeError as exc: + self._logger.error('executeInputOutput(): {}\n[{}]\n"{}"'.format( + str(exc), ','.join(argv), '' if inputString is None else inputString[0:80])) + rc = base.StringUtils.minimizeArrayUtfError(self._rawOutput.split(b'\n'), + self._logger if self._logger._verboseLevel >= 2 else None) + return rc + + def executeInChain(self, argv1, inputString, argv2, mode='shell', timeout=None): + '''Executes 2 programs with input from stdin as chain and returns the output. + @param argv1: a list of arguments for the first program, starting with the program name + @param inputString: None or the input for the first program as string + @param argv2: a list of arguments for the second program, starting with the program name + @param timeout: None or the timeout of the external program + @return: a list of lines (program output to stdout) + ''' + self._output = [] + self._error = [] + self._logger.log('executing: ' + ' '.join(argv1) + + '|' + ' '.join(argv2), base.Const.LEVEL_LOOP) + rc = [] + if mode == 'shell': + fnOut = tempfile.gettempdir() + '/dbtool.out.' + str(time.time()) + if inputString is None: + inputPart = '' + else: + fnIn = tempfile.gettempdir() + '/dbtool.in.' + str(time.time()) + inputPart = "< '" + fnIn + "' " + base.StringUtils.toFile(fnIn, inputString) + command = (argv1[0] + " '" + "' '".join(argv1[1:]) + "' " + inputPart + "| " + + argv2[0] + " '" + "' '".join(argv2[1:]) + "' > " + fnOut) + try: + subprocess.run([command], check=True, shell=True) + data = base.StringUtils.fromFile(fnOut) + rc = self._output = data.split('\n') + except Exception as exc: + self._logger.error(str(exc)) + if inputString is not None: + os.unlink(fnIn) + os.unlink(fnOut) + else: + try: + p1 = subprocess.Popen(argv1, stdout=subprocess.PIPE) + p2 = subprocess.Popen( + argv2, stdin=p1.stdout, stdout=subprocess.PIPE) + # Allow p1 to receive a SIGPIPE if p2 exits. + p1.stdout.close() + self.executeCommunicate(p2, None, True, timeout) + rc = self._output + except Exception as exc: + self._logger.error(str(exc)) + return rc + + def executeScript(self, script, node=None, logOutput=True, args=None, timeout=None): + '''Executes an external program with input from stdin. + @param script: content of the script + @param node: script name without path (optional) + @param logOutput: True: the result of stdout is written to stdout via logger. + Note: the raw output is available as self._output[] + @param args: None or an array of additional arguments, e.g. ['-v', '--dump'] + @param timeout: None or the timeout of the external program + @return: None (logOutput==False) or array of strings + ''' + self._logger.log('executing {}...'.format( + 'script' if node is None else node), base.Const.LEVEL_LOOP) + if node is None: + node = 'processtool.script' + fn = tempfile.gettempdir() + os.sep + node + str(time.time()) + base.StringUtils.toFile(fn, script) + os.chmod(fn, 0o777) + argv = [fn] + if args is not None: + argv += args + rc = self.execute(argv, logOutput, 'shell', timeout) + os.unlink(fn) + return rc + + def popd(self, directory): + '''Changes the current direcory (if needed and possible). + @param directory: None or the new current directory + @return None: directory = None + '': changing directory failed + otherwise: the current directory (before changing) + ''' + if directory is not None and directory != '': + os.chdir(directory) + if os.path.realpath(os.curdir) != os.path.realpath(directory): + self._logger.error('cannot change to directory ' + directory) + + def pushd(self, directory): + '''Changes the current direcory (if needed and possible). + @param directory: None or the new current directory + @return None: directory = None + '': changing directory failed + otherwise: the current directory (before changing) + ''' + if directory is None: + rc = None + else: + rc = os.curdir + os.chdir(directory) + if os.path.realpath(os.curdir) != os.path.realpath(directory): + os.chdir(rc) + self._logger.error('cannot change to directory ' + directory) + rc = '' + return rc diff --git a/base/Scheduler.py b/base/Scheduler.py new file mode 100644 index 0000000..838d958 --- /dev/null +++ b/base/Scheduler.py @@ -0,0 +1,128 @@ +''' +Administrates a time controlled list of tasks. + +Created: 2020.06.24 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import time +import random + + +class TaskInfo: + '''Abstract class of a task. + Override process(). + ''' + #@abstractmethod + + def process(self, sliceInfo): + '''Processes the task. + @param sliceInfo: the current slice + ''' + raise Exception('TaskInfo.process not overriden') + + +class SliceInfo: + '''Holds the information over an entry in the time table. + ''' + + def __init__(self, taskInfo, scheduler, countCalls=1, interval=60, precision=0.1): + '''Constructor. + @param taskInfo: the task to do + @param scheduler: the parent + @param countCalls: The task has to be repeated so many times + @param interval: the time between two process + @param precision: a factor (< 1.0) of interval to spread the processing timestamps + ''' + self._scheduler = scheduler + self._interval = interval + self._precision = precision + # None: forever otherwise: the number of rounds in the scheduler + self._countCalls = countCalls + self._taskInfo = taskInfo + self._id = scheduler.nextId() + self._nextCall = None + + def calculateNextTime(self, interval=None, precision=None): + '''Calculates the timepoint of the next call. + @param interval: None: _interval is taken otherwise: the amount of seconds to the next processing + @param precision: None: self._precision is taken otherwise: a factor of interval to spread processing timestamps + ''' + if interval is None: + interval = self._interval + if precision is None: + precision = self._precision + if precision == 0.0: + self._nextCall = time.time() + interval + else: + halfRange = interval * precision + rand = self._scheduler._random.randrange(0, 123456) / 123456.0 + offset = interval + 2 * halfRange * rand - halfRange + self._nextCall = time.time() + offset + + +class Scheduler: + '''Administrates a time controlled list of tasks. + Needed: overriding process() + ''' + + def __init__(self, logger): + '''Constructor. + @param logger: for messages + ''' + self._slices = [] + self._logger = logger + self._random = random.Random() + self._currentId = 0 + + def insertSlice(self, sliceInfo, startInterval=None, startPrecision=None): + '''Inserts a slice info into the time ordered slice list. + @param sliceInfo: the slice to insert + @param startInterval: None: sliceInfo._interval is taken. Otherwise: the amount of seconds to the next processing + @param startPrecision: a factor (< 1.0) of startInterval to spread the processing timestamps + ''' + ix = len(self._slices) - 1 + sliceInfo.calculateNextTime(startInterval, startPrecision) + if sliceInfo._id == 2: + sliceInfo._id = 2 + while ix >= 0 and self._slices[ix]._nextCall > sliceInfo._nextCall: + ix -= 1 + self._slices.insert(ix + 1 if ix >= 0 else 0, sliceInfo) + + def check(self): + '''Checks whether the next task should be processed and returns it. + @return: None: no processing is needed otherwise: the slice which must be processed + ''' + sliceInfo = None + if self._slices: + now = time.time() + found = self._slices[0]._nextCall <= now + if found: + sliceInfo = self._slices[0] + del self._slices[0] + return sliceInfo + + def checkAndProcess(self): + '''Checks whether the next task should be processed and processes it. + @return: true: processing has been done + ''' + sliceInfo = self.check() + if sliceInfo is not None: + sliceInfo._taskInfo.process(sliceInfo) + if sliceInfo._countCalls is not None: + sliceInfo._countCalls -= 1 + if sliceInfo._countCalls is None or sliceInfo._countCalls > 0: + sliceInfo.calculateNextTime() + self.insertSlice(sliceInfo) + return sliceInfo is not None + + def nextId(self): + '''Returns the next id for a slice. + @return: the next id + ''' + self._currentId += 1 + return self._currentId + + +if __name__ == '__main__': + pass diff --git a/base/SearchRule.py b/base/SearchRule.py new file mode 100644 index 0000000..4173708 --- /dev/null +++ b/base/SearchRule.py @@ -0,0 +1,639 @@ +''' +Created: 2020.07.19 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import base.StringUtils + + +class CommandData: + '''Properties given for a command (action except search and reposition). + @param register: None or the related register ('A'..'Z') + @param marker: None or the related marker: 'a' .. 'z' + @param text: None or the related text + @param text2: None or the related 2nd text + @param group: None or the related reg. expression group: 0..N + @param escChar: None or a prefix character to address registers (@see parseRuleReplace()) + @param options: None or an command specific options + ''' + + def __init__(self, register=None, register2=None, marker=None, text=None, text2=None, + group=None, escChar=None, options=None): + self._register = register + self._register2 = register2 + self._marker = marker + self._text = text + self._text2 = text2 + self._group = group + self._options = options + self._escChar = escChar + + def getText(self, state, second=False): + '''Replaces register placeholders with the register content. + Note: register placeholders starts with self._escChar followed by the register name, e.g. '$A' + @param state: the ProcessState instance with the registers + @param second: True: _text2 is used False: _text is used + @return text with replaced register placeholders + ''' + text = self._text2 if second else self._text + if self._escChar is None: + rc = text + else: + startIx = 0 + rc = '' + while startIx + 2 < len(text): + ix = text.find(self._escChar, startIx) + if ix < 0: + break + rc += text[startIx:ix] + name = text[ix + 1] + if 'A' <= name <= 'Z': + rc += state.getRegister(name) + else: + rc += self._escChar + name + startIx = ix + 2 + rc += text[startIx:] + return rc + + +class FlowControl: + '''Flow control of a rule: continue, stop or jump on a condition + ''' + # ..................................A.........A + + def __init__(self): + '''Constructor. + ''' + self._onSuccess = 'c' + self._onError = 'e' + + def setControl(self, control): + '''Translate the options given as string into the class variables. + @param control: the control as string + @param logger: only internal errors are possible + @return: None: success otherwise: error message + ''' + rc = None + reaction = None + if control[-1] == '%': + reaction = control[control.find('%'):] + elif control.endswith('continue'): + reaction = 'c' + elif control.endswith('stop'): + reaction = 's' + elif control.endswith('error'): + reaction = 'e' # error + else: + rc = 'unknown control: ' + control + if control.startswith('success'): + self._onSuccess = reaction + elif control.startswith('error'): + self._onError = reaction + else: + rc = 'unknown control statement: ' + control + return rc + + +class Position: + '''Constructor. + @param line: the line number + @param col: the column number + ''' + + def __init__(self, line, col): + self._line = line + self._col = col + + def toString(self): + '''Returns the position as string. + @return: : + ''' + return '{}:{}'.format(self._line, self._col) + + def check(self, lines, behindLineIsAllowed=False): + '''Checks, whether the instance is valid in lines. + @param lines: the list of lines to inspect + @param behindLineIsAllowed: True: the column may be equal the line length + @return: True: the cursor is inside the lines + ''' + rc = self._line < len(lines) and self._col <= len( + lines[self._line]) - (1 if behindLineIsAllowed else 0) + return rc + + def clone(self, source): + '''Transfers the internal state from the source to the self. + @param source: the Position instance to clone + ''' + self._line = source._line + self._col = source._col + + def compare(self, other): + '''Compares the instance with an other instance + @param other: the Position instance to compare + @return: <0: self < other 0: self==other >0: self>other + ''' + rc = self._line - other._line + if rc == 0: + rc = self._col - other._col + return rc + + def endOfLine(self, lines): + '''Tests whether the instance is one position behind the current line. + @param lines: the list of lines to inspect + @return: True: the instance points to the position one behind the current line or the beginning of the next line + ''' + rc = self._line == len( + lines) and self._col == 0 or self._col == len(lines[self._line]) + return rc + + +class ProcessState: + '''Reflects the state while processing a rule list. + ''' + + def __init__(self, lines, startRange, endRange, start, logger, maxLoops=10): + '''Constructor. + @param lines: the list of lines to inspect + @param startRange: the rule starts at this position + @param endRange: the end of the rules must be below this position + @param start: the rule starts from this position + @param logger: + @param maxLoops: the number of executed rules is limited to maxLoops*len(lines) + ''' + self._logger = logger + self._lines = lines + self._maxLoops = maxLoops + self._cursor = Position(start._line, start._col) + self._startRange = startRange + self._endRange = endRange + self._logger = logger + self._success = True + self._lastMatch = None + # replaces temporary _startRange or _endRange + self._tempRange = Position(0, 0) + self._safePosition = Position(0, 0) + # : Position + self._markers = {} + # : string + self._registers = {} + self._hasChanged = False + self._lastHits = 0 + + def deleteToMarker(self, name): + '''Deletes the text from the cursor to the marker. + @param name: a bound of the region to delete, _position is the other + ''' + + marker = self.getMarker(name) + self._success = marker is not None and self.inRange( + marker) and self.inRange() + if self._success: + comp = self._cursor.compare(marker) + start = marker if comp >= 0 else self._cursor + end = self._cursor if comp >= 0 else marker + ixStart = start._line + deletedLines = 0 + self._hasChanged = True + if start._line == end._line: + self._lines[ixStart] = self._lines[ixStart][0:start._col] + \ + self._lines[ixStart][end._col:] + else: + prefix = '' if start._col == 0 else self._lines[start._line][0:start._col] + ixEnd = end._line if end._col > 0 else end._line + 1 + if end._col > 0: + self._lines[end._line] = prefix + \ + self._lines[end._line][end._col:] + for ix in range(ixStart, ixEnd): + del self._lines[ix] + deletedLines += 1 + # Adapt the existing markers: + for name2 in self._markers: + current = self.getMarker(name2) + if current.compare(start) >= 0: + if current._line > end._line or current._line == end._line and end._col == 0: + current._line -= deletedLines + elif current._line == end._line: + if current._col > end._col: + current._col -= end.col + current.clone(start) + else: + current.clone(start) + + def insertAtCursor(self, text): + '''Inserts a text at the cursor. + @param text: the text to insert, may contain '\n' + ''' + self._success = self.inRange() + if self._success: + newLines = text.split('\n') + curLine = self._cursor._line + self._hasChanged = True + if len(newLines) == 1: + insertedLines = 0 + colNew = self._cursor._col + len(text) + self._lines[curLine] = (self._lines[curLine][0:self._cursor._col] + newLines[0] + + self._lines[curLine][self._cursor._col:]) + else: + insertedLines = len(newLines) + tail = '' + ixNew = 0 + if self._cursor._col > 0: + ixNew = 1 + tail = self._lines[self._cursor._line][self._cursor._col:] + self._lines[curLine] = self._lines[curLine][0:self._cursor._col] + newLines[0] + curLine += 1 + insertedLines -= 1 + ixLast = len(newLines) + while ixNew < ixLast: + self._lines.insert(curLine, newLines[ixNew]) + ixNew += 1 + curLine += 1 + self._lines[curLine - 1] = self._lines[curLine - 1] + tail + colNew = len(newLines[-1]) + for name in self._markers: + marker = self._markers[name] + if marker.compare(self._cursor) >= 0: + if marker._line == self._cursor._line and marker._col > self._cursor._col: + marker._line += insertedLines + marker._col += len(newLines[-1]) + elif marker._line == self._cursor._line > 0: + marker._line += insertedLines + self._cursor._line += insertedLines + self._cursor._col = colNew + + def getMarker(self, name): + '''Returns the marker given by the name ('a'..'z') + @param name: the marker's name: 'a'..'z' + @return: None: not found otherwise: the Position instance + ''' + rc = None if not name in self._markers else self._markers[name] + return rc + + def getRegister(self, name, maxLength=None): + '''Returns the marker given by the name ('a'..'z') + @param name: the marker's name: 'a'..'z' + @return: '': not found otherwise: the register content + ''' + rc = '' if not name in self._registers else self._registers[name] + if maxLength is not None: + rc = base.StringUtils.limitLength2( + rc, maxLength).replace('\n', '\\n') + return rc + + def inRange(self, position=None): + '''Returns whether a position is in the current range. + @param position: a Position instance to test + @return: position is between _startRange and _endRange + ''' + if position is None: + position = self._cursor + rc = (position._line > self._startRange._line + or position._line == self._startRange._line and position._col >= self._startRange._col) + rc = rc and (position._line < self._endRange._line or position._line == self._endRange._line + and position._col <= self._endRange._col) + return rc + + def putToRegister(self, name, text, append=False): + '''Sets the register with a text. + @param name: the register name: 'A'..'Z' + @param text: the text to set + @param append: True: the text will be appended False: the text will be set + ''' + if not append or not name in self._registers: + self._registers[name] = text + else: + self._registers[name] += text + + def setMarker(self, name): + '''Sets the marker from the current position. + @param name: the marker name: 'a'..'z' + ''' + if not name in self._markers: + self._markers[name] = Position(0, 0) + self._markers[name].clone(self._cursor) + + def textToMarker(self, name): + '''Returns the text between the marker name and the cursor. + @param name: the marker's name + @return: the text between marker and cursor (current position) + ''' + rc = '' + marker = self.getMarker(name) + if marker is not None and self.inRange(marker) and self.inRange(): + comp = self._cursor.compare(marker) + start = marker if comp >= 0 else self._cursor + end = self._cursor if comp >= 0 else marker + ixStart = start._line + if start._line == end._line: + rc = self._lines[start._line][start._col:end._col] + else: + if start._col > 0: + prefix = self._lines[start._line][start._col:] + ixStart += 1 + rc = '\n'.join(self._lines[ixStart:end._line]) + if start._col > 0: + if rc == '': + rc = prefix + else: + rc = prefix + '\n' + rc + if end._col > 0: + if rc != '': + rc += '\n' + rc += self._lines[end._line][0:end._col] + return rc + + +class Region: + '''Stores the data of a region, which is a part of the file given by a start and an end position. + ''' + + def __init__(self, parent, startRuleList=None, endRuleList=None, endIsIncluded=False): + '''Constructor. + @param parent: the TextProcessor instance + @param startRuleList: None or the compiled rules defining the start of the region + @param startRuleList: None or the compiled rules defining the end of the region + @param endIsIncluded: True: if the last rule is a search: the hit belongs to the region + ''' + self._parent = parent + self._startRules = startRuleList + self._endRules = endRuleList + self._startPosition = Position(0, 0) + # index of the first line below the region (exclusive) + self._endPosition = Position(0, 0) + self._endIsIncluded = endIsIncluded + self._start = None + self._end = None + + def next(self): + '''Search the next region from current region end. + @param ixFirst: the index of the first line (in parent._lines) to inspect + @return: True: the start has been found False: not found + ''' + rc = self._parent.apply( + self._startRules, self._endPosition, self._parent._endOfFile) + return rc + + def find(self, pattern): + '''Searches the pattern in the region. + @param pattern: a string or a RegExp instance to search + @return: -1: not found otherwise: the index of the first line matching the pattern + ''' + rc = self._parent.findLine(pattern, self._start, self._end) + return rc + + +class SearchRule: + '''Describes a single action: search, reposition, set label/bookmark, print.... + @see SearchRuleList vor details. + ''' + + def __init__(self, ruleType, param=None): + '''Constructor. + @param ruleType: the type: '<' (search backwards) '>': search forward 'l': line:col 'a': anchor + @param parameter: parameter depending on ruleType: RegExp instance for searches, + names for anchors or a [, ] array for ruleType == 'l' + ''' + self._ruleType = ruleType + self._param = param + self._flowControl = FlowControl() + + def name(self, extended=False): + '''Returns the command name. + @return the command name + ''' + rc = None + if self._ruleType == '%': + rc = 'label' + ((' ' + self._param) if extended else '') + elif self._ruleType == '>': + rc = 'search (forward)' + ((' /' + + self._param._regExpr.pattern + '/') if extended else '') + elif self._ruleType == '<': + rc = 'search (backward)' + ((' /' + + self._param._regExpr.pattern + '/') if extended else '') + elif self._ruleType == '+': + rc = 'reposition' + if extended: + rc += ' {}{}:{}'.format(self._param[2], + self._param[0], self._param[1]) + elif self._ruleType == 'anchor': + rc = self._param + else: + rc = self._ruleType + return rc + + def searchForward(self, state): + '''Searches forward in lines in the range given by startRange and endRange. + @param processState: IN/OUT: the context of searching, an instance of ProcessState + ''' + state._safePosition.clone(state._cursor) + state._success = state.inRange() + if state._success: + startIx = state._cursor._line + endIx = min(len(state._lines), state._endRange._line + 1) + if self._param._rangeLines is not None: + endIx = min(startIx + self._param._rangeLines, endIx) + match = None + regExpr = self._param._regExpr + for ix in range(startIx, endIx): + if ix == state._startRange._line: + match = regExpr.search( + state._lines[ix], state._startRange._col) + elif ix == state._endRange._line: + match = regExpr.search( + state._lines[ix], 0, state._endRange._col) + else: + match = regExpr.search( + state._lines[ix], state._startRange._col) + if match is not None: + break + state._success = match is not None + state._lastMatch = match + if state._success: + state._cursor._line = ix + state._cursor._col = match.end( + 0) if self._param._useEnd else match.start(0) + state._success = state.inRange() + + def searchBackward(self, state): + '''Searches backward in lines in the range given by startRange and endRange. + @param processState: IN/OUT: the context of searching, an instance of ProcessState + ''' + state._safePosition.clone(state._cursor) + state._success = state.inRange() + if state._success: + regExpr = self._param._regExpr + startIx = max(0, min(state._cursor._line, len(state._lines) - 1)) + endIx = state._startRange._line - 1 + state._lastMatch = None + if self._param._rangeLines is not None: + endIx = max(state._startRange._line - 1, + max(-1, startIx - self._param._rangeLines)) + for ix in range(startIx, endIx, -1): + if ix == state._startRange._line: + iterator = regExpr.finditer( + state._lines[ix], state._startRange._col) + elif ix == state._endRange._line: + iterator = regExpr.finditer( + state._lines[ix], 0, state._endRange._col) + else: + iterator = regExpr.finditer( + state._lines[ix], state._startRange._col) + # Look for the last match: + for match in iterator: + state._lastMatch = match + if state._lastMatch is not None: + break + state._success = state._lastMatch is not None + if state._success: + state._cursor._line = ix + state._cursor._col = state._lastMatch.end( + 0) if self._param._useEnd else state._lastMatch.start(0) + state._success = state.inRange() + + def reposition(self, state): + '''Apply the reposition rule: an anchor or a line/col move. + @param processState: IN/OUT: the context of searching, an instance of ProcessState + ''' + if self._ruleType == '+': + if self._param[2] == ':': + state._cursor._line = self._param[0] + state._cursor._col = self._param[1] + else: + state._cursor._line += self._param[0] + state._cursor._col += self._param[1] + # if the new line is shorter than the old col position: goto last column + # except the column is explicitly set + if self._param[1] == 0 and state._cursor._line < len(state._lines): + lineLength = len(state._lines[state._cursor._line]) + if state._cursor._col >= lineLength: + state._cursor._col = max(0, lineLength - 1) + elif self._param == 'bof': + state._cursor._line = state._cursor._col = 0 + elif self._param == 'eof': + state._cursor._line = len(state._lines) + state._cursor._col = 0 + elif self._param == 'bol': + state._cursor._col = 0 + elif self._param == 'eol': + # overflow is allowed: + state._cursor._line += 1 + state._cursor._col = 0 + elif self._param == 'bopl': + state._cursor._line -= 1 + state._cursor._col = 0 + elif self._param == 'eopl': + state._cursor._col = 0 + elif self._param == 'bonl': + # overflow is allowed: + state._cursor._line += 1 + state._cursor._col = 0 + elif self._param == 'eonl': + # overflow is allowed: + state._cursor._line += 2 + state._cursor._col = 0 + else: + state._logger.error( + 'reposition(): unknown anchor: {}'.format(self._param)) + state._success = state.inRange() + + def state(self, after, state): + '''Returns the "state" of the rule, used for tracing. + @param after: True: the state is after the rule processing + @param state: the ProcessState instance + @return: the specific data of the rule + ''' + def cursor(): + return state._cursor.toString() + + def marker(): + name = self._param._marker + return '{}[{}]'.format(name, '-' if name not in state._markers else state.getMarker(self._param._marker).toString()) + + def register(): + name = self._param._register + return ((name if name is not None else '') + ':' + + ('' if name not in state._registers else state.getRegister(name, 40))) + name = self._ruleType + if not after: + state._traceCursor = cursor() + state._traceState = '' + rc = '' + else: + rc = '{} => {}, {} => '.format( + state._traceCursor, cursor(), state._traceState) + if name in ('>', '<', '+', 'anchor', 'swap'): + rc += '-' + elif name in ('add', 'insert', 'set', 'expr', 'state'): + rc += register() + elif name == 'cut': + # cut-m + # cut-R-m + rc += '' if after else marker() + ' / ' + if after and self._param._register is not None: + rc += register() + else: + rc += '-' + elif name == 'group': + rc += register() + elif name == 'jump': + if after and self._param._marker is not None: + rc += state.getMarker(self._param._marker).toString() + else: + rc += '-' + elif name == 'mark': + rc += marker() + elif name == 'print': + if self._param._marker is not None: + rc += marker() if after else '-' + elif self._param._register is not None: + rc += register() if after else '' + elif name == 'replace': + if self._param._register is not None: + rc += register() + elif self._param._marker is not None and after: + rc += marker() + else: + rc += '-' + if after: + rc += ' hits: {}'.format(state._lastHits) + if not after: + state._traceState = rc + return rc + + def toString(self): + '''Returns a string describing the instance. + @return: a string describing the instance + ''' + name = self._ruleType + if name in ('>', '<'): + rc = name + '/' + \ + base.StringUtils.limitLength2( + self._param._regExpr.pattern, 40) + '/' + elif name == '+': + rc = 'reposition {}{}:{}'.format( + self._param[2], self._param[0], self._param[1]) + elif name == 'anchor': + rc = self._param + elif name == 'jump': + rc = name + '-' + \ + (self._param._marker if self._param._marker is not None else self._param._text) + elif name == '%': + rc = 'label ' + self._param + elif name == 'replace': + rc = name + if self._param._register is not None: + rc += '-' + self._param._register + if self._param._marker is not None: + rc += '-' + self._param._marker + rc += (':/' + base.StringUtils.limitLength2(self._param._text, 20) + + '/' + base.StringUtils.limitLength2(self._param._text2, 20)) + else: + rc = name + if self._param._register is not None: + rc += '-' + self._param._register + if self._param._marker is not None: + rc += '-' + self._param._marker + if self._param._text is not None: + rc += ':"' + \ + base.StringUtils.limitLength2(self._param._text, 40) + '"' + return rc diff --git a/base/SearchRuleList.py b/base/SearchRuleList.py new file mode 100644 index 0000000..dada11c --- /dev/null +++ b/base/SearchRuleList.py @@ -0,0 +1,869 @@ +''' +Created: 2020.07.19 +@license: CC0 https://creativecommons.org/publicdomain/zero/1.0 +@author: hm +''' +import re + +import base.Const +import base.StringUtils +import base.SearchRule + +class SearchRuleList: + '''A list of rules to find a new position or do some other things. + @see describe() for detailed description. + ''' + #..........................rule + # .........................1 + __reRule = re.compile(r'%[a-zA-Z_]\w*%:' + # rule + #........A + + r'|(?:[be]of|[be]o[pn]?l' + #...........line........col + #.............1...1......2...2 + + r'|[+-]?(\d+):[+-]?(\d+)' + #..............sep.....sep rows/cols + #..............3..3........C.......C + + r'|[<>FB](\S).+?\3\s?(?::?\d+)?\s?[ie]{0,2}' + #.command.name + # .......4E + + r'|((?:add|cut|expr|group|insert|jump' + # ........................................./name + # .........................................E + + r'|mark|print|replace|set|state|swap)' + #..suffix1 name...............suffix2.........text.delim......txt-opt /t /command + # ......F...G........... ....GF.H...........H.I...5.....5.....J......J.I.4 + + r'(?:-(?:[a-zA-Z]|\d\d?))?(?:-[a-zA-Z])?(?::([^\s]).*?\5(?:e=\S)?)?)' + #.......A + + r')') + __reRuleExprParams = re.compile(r'[-+/*%](\$[A-Z]|\d+)?') + # .........................1......12..............2.3.........3 + __reCommand = re.compile(r'([a-z]+)(-[a-zA-Z]|-\d+)?(-[a-zA-Z])?') + __reRuleStateParam = re.compile(r'rows?|col|size-[A-Z]|rows-[A-Z]|hits') + __reFlowControl = re.compile( + r'(success|error):(continue|error|stop|%\w+%)') + # ....................................A......... A..1......12...2..3..3..4...........4 + __reRuleReplace = re.compile( + r'replace(?:-[a-zA-Z])?:([^\s])(.+?)\1(.*?)\1(e=.|,|c=\d+)*') + # x=re.compile(r'replace:([^\s])(.+)\1(.*)\1(e=.)?') + + def __init__(self, logger, rules=None): + '''Constructor. + @param rules: None or the rules as string + Example: '>/logfile:/ -2:0 bol': + search forwards "logfile:" go backward 2 line 0 column, go to begin of line + ''' + self._logger = logger + self._col = 0 + self._currentRule = '' + self._errorCount = 0 + self._rules = [] + self._labels = {} + self._markers = {} + self._fpTrace = None + self._maxLoops = None + if rules is not None: + self.parseRules(rules) + + def appendCommand(self, name, commandData): + '''Stores the command data in the _rules. + Stores markers defined by "mark". + Tests used markers for a previous definition. + @param name: the name of the command, e.g. 'add'. Will be used as _ruleType + @param commandData: an instance of base.SearchRule.CommandData + ''' + if name == 'mark': + self._markers[commandData._marker] = self._col + elif commandData._marker is not None and commandData._marker not in self._markers: + self.parseError( + 'marker {} was not previously defined'.format(commandData._marker)) + self._rules.append(base.SearchRule.SearchRule(name, commandData)) + + def apply(self, state): + '''Executes the internal stored rules in a given list of lines inside a range. + @param state: IN/OUT IN: the context to search OUT: the state at the end of applying the rule list + ''' + ix = 0 + count = 0 + maxCount = len(state._lines) * state._maxLoops + while ix < len(self._rules): + if count >= maxCount: + state._logger.error( + 'base.SearchRule.SearchRule.apply(): to many loops: {}'.format(self._maxLoops)) + break + item = self._rules[ix] + if self._fpTrace is not None: + ixTrace = ix + self.trace(ixTrace, False, state) + flowControl = self._rules[ix]._flowControl + ix += 1 + if item._ruleType == '>': + item.searchForward(state) + elif item._ruleType == '<': + item.searchBackward(state) + elif item._ruleType == '%': + # label + pass + elif item._ruleType == 'anchor' or item._ruleType == '+': + item.reposition(state) + elif item._ruleType >= 'p': + self.applyCommand2(item, state) + else: + ix2 = self.applyCommand1(item, state) + if ix2 is not None: + ix = ix2 + if self._fpTrace is not None: + self.trace(ixTrace, True, state) + if flowControl is not None: + reaction = flowControl._onSuccess if state._success else flowControl._onError + if reaction == 'c': + pass + elif reaction == 's': + break + elif reaction == 'e': + self._logger.error('{} stopped with error') + break + elif reaction in self._labels: + ix = self._labels[reaction] + 1 + + def applyCommand1(self, rule, state): + '''Executes the action named 'a*' to 'p*' (exclusive) + @param processState: IN/OUT IN: the context to search OUT: the state at the end of applying the rule list + @return: None: normal processing otherwise: the index of the next rule to process + ''' + rc = None + name = rule._ruleType + checkPosition = False + if name == 'add': + # add-R-m + # add-R-S + # add-R DD + if rule._param._marker is not None: + text = state.textToMarker(rule._param._marker) + elif rule._param._register2 is not None: + text = state.getRegister(rule._param._register2) + elif rule._param._text is not None: + text = rule._param.getText(state) + else: + state._logger.error('add: nothing to do') + text = '' + state.putToRegister(rule._param._register, text, append=True) + elif name == 'cut': + # cut-m + # cut-R-m + if rule._param._register is not None: + text = state.textToMarker(rule._param._marker) + state.putToRegister(rule._param._register, text) + state.deleteToMarker(rule._param._marker) + elif name == 'expr': + # expr-R:"+4" + value = base.StringUtils.asInt( + state.getRegister(rule._param._register), 0) + param = rule._param.getText(state) + value2 = base.StringUtils.asInt(param[1:], 0) + op = param[0] + if op == '+': + value += value2 + elif op == '-': + value -= value2 + elif op == '*': + value *= value2 + elif op == '/': + if value2 == 0: + state._success = self._logger.error( + 'division by 0 is not defined') + else: + value //= value2 + elif op == '%': + if value2 == 0: + state._success = self._logger.error( + 'modulo 0 is not defined') + else: + value %= value2 + state._registers[rule._param._register] = str(value) + elif name == 'insert': + # insert-R + # insert DD + text = '' + if rule._param._register is not None: + text = state.getRegister(rule._param._register) + elif rule._param._text is not None: + text = rule._param.getText(state) + state.insertAtCursor(text) + elif name == 'group': + # group-G-R + state._success = state._lastMatch is not None and state._lastMatch.lastindex <= rule._param._group + if state._success: + text = '' if state._lastMatch.lastindex < rule._param._group else state._lastMatch.group( + rule._param._group) + state.putToRegister(rule._param._register, text) + elif name == 'jump': + if rule._param._marker is not None: + state._cursor.clone(state.getMarker(rule._param._marker)) + checkPosition = True + else: + rc = self._labels[rule._param._text] + elif name == 'mark': + state.setMarker(rule._param._marker) + else: + state._logger.error('applyCommand1: unknown command') + if checkPosition: + state._success = state.inRange() + return rc + + def applyCommand2(self, rule, state): + '''Executes the actions named 'p*' to 'z*' (inclusive) + @param processState: IN/OUT IN: the context to search OUT: the state at the end of applying the rule list + ''' + name = rule._ruleType + if name == 'print': + state._success = True + if rule._param._register is not None: + print(state.getRegister(rule._param._register)) + elif rule._param._marker is not None: + print(state.textToMarker(rule._param._marker)) + elif rule._param._text is not None: + print(rule._param.getText(state)) + elif name == 'replace': + param = rule._param + if param._register is not None: + replaced, state._lastHits = re.subn( + param._text, param._text2, state.getRegister(param._register)) + state._registers[param._register] = replaced + elif param._marker is not None: + SearchRuleList.applyReplaceRegion(state._cursor, state.getMarker(param._marker), + re.compile(param._text), param._text2, state) + else: + # replace in the current line: + line = state._lines[state._cursor._line] + replaced, state._lastHits = re.subn( + param._text, param._text2, line) + if line != replaced: + state._hasChanged = True + state._lines[state._cursor._line] = replaced + elif name == 'set': + if rule._param._marker is not None: + text = state.textToMarker(rule._param._marker) + elif rule._param._register2 is not None: + text = state.textToMarker(rule._param._marker) + elif rule._param._text is not None: + text = rule._param.getText(state) + else: + state._logger.error('set: nothing to do') + text = '' + state.putToRegister(rule._param._register, text) + elif name == 'state': + name = rule._param._text + if name == 'row': + value = state._cursor._line + 1 + elif name == 'col': + value = state._cursor._col + 1 + elif name == 'rows': + value = len(state._lines) + elif name.startswith('size-'): + value = len(state.getRegister(name[5])) + elif name.startswith('rows-'): + value = state.getRegister(name[5]).count('\n') + elif name == 'hits': + value = state._lastHits + else: + value = '?' + state._registers[rule._param._register] = str(value) + elif name == 'swap': + marker = state.getMarker(rule._param._marker) + if marker is None: + state._success = False + state._logger.error( + 'swap: marker {} is not defined'.format(rule._param._marker)) + else: + state._tempRange.clone(state._cursor) + state._cursor.clone(marker) + marker.clone(state._tempRange) + state._success = state.inRange() + else: + self._logger.error( + 'unknown command {} in {}'.format(name, rule._ruleType)) + + @staticmethod + def applyReplaceRegion(start, end, what, replacement, state): + '''Replaces inside the region. + @param start: first bound of the region + @param end: second bound of the region + @param what: the regular expression to search + @param replacement: the string to replace + @param state: a base.SearchRule.ProcessState instance + ''' + if start.compare(end) > 0: + start, end = end, start + state._lastHits = 0 + if start._line == end._line: + value = state._lines[start._line][start._col:end._col] + value2, hits = what.subn(replacement, value) + if value != value2: + state._lastHits += hits + state._hasChanged = True + state._lines[start._line] = state._lines[start._line][0:start._col] + \ + value2 + state._lines[end._line][end._col:] + else: + startIx = start._line + if start._col > 0: + value = state._lines[start._line][start._col:] + value2, hits = what.subn(replacement, value) + if value != value2: + state._lastHits += hits + state._hasChanged = True + state._lines[start._line] = state._lines[start._line][0:start._col] + value2 + startIx += 1 + for ix in range(startIx, end._line): + value = state._lines[ix] + value2, hits = what.subn(replacement, value) + if value != value2: + state._lastHits += hits + state._hasChanged = True + state._lines[ix] = value2 + if end._col > 0: + value = state._lines[end._line][0:end._col] + value2, hits = what.subn(replacement, value) + if value != value2: + state._lastHits += hits + state._hasChanged = True + state._lines[end._line] = value2 + \ + state._lines[end._line][end._col:] + + def check(self): + '''Tests the compiled rules, e.g. existence of labels. + @return: None OK otherwise: the error message + ''' + self._labels = {} + ix = -1 + for rule in self._rules: + ix += 1 + if rule._ruleType == '%': + self._labels[rule._param] = ix + for rule in self._rules: + if rule._flowControl is not None: + label = rule._flowControl._onSuccess + if label.startswith('%') and label not in self._labels: + self._logger.error( + 'unknown label (on success) {}'.format(label)) + self._errorCount += 1 + label = rule._flowControl._onError + if label.startswith('%') and label not in self._labels: + self._logger.error( + 'unknown label (on error): {}'.format(label)) + self._errorCount += 1 + if rule._ruleType == 'jump' and rule._param._text is not None and rule._param._text not in self._labels: + self._logger.error( + 'unknown jump target: {}'.format(rule._text)) + self._errorCount += 1 + rc = self._errorCount == 0 + return rc + + @staticmethod + def describe(): + '''Describes the rule syntax. + ''' + print(r'''A "rule" describes a single action: find a new position, set a marker/register, display... +A "register" is a container holding a string with a single uppercase letter as name, e.g. 'A' +A "marker" is a position in the text with a lowercase letter as name, e.g. "a" +A "label" is named position in the rule list delimited by '%', e.g. '%eve_not_found%' +Legend: + D is any printable ascii character (delimiter) except blank and ':', e.g. '/' or "X" + R is a uppercase character A..Z (register name), e.g. "Z" + m is a lowercase ascii char a..z (marker name), e.g. "f" + G is a decimal number, e.g. 0 or 12 (group number) + cursor: the current position +Rules: +a label: +