From: Hamatoma Date: Thu, 15 Jul 2021 20:37:16 +0000 (+0200) Subject: daily work X-Git-Url: https://gitweb.hamatoma.de/?a=commitdiff_plain;h=152026e95a3ec0619c2aab5b29c8ca7a911ed894;p=exhibition.git daily work * sub project rest_server runs now (not complete) * sub project client_server runs now (not complete) --- diff --git a/lib/persistence/persistence.dart b/lib/persistence/persistence.dart new file mode 100644 index 0000000..ec23a68 --- /dev/null +++ b/lib/persistence/persistence.dart @@ -0,0 +1,15 @@ +abstract class Persistence { + /// Fetches data specified by [what] and some [parameters]. + /// Returns null or a record (as a Map instance) + /// or a list of records (as a List instance). + Future query({required String what, Map? parameters}); + + /// Executes a SQL statement with not or a very short result: + /// insert: answers the primary key. + /// update: returns the count of changed records. + /// delete: ... + /// Returns the answer. If a blank is part of the answer that is an error + /// message. + Future store( + {required String what, required Map data}); +} diff --git a/lib/persistence/rest_persistence.dart b/lib/persistence/rest_persistence.dart new file mode 100644 index 0000000..5ec3d77 --- /dev/null +++ b/lib/persistence/rest_persistence.dart @@ -0,0 +1,108 @@ +import 'dart:convert' as convert; +import 'dart:io'; + +import 'package:dart_bones/dart_bones.dart'; +import 'package:http/http.dart' as http; +import 'persistence.dart'; +import '../setting/error_handler_exhibition.dart'; + +/// A persistence layer storing/fetching data with a REST interface. +class RestPersistence extends Persistence { + static final jsonHeader = { + 'Content-Type': 'application/json; charset=utf-8', + }; + final String version; + final BaseLogger logger; + int sqlTraceLimit = 80; + ErrorHandlerExhibition? errorHandler; + final String scheme; + final String host; + final int port; + final int sessionTimeout; + String uriAuthority = ''; + NetStatus netStatus = NetStatus.undefined; + + void setErrorHandler(ErrorHandlerExhibition? errorHandler) => + this.errorHandler = errorHandler; + + RestPersistence.fromConfig(BaseConfiguration configuration, BaseLogger logger, + {String section = 'productive'}) + : this( + version: configuration.asString('version', section: section) ?? '', + host: configuration.asString('host', section: section) ?? '', + port: configuration.asInt('port', section: section) ?? 3306, + scheme: configuration.asString('scheme', section: section) ?? '', + sessionTimeout: + configuration.asInt('timeout', section: section) ?? 30, + logger: logger); + + RestPersistence( + {required this.version, + this.scheme = 'https', + required this.host, + required this.port, + required this.sessionTimeout, + required this.logger}); + + /// Handles a HTTP request with a single HTTP connection. + /// [what]: defines the requested data. + Future runRequest(String what, + {String? body, Map? headers}) async { + var rc = ''; + final uri = + Uri(scheme: scheme, host: host, port: port, path: '/$what/$version'); + http.Response response; + logger.log('request: POST $uri', LEVEL_LOOP); + try { + response = await http.post(uri, body: body, headers: headers); + logger.log('status: ${response.statusCode}', LEVEL_LOOP); + if (response.statusCode != 200) { + logger.error('$uri: status: ${response.statusCode}'); + netStatus = NetStatus.fail; + } else { + netStatus = NetStatus.ok; + rc = response.body; + } + } on SocketException catch (exc) { + netStatus = NetStatus.noConnection; + errorHandler?.criticalError(exc.toString(), + caller: 'RestPersistence.runRequest'); + } on Exception catch(exc){ + logger.error('$exc'); + netStatus = NetStatus.noConnection; + } + return rc; + } + + @override + Future store( + {required String what, Map? data}) async { + var rc = 'OK'; + final data2 = data == null ? '{}' : convert.jsonEncode(data); + final answer = await runRequest(what, body: data2, headers: jsonHeader); + if (answer.isNotEmpty) { + rc = answer; + } + return rc; + } + + @override + Future query( + {required String what, Map? parameters}) async { + var rc; + final params2 = parameters == null ? '{}' : convert.jsonEncode(parameters); + final answer = await runRequest(what, body: params2, headers: jsonHeader); + if (answer.isNotEmpty && (answer.startsWith('{') || answer.startsWith('['))) { + rc = convert.jsonDecode(answer); + } else { + rc = answer; + } + if (logger.logLevel >= LEVEL_FINE) { + logger.log('answer: ${limitString(answer, sqlTraceLimit)}'); + } + return rc; + } +} +enum NetStatus { + undefined, noConnection, fail, ok +} \ No newline at end of file diff --git a/lib/setting/error_handler_exhibition.dart b/lib/setting/error_handler_exhibition.dart new file mode 100644 index 0000000..46da277 --- /dev/null +++ b/lib/setting/error_handler_exhibition.dart @@ -0,0 +1,7 @@ +/// Handles a critical error: "translate" a technical message into a user +/// readable message... +abstract class ErrorHandlerExhibition { + /// Returns false (for chaining) + bool criticalError(String errorMessage, + {required String caller, String? customParameter}); +} diff --git a/rest_client/README.md b/rest_client/README.md new file mode 100644 index 0000000..44ce34c --- /dev/null +++ b/rest_client/README.md @@ -0,0 +1,6 @@ +# Purpose +This project is for unit tests only. + +It is complicated to debug two processes in one IDE. +Therefore the rest server and the rest client is splitted +into two DART projects. diff --git a/rest_client/test/rest_client_test.dart b/rest_client/test/rest_client_test.dart new file mode 100644 index 0000000..6a79c3b --- /dev/null +++ b/rest_client/test/rest_client_test.dart @@ -0,0 +1,51 @@ +import 'package:test/test.dart'; +import 'package:path/path.dart' as path; +import 'package:dart_bones/dart_bones.dart'; + +import '../lib/persistence/rest_persistence.dart'; + +String init(BaseLogger logger) { + final fileSync = FileSync(); + var rc = path.join(fileSync.tempDirectory('unittest'), 'config.yaml'); + fileSync.toFile(rc, '''--- +productive: + version: 1 + host: localhost + port: 58031 + scheme: http + timeout: 30 +'''); + return rc; +} + +void main() async { + final logger = MemoryLogger(LEVEL_FINE); + final config = init(logger); + final configuration = Configuration.fromFile(config, logger); + final client = RestPersistence.fromConfig(configuration, logger); + group('query', () { + Map result; + Map parameters; + test('record', () async { + parameters = {'module': 'Persons', 'sql': 'byId', ':id': '13'}; + var result = await client.query(what: 'query', parameters: parameters); + expect(result, isNotNull); + expect(result, isNotEmpty); + expect(result['person_id'], 13); + }); + test('list', () async { + parameters = {'module': 'Persons', 'sql': 'list'}; + var result = await client.query(what: 'query', parameters: parameters); + expect(result, isNotNull); + expect(result is Iterable, isTrue); + expect(result.length, greaterThan(2)); + final record = result[0]; + expect(record['person_id'], 13); + }); + }); + group('store', () { + Map parameters; + parameters = { }; + //client.store(what: 'store', data: parameters); + }); +} diff --git a/rest_server/bin/rest_server_test.dart b/rest_server/bin/rest_server_test.dart new file mode 100644 index 0000000..12cd6d3 --- /dev/null +++ b/rest_server/bin/rest_server_test.dart @@ -0,0 +1,149 @@ +import 'dart:io'; +//import 'package:test/test.dart'; +import 'package:path/path.dart' as path; +import 'package:dart_bones/dart_bones.dart'; +import 'package:rest_server/services.dart'; + +void main() async { + var logger = MemoryLogger(4); + FileSync.initialize(logger); + final configFile = init(logger); + final configuration = Configuration.fromFile(configFile, logger); + //final fullTest = 'Never'.contains('x'); + await TestDbInitializer.initTest(configuration, logger); + // test('help', () async { + // expect(await run(['--help']), 0); + // }); + //test('test daemon', () async { + await run( + ['daemon', '--configuration=$configFile']); + //}); +} + +String init(BaseLogger logger) { + final fileSync = FileSync(); + var rc = path.join(fileSync.tempDirectory('unittest'), 'config.yaml'); + final sqlFile = '/tmp/unittest/sql/persons.sql.yaml'; + fileSync.toFile(rc, '''--- +# Example configuration file for the rest server. Created by rest_server. +service: + address: 0.0.0.0 + port: 58031 + dataDirectory: /tmp/unittest/data + sqlDirectory: ${path.dirname(sqlFile)} + threads: 1 + watchDogPause: 60 + # logFile: /var/log/local/exhibition.log +trace: + answerLength: 200 +db: + db: dbtest + user: dbtest + code: "TopSecret" + host: localhost + port: 3306 + timeout: 30 + traceDataLength: 200 +clientSessionTimeout: 900 +'''); + fileSync.ensureDirectory('/tmp/unittest/sql'); + fileSync.ensureDirectory('/tmp/unittest/data'); + final file = File(sqlFile); + file.writeAsStringSync('''--- +# SQL statements of the module "Persons": +module: Persons +list: + type: list + parameters: [] + sql: select * from persons; +byId: + type: record + parameters: [ ":id" ] + sql: "select * from persons where person_id=:id;" +update: + type: update + parameters: [":id", ":name", ":email", ":changedby"] + sql: "UPDATE persons SET + person_name=:name, person_email=:email, person_changed=NOW(), person_changedby=:changedby + WHERE person_id=:id;" +insert: + type: insert + parameters: [":name", ":email", ":createdby"] + sql: "INSERT INTO persons(person_name, person_email, person_changedby) + VALUES(:name, :displayname, :email, NOW(), :createdby);" +delete: + type: delete + parameters: [':id'] + sql: "DELETE FROM persons WHERE person_id=:id;" +'''); + return rc; +} + +class TestDbInitializer { + MySqlDb? db; + final BaseConfiguration configuration; + final BaseLogger logger; + List? tables; + TestDbInitializer(this.configuration, this.logger) { + db = MySqlDb.fromConfiguration(configuration, logger); + db!.timeout = 30; + } + + Future init() async { + if (!(db?.hasConnection ?? false)) { + await db!.connect(); + } + if (!(db?.hasConnection ?? false)) { + logger.error('cannot connect to database'); + } else { + tables = await db!.readAllAsLists('show tables;'); + } + } + + Future initPersons(MySqlDb db, BaseLogger logger) async { + String sql; + if (!await db.hasTable('persons')) { + sql = '''CREATE TABLE persons( + person_id int(10) unsigned NOT NULL AUTO_INCREMENT, + person_name varchar(200) NOT NULL, + person_email varchar(200) NOT NULL, + person_created timestamp NULL, + person_createdby varchar(16), + person_changed timestamp NULL, + PRIMARY KEY (person_id) +); +'''; + if (!await db.execute(sql)) { + logger.error('cannot create persons'); + } + } + final count = await db.readOneInt('select count(*) from persons;'); + if (count == null || count < 3) { + sql = '''insert into persons +(person_id, person_name, person_email, person_created, person_createdby) values +(11, 'Jones', 'jones@hamatoma.de', NOW(), 'daemon'), +(12, 'Miller', 'miller@hamatoma.de', NOW(), 'daemon'), +(13, 'Smith', 'smith@hamatoma.de', NOW(), 'daemon'); +'''; + if (!await db.execute(sql)) { + logger.error('cannot create apiaries records'); + } + } + } + + Future initDbData() async { + try { + await init(); + await initPersons(db!, logger); + } finally { + db?.close(); + } + } + + static Future initTest( + BaseConfiguration configuration, BaseLogger logger) async { + final initializer = TestDbInitializer(configuration, logger); + await initializer.initDbData(); + initializer.db?.close(); + } +} diff --git a/rest_server/lib/rest_server.dart b/rest_server/lib/rest_server.dart index 3a28566..0236752 100644 --- a/rest_server/lib/rest_server.dart +++ b/rest_server/lib/rest_server.dart @@ -3,11 +3,11 @@ import 'dart:convert' as convert; import 'dart:io'; import 'dart:isolate'; +import 'package:args/args.dart'; +import 'package:crypto/crypto.dart'; import 'package:dart_bones/dart_bones.dart'; import 'package:http/http.dart' as http; -import 'package:crypto/crypto.dart'; import 'package:path/path.dart' as path; -import 'package:args/args.dart'; import 'package:rest_server/sql_storage.dart'; const forbidden = 'forbidden'; @@ -117,7 +117,7 @@ class RestServer { 'db: ${configuration.asString('db', section: 'db')}', LEVEL_DETAIL); } - /// Initialisiert eine Isolate-Instanz. + /// Initializes an isolate instance. Future initIsolate() async { Completer completer = Completer(); var isolateToMainStream = ReceivePort(); @@ -132,8 +132,8 @@ class RestServer { return completer.future as dynamic; } - /// Startet die Isolate-Instanzen des Webservers. - void run() async { + /// Starts the isolate instances of the rest_server + Future run() async { final cpus = Platform.numberOfProcessors; var count = configuration.asInt('threads', section: 'service') ?? cpus; count = count <= 0 ? 1 : count; @@ -143,27 +143,28 @@ class RestServer { WorkerParameters(ix + 1, configuration, serviceName).asString()); } final mainToIsolateStream = await initIsolate(); - // der letzte Worker ist der Observer: + // the worker with id 1 is the observer: mainToIsolateStream .send(WorkerParameters(1, configuration, serviceName).asString()); } /// Starts the daemon process. /// [args] contains the program arguments service name and user, - /// e.g. ['pollsam', 'pollsam']. + /// e.g. ['exhibition', 'exhibition']. /// [results] contains the options. - static void daemon(List args, ArgResults results) { - final service = args.isEmpty ? 'pollsam' : args[0]; + static Future daemon(List args, ArgResults results) async { + final service = args.isEmpty ? 'exhibition' : args[0]; var filename = results['configuration']; - if (filename == '/etc/pollsam/pollsam.yaml' && service != 'pollsam') { - filename = '/etc/pollsam/$service.yaml'; + if (filename == '/etc/exhibition/exhibition.yaml' && + service != 'exhibition') { + filename = '/etc/exhibition/$service.yaml'; } if (!File(filename).existsSync()) { print('+++ missing $filename'); unittestLogger?.error('missing configuration: $filename'); } else { final server = RestServer.fromConfig(filename, serviceName: service); - server.run(); + await server.run(); } } @@ -177,11 +178,11 @@ service: sqlDirectory: /etc/rest_server/sql.d threads: 2 watchDogPause: 60 - # logFile: /var/log/local/pollsam.log + # logFile: /var/log/local/exhibition.log trace: answerLength: 200 db: - db: apppolladm + db: appexhibition user: jonny code: "Top Secret" host: localhost @@ -189,7 +190,7 @@ db: timeout: 30 traceDataLength: 200 clientSessionTimeout: 900 -# put this content to /etc/pollsam/pollsam.yaml +# put this content to /etc/exhibition/exhibition.yaml '''; return rc; } @@ -204,13 +205,13 @@ clientSessionTimeout: 900 if (!userInfo.isRoot) { logger.error('Be root!'); } else { - final appName = args.length >= 2 ? args[1] : 'pollsam'; + final appName = args.length >= 2 ? args[1] : 'exhibition'; var executable = path.absolute(args.length > 1 ? args[0] : Platform.executable); if (!executable.startsWith(Platform.pathSeparator)) { executable = processSync.executeToString('which', [executable]).trim(); } - final fnConfig = '/etc/pollsam/$appName.yaml'; + final fnConfig = '/etc/exhibition/$appName.yaml'; fileSync.ensureDirectory(path.dirname(fnConfig)); if (File(fnConfig).existsSync()) { logger.log('= $fnConfig already exists.'); @@ -232,7 +233,7 @@ clientSessionTimeout: 900 static void uninstall(List args) { final logger = BaseLogger(LEVEL_DETAIL); ProcessSync.initialize(logger); - final appName = args.isEmpty ? 'pollsam' : args[0]; + final appName = args.isEmpty ? 'exhibition' : args[0]; final service = OsService(logger); service.uninstallService(appName, user: appName, group: appName); } @@ -269,6 +270,21 @@ class ServiceWorker { sqlStorage = SqlStorage(logger); } + /// Puts a [list] into the [buffer]. + /// We cannot use jsonDecode(): data types like DateTime are not supported. + StringBuffer arrayToJson(Iterable list, StringBuffer buffer) { + buffer.write('['); + for (var item in list) { + if (item is Iterable) { + arrayToJson(item, buffer); + } else if (item is Map) { + rowToJson(item, buffer); + } + } + buffer.write(']'); + return buffer; + } + /// Checks whether a valid connection is available. If not a reconnection is /// done. Future checkConnection() async { @@ -288,6 +304,14 @@ class ServiceWorker { } } + void checkParameters(List names, Map parameters) { + for (var name in names) { + if (!parameters.containsKey(name)) { + throw FormatException('missing "$name" in parameters'); + } + } + } + /// Prüft die Session-ID. /// Liefert true, wenn OK, false sonst. Future checkSession(Map parameters) async { @@ -363,38 +387,6 @@ WHERE return rc; } - /// Bearbeitet die Anforderung 'hives': - /// Liefert Map mit Bienenstockinfo oder [forbidden], wenn Session-ID unbekannt. - Future queryBySql(Map parameters) async { - String rc; - final sql = '''SELECT * -FROM hives -WHERE hive_apiaryid=( - SELECT apiarist_apiaryid FROM apiarists - WHERE apiarist_id=( - SELECT connection_apiaristid FROM connections - WHERE connection_name=?) - ) -ORDER BY hive_name -; -'''; - final params = [parameters['sessionid']]; - final records = await db!.readAll(sql, params: params); - if (records == null) { - rc = wrongData; - } else { - final list = >[]; - records.forEach((record) => list.add({ - 'hiveid': record['hive_id'], - 'name': record['hive_name'], - 'lat': record['hive_latitude'], - 'long': record['hive_longitude'] - })); - rc = convert.jsonEncode({'list': list}); - } - return rc; - } - /// Bearbeitet die Anforderung 'sessionid': /// Prüfen der Lizenz ("Token"). Wenn erfolgreich, wird die Session /// in die DB eingetragen. @@ -634,12 +626,85 @@ WHERE apiarist_registername=?; } // URI: // if (list.length < 2) { - throw Exception('illegal request URI: ${list.join("/")}'); + throw FormatException('illegal request URI: ${list.join("/")}'); } what = list[0]; restVersion = list[1]; } + /// Handles the request of "query". + /// [parameters] contains the specification of the requested data, + /// e.g { 'module': 'persons', 'sql': 'record', ':id': '113' } + /// Returns the JSon formatted record list or an error message, e.g. + /// '{ data: { "person_id": 1, "person_name": "Joe" }}' + /// Note: Json data always starts with '{'. + Future queryBySql(Map parameters) async { + String rc; + checkParameters(['module', 'sql'], parameters); + final module = parameters['module']; + final sqlName = parameters['sql']; + final sqlStatement = sqlStorage.sqlStatement(module, sqlName); + checkParameters(sqlStatement.parameters, parameters); + final positionalParameters = []; + final sql = sqlStatement.sqlStatement(parameters, positionalParameters); + if (sqlStatement.type == SqlStatementType.record) { + final record = await db!.readOneAsMap(sql, params: positionalParameters); + rc = record == null ? 'NONE' : rowToJson(record, StringBuffer()).toString(); + } else { + final records = await db!.readAll(sql, params: positionalParameters); + rc = records == null ? 'NONE' : arrayToJson(records, StringBuffer()).toString(); + } + return rc; + } + + /// Puts a [row] into the [buffer]. + /// We cannot use jsonDecode(): data types like DateTime are not supported. + StringBuffer rowToJson(Map row, StringBuffer buffer) { + buffer.write('{'); + bool first = true; + for (var key in row.keys) { + if (first) { + first = false; + } else { + buffer.write(','); + } + buffer.write('"'); + buffer.write(key); + buffer.write('":'); + final value = row[key]; + if (value == null) { + buffer.write('null'); + } else if (value is String) { + buffer.write('"'); + String value2 = value; + for (var ix = 0; ix < value2.length; ix++) { + switch (value2[ix]) { + case '\\': + buffer.write('\\\\'); + break; + case '"': + buffer.write('\\"'); + break; + default: + buffer.write(value2[ix]); + break; + } + } + buffer.write('"'); + } else if (value is DateTime) { + buffer.write('"'); + buffer.write(dateAsString(value, withoutSeconds: true)); + buffer.write('"'); + } else if (value is bool) { + buffer.write(value ? 'true' : 'false'); + } else { + buffer.write('$value'); + } + } + buffer.write('}'); + return buffer; + } + /// Fordert per POST eine Aktion an. /// [what] legt die Aktion fest. /// [body] ist der Text, der mit der Anfrage mitgeliefert wird. @@ -675,33 +740,21 @@ WHERE apiarist_registername=?; this.restVersion = restVersion; } - /// Speichert [content] im Verzeichnis [directory] unter dem Namen [node]. - Future storeFile(String directory, String node, String content) async { - if (directory.isEmpty) { - logger.error('storeFile: missing data directory'); - } else { - _fileSync!.ensureDirectory(directory); - final fn = path.join(directory, node); - await File(fn).writeAsString(content); - } - } - - /// Executes a SQL statement without returning a value, e.g. an update. + /// Executes a SQL statement without returning nothing or a simple string, + /// e.g. an update. /// [parameters] is a map containing all named parameters of the SQL statement /// with its values. /// Returns 'OK' on success. Future storeBySql(Map parameters) async { String rc = 'OK'; - if (!testParam(['module', 'sql'], parameters, - 'storeBySql')) { + if (!testParam(['module', 'sql'], parameters, 'storeBySql')) { rc = wrongParameters; } else { - final sqlStatement = sqlStorage.sqlStatement(parameters['module'], - parameters['sql']); + final sqlStatement = + sqlStorage.sqlStatement(parameters['module'], parameters['sql']); final positionalParameters = []; final sql = sqlStatement.sqlStatement(parameters, positionalParameters); - var rc = 'OK'; - switch(sqlStatement.type){ + switch (sqlStatement.type) { case SqlStatementType.execute: case SqlStatementType.delete: await db!.execute(sql, params: positionalParameters); @@ -709,15 +762,17 @@ WHERE apiarist_registername=?; case SqlStatementType.insert: final id = await db!.insertOne(sql, params: positionalParameters); if (id <= 0) { - logger.error('insert failed: ${parameters['module']}.${parameters['sql']}:' - '\nsql: $sql\nparams: ${positionalParameters.join('|')}'); + logger.error( + 'insert failed: ${parameters['module']}.${parameters['sql']}:' + '\nsql: $sql\nparams: ${positionalParameters.join('|')}'); rc = 'ERROR'; } else { rc = 'id:$id'; } break; - case SqlStatementType.update: - final success = await db!.updateOne(sql, params: positionalParameters); + case SqlStatementType.update: + final success = + await db!.updateOne(sql, params: positionalParameters); rc = success ? 'OK' : 'FAILED'; break; default: @@ -728,8 +783,19 @@ WHERE apiarist_registername=?; return rc; } - /// Liefert eine Kopie von [source] mit Schlüsseln und Werten ohne Delimiter. - /// Beispiel: { "'a'": '"1"' } => { "a": '1' } + /// Speichert [content] im Verzeichnis [directory] unter dem Namen [node]. + Future storeFile(String directory, String node, String content) async { + if (directory.isEmpty) { + logger.error('storeFile: missing data directory'); + } else { + _fileSync!.ensureDirectory(directory); + final fn = path.join(directory, node); + await File(fn).writeAsString(content); + } + } + + /// Returns a copy of [source] with keys and values without delimiters. + /// Example: { "'a'": '"1"' } => { "a": '1' } Map stripDelimiters(Map source) { final rc = {}; for (var key in source.keys) { diff --git a/rest_server/lib/services.dart b/rest_server/lib/services.dart index da5a256..88dff1b 100644 --- a/rest_server/lib/services.dart +++ b/rest_server/lib/services.dart @@ -19,7 +19,8 @@ Future run(List args) async { if (results['help']) { try { try { - print('''Usage rest_server [mode] [options] + final serviceName = 'exhibition-sv'; + print('''Usage $serviceName [mode] [options] Starts a REST server answering the client database requests. : daemon @@ -33,13 +34,13 @@ Future run(List args) async {