import 'package:crypto/crypto.dart';
import 'package:path/path.dart' as path;
import 'package:args/args.dart';
+import 'package:rest_server/sql_storage.dart';
const forbidden = 'forbidden';
const wrongData = 'wrong data';
if (data is String && data.startsWith('WorkerParameter')) {
final params = WorkerParameters.fromString(data);
final worker =
- ServiceWorker(params.id, params.configuration, params.serviceName);
+ ServiceWorker(params.id, params.configuration, params.serviceName);
if (params.id == 1) {
worker.observe();
} else {
var clientSessionTimeout = 15 * 60;
String serviceName = 'exhibition';
RestServer(
- int port,
- this.logger, {
- String address = 'localhost',
- int sessionTimeout = 300,
- int answerDumpLength = 80,
- int dataDumpLength = 200,
- String db = 'appexhibition',
- String dbUser = 'exhibition',
- String dbCode = 'TopSecret',
- String dbHost = 'localhost',
- int dbPort = 3306,
- int watchDogPause = 60,
- int traceDataLength = 80,
- clientSessionTimeout = 15 * 60,
- String dataDirectory = 'data',
- }) {
+ int port,
+ this.logger, {
+ String address = 'localhost',
+ int sessionTimeout = 300,
+ int answerDumpLength = 80,
+ int dataDumpLength = 200,
+ String db = 'appexhibition',
+ String dbUser = 'exhibition',
+ String dbCode = 'TopSecret',
+ String dbHost = 'localhost',
+ int dbPort = 3306,
+ int watchDogPause = 60,
+ int traceDataLength = 80,
+ clientSessionTimeout = 15 * 60,
+ String dataDirectory = 'data',
+ String sqlDirectory = 'sql',
+ }) {
if (unittestLogger != null) {
logger = unittestLogger!;
}
'watchDogPause': watchDogPause,
'address': address,
'dataDirectory': dataDirectory,
+ 'sqlDirectory': sqlDirectory,
'threads': Platform.numberOfProcessors,
},
'trace': {
final logger2 = unittestLogger ?? MemoryLogger();
configuration = Configuration.fromFile(filename, logger2);
final logFile = configuration.asString('logFile',
- section: 'service',
- defaultValue: '/var/log/local/$serviceName.log') ??
+ section: 'service',
+ defaultValue: '/var/log/local/$serviceName.log') ??
'';
final level =
configuration.asInt('logLevel', section: 'service') ?? LEVEL_SUMMERY;
address: 0.0.0.0
port: 58021
dataDirectory: /var/cache/rest_server/data
+ sqlDirectory: /etc/rest_server/sql.d
threads: 2
watchDogPause: 60
# logFile: /var/log/local/pollsam.log
} else {
final appName = args.length >= 2 ? args[1] : 'pollsam';
var executable =
- path.absolute(args.length > 1 ? args[0] : Platform.executable);
+ path.absolute(args.length > 1 ? args[0] : Platform.executable);
if (!executable.startsWith(Platform.pathSeparator)) {
executable = processSync.executeToString('which', [executable]).trim();
}
MySqlDb? db;
String restVersion = '';
FileSync? _fileSync = FileSync();
-
+ SqlStorage sqlStorage = SqlStorage(globalLogger);
ServiceWorker(this.threadId, this.configuration, this.serviceName) {
final fnLog = '/var/log/local/$serviceName.$threadId.log';
logger = RestServer.unittestLogger ??
db = MySqlDb.fromConfiguration(configuration, logger);
clientSessionTimeout = configuration.asInt('clientSessionTimeout') ?? 30;
_fileSync = FileSync(logger);
+ sqlStorage = SqlStorage(logger);
}
/// Checks whether a valid connection is available. If not a reconnection is
/// Bearbeitet die Anforderung 'hives':
/// Liefert Map mit Bienenstockinfo oder [forbidden], wenn Session-ID unbekannt.
- Future<String> queryData(Map<String, dynamic> parameters) async {
+ Future<String> queryBySql(Map<String, dynamic> parameters) async {
String rc;
final sql = '''SELECT *
FROM hives
} else {
final list = <Map<String, dynamic>>[];
records.forEach((record) => list.add({
- 'hiveid': record['hive_id'],
- 'name': record['hive_name'],
- 'lat': record['hive_latitude'],
- 'long': record['hive_longitude']
- }));
+ 'hiveid': record['hive_id'],
+ 'name': record['hive_name'],
+ 'lat': record['hive_latitude'],
+ 'long': record['hive_longitude']
+ }));
rc = convert.jsonEncode({'list': list});
}
return rc;
/// Use only 31 bit (non negativ numbers on 32 bit clients):
final sessionId = int.parse(
- buildMd5Hash(
- token + DateTime.now().microsecondsSinceEpoch.toString())
- .substring(0, 8),
- radix: 16) &
- 0x7fffffff;
+ buildMd5Hash(
+ token + DateTime.now().microsecondsSinceEpoch.toString())
+ .substring(0, 8),
+ radix: 16) &
+ 0x7fffffff;
final params2 = [sessionId, apiaristId];
final id = await db!.insertOne(sql, params: params2);
if (id <= 0) {
await checkConnection();
}
try {
- if (withSession && what != 'sessionid' &&
+ if (withSession &&
+ what != 'sessionid' &&
what != 'register' &&
what != 'watchdog' &&
!await checkSession(parameters)) {
rc = await getSessionId(parameters);
break;
case 'store':
- rc = await storeData(parameters);
+ rc = await storeBySql(parameters);
break;
case 'query':
- rc = await queryData(parameters);
+ rc = await queryBySql(parameters);
break;
case 'register':
rc = await getToken(parameters);
/// @precondition: Must be called after the constructor!
Future initAsync() async {
await db!.connect();
+ final directory =
+ configuration.asString('sqlDirectory', section: 'service') ??
+ '/etc/rest_server/sql.d';
+ sqlStorage.read(directory);
}
/// Connects to a listening address/port and waits for requests.
Future observe() async {
final duration = Duration(
seconds:
- configuration.asInt('watchDogPause', section: 'service') ?? 60);
+ configuration.asInt('watchDogPause', section: 'service') ?? 60);
logger.log('watchdog pause: ${duration.inSeconds}', LEVEL_DETAIL);
var counter = 0;
while (true) {
final port = configuration.asInt('port', section: 'service') ?? 58011;
var rc = '';
final uri =
- Uri(scheme: 'http', host: 'localhost', port: port, path: '/$what/1.0');
+ Uri(scheme: 'http', host: 'localhost', port: port, path: '/$what/1.0');
http.Response response;
// logger.log('request: POST $uri', LEVEL_LOOP);
try {
}
}
- /// Bearbeitet die Anforderung 'sample': Speichern einer Probe.
- Future<String> storeData(Map<String, dynamic> parameters) async {
- String rc;
- if (!testParam(
- ['module', 'sql', 'sampleid', 'sessionid'], parameters, 'storeData')) {
+ /// Executes a SQL statement without returning a value, e.g. an update.
+ /// [parameters] is a map containing all named parameters of the SQL statement
+ /// with its values.
+ /// Returns 'OK' on success.
+ Future<String> storeBySql(Map<String, dynamic> parameters) async {
+ String rc = 'OK';
+ if (!testParam(['module', 'sql'], parameters,
+ 'storeBySql')) {
rc = wrongParameters;
} else {
-
- final sql = '''INSERT
- INTO samples
- (sample_code, sample_timestamp, sample_hiveid, sample_uuid, sample_connection,
- sample_apiaristid, sample_raw, sample_encoding, created, createdby)
- VALUES (?, ?, ?, ?, ?,
- (SELECT MAX(connection_apiaristid) FROM connections WHERE connection_name=?),
- ?, ?, NOW(), 'POLLSAM');
- ''';
- final connection = parameters['sessionid'];
- final params = [
- parameters['sampleid'],
- parameters['time'],
- parameters['hiveid'],
- parameters['uuid'],
- connection,
- connection,
- parameters.containsKey('raw') ? parameters['raw'] : null,
- parameters.containsKey('encoding') ? parameters['encoding'] : null,
- ];
- final id = await db!.insertOne(sql, params: params);
- if (id <= 0) {
- logger.error('insert into samples failed: ${parameters['uuid']}');
- } else {
- logger.log('sample: $id', LEVEL_DETAIL);
+ final sqlStatement = sqlStorage.sqlStatement(parameters['module'],
+ parameters['sql']);
+ final positionalParameters = <String>[];
+ final sql = sqlStatement.sqlStatement(parameters, positionalParameters);
+ var rc = 'OK';
+ switch(sqlStatement.type){
+ case SqlStatementType.execute:
+ case SqlStatementType.delete:
+ await db!.execute(sql, params: positionalParameters);
+ break;
+ case SqlStatementType.insert:
+ final id = await db!.insertOne(sql, params: positionalParameters);
+ if (id <= 0) {
+ logger.error('insert failed: ${parameters['module']}.${parameters['sql']}:'
+ '\nsql: $sql\nparams: ${positionalParameters.join('|')}');
+ rc = 'ERROR';
+ } else {
+ rc = 'id:$id';
+ }
+ break;
+ case SqlStatementType.update:
+ final success = await db!.updateOne(sql, params: positionalParameters);
+ rc = success ? 'OK' : 'FAILED';
+ break;
+ default:
+ logger.error('unexpected type ${sqlStatement.type} in storeBySql');
+ rc = 'ERROR';
}
- final hive = parameters['hiveid'].toString();
- final data = <String, String>{
- 'uuid': parameters['uuid'],
- 'time': parameters['time'],
- 'sampleid': parameters['sampleid'],
- 'hiveid': hive,
- };
- final content = convert.jsonEncode(data);
- await storeFile(
- configuration.asString(
- 'dataDirectory',
- section: 'service',
- ) ??
- '',
- parameters['uuid'] as String,
- content);
- rc = 'OK';
}
return rc;
}
factory WorkerParameters.fromString(String data) {
final parts = data.split('\t');
const baseService = 2;
- const baseTrace = baseService + 4;
+ const baseTrace = baseService + 5;
const baseDb = baseTrace + 1;
const baseRest = baseDb + 7;
final configuration = BaseConfiguration({
'address': parts[baseService],
'port': parts[baseService + 1],
'watchDogPause': parts[baseService + 2],
- 'dataDirectory': parts[baseService + 3]
+ 'dataDirectory': parts[baseService + 3],
+ 'sqlDirectory': parts[baseService + 4]
},
'trace': {
'answerLength': int.parse(parts[baseTrace]),
(configuration.asInt('port', section: section) ?? 58011).toString(),
(configuration.asInt('watchDogPause', section: section) ?? 60).toString(),
configuration.asString('dataDirectory', section: section) ?? 'data',
+ configuration.asString('sqlDirectory', section: section) ?? 'sql',
configuration
.asInt('answerLength', section: section2, defaultValue: 200)
.toString(),
return rc;
}
}
-enum SqlStatementType {
- insert, list, query, update, delete
-}
-class SqlException extends FormatException{
- final String data;
- SqlException(this.data);
- @override
- String toString() => 'SqlException: $data';
-}
-class SqlStatement {
- final String name;
- final List<String> parameters;
- final String sql;
- final SqlStatementType type;
- final SqlModule parent;
- SqlStatement(this.name, this.parameters, this.sql, this.type, this.parent);
- /// Returns a SQL statement and a parameter list (positional parameters).
- /// [map]: the current parameters as named parameters: <name>: <value>
- /// [parameters]: OUT the positional parameters
- String sqlStatement(Map map, List<String> parameters){
- for (var parameter in parameters){
- if (! map.containsKey(parameter)) {
- throw SqlException('${toString()}: missing parameter "$parameter"');
- } else {
- parameters.add(map[parameter]);
- }
- }
- return sql;
- }
- @override
- String toString(){
- String rc = '${parent.name}.$name';
- return rc;
- }
-}
-class SqlModule{
- final String name;
- final Map<String, SqlStatement> sqlStatements = {};
- final SqlStorage parent;
- SqlModule(this.name, this.parent);
- /// Adds a statement to the map.
- void add(SqlStatement statement) {
- if (sqlStatements.containsKey(statement.name)){
- throw SqlException('module $name contains already a statement "${statement.name}"');
- }
- sqlStatements[statement.name] = statement;
- }
-}
-class SqlStorage{
- final BaseLogger logger;
- Map<String, SqlModule> modules = {};
- SqlStorage(this.logger);
- void readModule(Map map, String filename){
- String moduleName = '<unknown>';
- if (map.containsKey('module')){
- moduleName = map['module'];
- } else {
- logger.error('$filename: missing "module"');
- }
- if (! modules.containsKey(moduleName)){
- modules[moduleName] = SqlModule(moduleName, this);
- }
- final module = modules[moduleName];
- for (var name in map.keys){
- switch(name){
- case 'module':
- // already done.
- break;
- default:
- final map2 = map[name];
- if (map2 is! Map){
- logger.error('$filename: "$name" is not a map');
- } else if (! map2.containsKey('type')){
- logger.error('$filename: "$name": missing type');
- } else if (! map2.containsKey('parameters')){
- logger.error('$filename: "$name": missing parameters');
- } else if (! map2.containsKey('sql')){
- logger.error('$filename: "$name": missing sql');
- }
- final type = map2['type'];
- final parameters = map2['parameters'];
- final sql = map2['sql'];
- if (type is! String){
- logger.error('$filename: "$name": type is not a string');
- } else if (parameters is! Iterable){
- logger.error('$filename: "$name": type is not an array');
- } else if (sql is! String){
- logger.error('$filename: "$name": type is not a string');
- } else {
- final parameters2 = <String>[];
- int no = -1;
- for (var item in parameters){
- no++;
- if (item is! String){
- logger.error('$filename: "$name": parameter[$no] is not a string');
- } else {
-
- }
- }
- var type2 = stringToEnum(type, SqlStatementType.values);
- if (type2 == null){
- logger.error('$filename: "$name": unknown type: $type. Using "query"');
- type2 = SqlStatementType.query;
- }
- modules[moduleName]!.add(SqlStatement(name, parameters2, sql, type2, module!));
- }
- }
- }
- }
-}
--- /dev/null
+import 'dart:io';
+
+import 'package:dart_bones/dart_bones.dart';
+import 'package:path/path.dart';
+import 'package:yaml/yaml.dart';
+
+class SqlException extends FormatException {
+ final String data;
+ SqlException(this.data);
+ @override
+ String toString() => 'SqlException: $data';
+}
+
+/// Stores all sql statements of a module.
+class SqlModule {
+ final String name;
+ final Map<String, SqlStatement> sqlStatements = {};
+ final SqlStorage sqlStorage;
+ SqlModule(this.name, this.sqlStorage);
+
+ /// Adds a statement to the map.
+ void add(SqlStatement statement) {
+ if (sqlStatements.containsKey(statement.name)) {
+ throw SqlException(
+ 'module $name contains already a statement "${statement.name}"');
+ }
+ sqlStatements[statement.name] = statement;
+ }
+
+ /// Returns the [SqlStatement] from the map given by [name].
+ SqlStatement sqlByName(String name) {
+ if (!sqlStatements.containsKey(name)) {
+ throw SqlException('missing statement "$name" in module "${this.name}"');
+ }
+ final rc = sqlStatements[name];
+ return rc!;
+ }
+}
+
+/// Stores a SQL statement and things that allow named parameters to be used,
+/// although only positional parameters can be processed (restriction of the
+/// mysql package).
+/// Note: Named parameters: :id or :date
+/// Positional parameters: ? (question mark only)
+class SqlStatement {
+ final String name;
+ final List<String> parameters;
+
+ /// The SQL statement with named parameters.
+ final String sql;
+
+ /// The Sql statement with positional parameters.
+ String sqlPrepared = '';
+
+ /// The named parameters in the order of [sql] (multiple occurrences are possible).
+ final List<String> orderOfParameters = [];
+ final SqlStatementType type;
+ final SqlModule sqlModule;
+ SqlStatement(
+ this.name, this.parameters, this.sql, this.type, this.sqlModule) {
+ var parameters2 = parameters.toList();
+ parameters2.sort();
+ // revers sorting: if a member is a prefix of another member then it is
+ // positioned behind: example [':id1', ':id' ]
+ // The longer member is found first.
+ parameters2 = parameters2.reversed.toList();
+ RegExp regExp = RegExp('(' + parameters2.join('|') + r')\b');
+ for (var match in regExp.allMatches(sql)) {
+ orderOfParameters.add(match.group(0)!);
+ }
+ sqlPrepared = sql.replaceAll(regExp, '?');
+ }
+
+ /// Returns a SQL statement and a parameter list (positional parameters).
+ /// [map]: the current parameters as named parameters: <name>: <value>
+ /// [parameters]: OUT the positional parameters
+ String sqlStatement(Map map, List<String> parameters) {
+ for (var parameter in orderOfParameters) {
+ if (!map.containsKey(parameter)) {
+ throw SqlException('${toString()}: missing parameter "$parameter"');
+ } else {
+ parameters.add(map[parameter]);
+ }
+ }
+ return sqlPrepared;
+ }
+
+ @override
+ String toString() {
+ String rc = '${sqlModule.name}.$name';
+ return rc;
+ }
+}
+
+enum SqlStatementType { delete, execute, insert, list, record, update }
+
+/// Stores all sql modules.
+class SqlStorage {
+ final BaseLogger logger;
+ Map<String, SqlModule> modules = {};
+ SqlStorage(this.logger);
+
+ /// Reads multiple module data from a directory.
+ /// Only files ending with '.yaml' and containing '.sql.' are respected.
+ void read(String path) {
+ for (var entry in Directory(path).listSync()) {
+ final full = entry.path;
+ final node = basename(full);
+ if (node.endsWith('.yaml') && node.contains('.sql.') && entry is File) {
+ final contents = entry.readAsStringSync();
+ final map = loadYaml(contents);
+ readModule(map, node);
+ }
+ }
+ }
+
+ void readModule(Map map, String filename) {
+ String moduleName = '<unknown>';
+ if (map.containsKey('module')) {
+ moduleName = map['module'];
+ } else {
+ logger.error('$filename: missing "module"');
+ }
+ if (!modules.containsKey(moduleName)) {
+ modules[moduleName] = SqlModule(moduleName, this);
+ }
+ final module = modules[moduleName];
+ for (var name in map.keys) {
+ switch (name) {
+ case 'module':
+ // already done.
+ break;
+ default:
+ final map2 = map[name];
+ if (map2 is! Map) {
+ logger.error('$filename: "$name" is not a map');
+ } else if (!map2.containsKey('type')) {
+ logger.error('$filename: "$name": missing type');
+ } else if (!map2.containsKey('parameters')) {
+ logger.error('$filename: "$name": missing parameters');
+ } else if (!map2.containsKey('sql')) {
+ logger.error('$filename: "$name": missing sql');
+ } else {
+ final type = map2['type'];
+ final parameters = map2['parameters'];
+ final sql = map2['sql'];
+ if (type is! String) {
+ logger.error('$filename: "$name": type is not a string');
+ } else if (parameters is! Iterable) {
+ logger.error('$filename: "$name": type is not an array');
+ } else if (sql is! String) {
+ logger.error('$filename: "$name": type is not a string');
+ } else {
+ final parameters2 = <String>[];
+ int no = -1;
+ for (var item in parameters) {
+ no++;
+ if (item is! String) {
+ logger.error(
+ '$filename: "$name": parameter[$no] is not a string');
+ } else {
+ parameters2.add(item);
+ }
+ }
+ var type2 = stringToEnum(type, SqlStatementType.values);
+ if (type2 == null) {
+ logger.error(
+ '$filename: "$name": unknown type: $type. Using "query"');
+ type2 = SqlStatementType.record;
+ }
+ modules[moduleName]!
+ .add(SqlStatement(name, parameters2, sql, type2, module!));
+ }
+ }
+ }
+ }
+ }
+
+ /// Returns the [SqlStatement] from the map given by [name].
+ SqlStatement sqlStatement(String module, String name) {
+ if (!modules.containsKey(module)) {
+ throw SqlException('unknown module: "$module"');
+ }
+ final rc = modules[module]!.sqlByName(name);
+ return rc;
+ }
+}
--- /dev/null
+import 'package:test/test.dart';
+import 'package:rest_server/sql_storage.dart';
+import 'package:dart_bones/dart_bones.dart';
+
+void main() {
+ final logger = MemoryLogger(LEVEL_DETAIL);
+
+ group('SqlStorage', () {
+ test('read()', () {
+ final sqlStorage = SqlStorage(logger);
+ //print('current directory: ${Directory.current.path}');
+ sqlStorage.read('data/sql');
+ expect(sqlStorage.sqlStatement('Users', 'list'), isNotNull);
+ expect(sqlStorage.sqlStatement('Users', 'byId'), isNotNull);
+ SqlStatement update;
+ expect(update = sqlStorage.sqlStatement('Users', 'update'), isNotNull);
+ expect(update.orderOfParameters,
+ ':name,:displayname,:email,:changedby,:id'.split(','));
+ expect(
+ update.sqlPrepared,
+ 'UPDATE loginusers SET user_name=?, user_displayname=?, '
+ 'user_email=?, user_changed=NOW(), user_changedby=? WHERE user_id=?;');
+ expect(update.type, SqlStatementType.update);
+ expect(sqlStorage.sqlStatement('Users', 'insert'), isNotNull);
+ });
+ });
+ group('SqlModule', () {
+ final sqlStorage = SqlStorage(logger);
+ test('basic', () {
+ final sqlModule = SqlModule('users', sqlStorage);
+ sqlModule.add(SqlStatement(
+ 'record',
+ [':id'],
+ 'select * from users where id=:id',
+ SqlStatementType.record,
+ sqlModule));
+ sqlModule.add(SqlStatement(
+ 'insert',
+ [':name', ':role'],
+ 'insert into users (name, role) values (:name, :role);',
+ SqlStatementType.insert,
+ sqlModule));
+ SqlStatement statement;
+ expect(statement = sqlModule.sqlByName('record'), isNotNull);
+ expect(statement.name, 'record');
+ expect(statement.sqlPrepared, 'select * from users where id=?');
+ expect(statement = sqlModule.sqlByName('insert'), isNotNull);
+ expect(statement.name, 'insert');
+ expect(statement.sqlPrepared,
+ 'insert into users (name, role) values (?, ?);');
+ expect(statement.orderOfParameters, [':name', ':role']);
+ });
+ });
+
+ group('SqlStatement', () {
+ final sqlStorage = SqlStorage(logger);
+ final sqlModule = SqlModule('standard', sqlStorage);
+ test('one parameter', () {
+ final sqlStatement = SqlStatement('list', [':id'],
+ 'select * from x where id=:id;', SqlStatementType.record, sqlModule);
+ expect(sqlStatement.parameters, [':id']);
+ expect(sqlStatement.orderOfParameters, [':id']);
+ expect(sqlStatement.sqlPrepared, 'select * from x where id=?;');
+ expect(sqlStatement.type, SqlStatementType.record);
+ final parameters = <String>[];
+ final map = <String, String?>{':id': '223'};
+ expect(sqlStatement.sqlStatement(map, parameters),
+ 'select * from x where id=?;');
+ expect(parameters, ['223']);
+ });
+ test('many parameters multiple occurrences', () {
+ final sql = '''select
+from users uu
+ left join role rr ON rr.id=uu.role
+where
+ u.name like :name AND uu.created >= :from
+ AND rr.name like :nameRole AND rr.created >= :from
+;
+''';
+ final expectedOrder = ':name,:from,:nameRole,:from'.split(',');
+ final sqlExpected = '''select
+from users uu
+ left join role rr ON rr.id=uu.role
+where
+ u.name like ? AND uu.created >= ?
+ AND rr.name like ? AND rr.created >= ?
+;
+''';
+ final sqlStatement = SqlStatement('list', [':name', ':nameRole', ':from'],
+ sql, SqlStatementType.list, sqlModule);
+ expect(sqlStatement.parameters, [':name', ':nameRole', ':from']);
+ expect(sqlStatement.sqlPrepared, sqlExpected);
+ expect(sqlStatement.orderOfParameters, expectedOrder);
+ expect(sqlStatement.type, SqlStatementType.list);
+ final parameters = <String>[];
+ final map = <String, String?>{
+ ':name': 'a%',
+ ':nameRole': 'b%',
+ ':from': '2021-06-01'
+ };
+ expect(sqlStatement.sqlStatement(map, parameters), sqlExpected);
+ expect(parameters, ['a%', '2021-06-01', 'b%', '2021-06-01']);
+ });
+ });
+}
projects / exhibition.git / commitdiff