From 1ed0627853bf04640694822572c76253270c0165 Mon Sep 17 00:00:00 2001 From: Martin Schett Date: Sat, 9 Jan 2021 21:47:03 +0100 Subject: [PATCH] Updated recovery logic to fall back on older version if no image-hash is valid --- iotclient/iot_client.py | 24 ++++++++- .../services/StorageServiceInterface.py | 9 ++++ middleware/app_be/services/dropboxservice.py | 14 ++++++ middleware/app_be/services/minioservice.py | 21 ++++++++ middleware/app_be/services/mongodbservice.py | 7 +++ middleware/app_be/urls.py | 1 + middleware/app_be/views/rest_api.py | 50 +++++++++++++------ 7 files changed, 111 insertions(+), 15 deletions(-) diff --git a/iotclient/iot_client.py b/iotclient/iot_client.py index cb3788b..9a6d5c4 100644 --- a/iotclient/iot_client.py +++ b/iotclient/iot_client.py @@ -73,11 +73,13 @@ def update_image(identifier, image_path, metadata_payload): return True -def get_image(identifier): +def get_image(identifier, version=None): print("Getting image with identifier " + identifier) baseurl = "http://127.0.0.1:8000" get_url = "/image/get/" + identifier + if version is not None: + get_url = get_url + "/version/"+str(version) try: response = requests.get(baseurl + get_url) @@ -248,6 +250,7 @@ while (command.lower() not in ["exit", "quit", "end"]): print("trigger - next image in line is sent to backend") print("update - next image in line is updated in backend") print("fetch - gets the next image from database if exists") + print("fetchversion - gets the next image with specified version from database if exists") print("fetchall - get all images") print("delete - delets the next image from db if exists") print("deleteall - delets all images from the service") @@ -354,6 +357,25 @@ while (command.lower() not in ["exit", "quit", "end"]): get_image(filename[:-4]) print_cursor() + if command.lower() == "fetchversion": + if (len(attributes) < 1): + print("Error: No version supplied") + continue + if not attributes[0].isnumeric(): + print("Version is no number") + continue + version = int(attributes[0]) + if metadata is None: + print("No metadata loaded") + continue + if image_folder is None: + print("No image folder selected") + continue + meta_payload = metadata[index] + filename = meta_payload['filename'] + get_image(filename[:-4], version) + print_cursor() + if command.lower() == "fetchall": if metadata is None: print("No metadata loaded") diff --git a/middleware/app_be/services/StorageServiceInterface.py b/middleware/app_be/services/StorageServiceInterface.py index 1442d3d..a0f3688 100644 --- a/middleware/app_be/services/StorageServiceInterface.py +++ b/middleware/app_be/services/StorageServiceInterface.py @@ -2,6 +2,15 @@ class StorageServiceInterface: name = "StorageServiceInterface" + @staticmethod + def get_last_modified(filename: str): + """Get last modified time of file + + :param filename: filename + :return: Date of last modification or None + """ + pass + @staticmethod def check() -> bool: pass diff --git a/middleware/app_be/services/dropboxservice.py b/middleware/app_be/services/dropboxservice.py index 6ae79ae..acbf651 100644 --- a/middleware/app_be/services/dropboxservice.py +++ b/middleware/app_be/services/dropboxservice.py @@ -9,6 +9,20 @@ from app_be.services.StorageServiceInterface import StorageServiceInterface class DropboxService(StorageServiceInterface): name = "Dropbox" + @staticmethod + def get_last_modified(filename: str): + """Get last modified time of file + + :param filename: filename + :return: Date of last modification or None + """ + try: + dbx = dropbox.Dropbox(settings.DROPBOX_OAUTH2_ACCESS_TOKEN) + return dbx.files_get_metadata(settings.DROPBOX_IMAGE_FOLDER + filename).server_modified + except: + return None + return None + @staticmethod def check() -> bool: try: diff --git a/middleware/app_be/services/minioservice.py b/middleware/app_be/services/minioservice.py index 572b2f3..b7afc20 100644 --- a/middleware/app_be/services/minioservice.py +++ b/middleware/app_be/services/minioservice.py @@ -1,5 +1,6 @@ import base64 import hashlib +from datetime import datetime from typing import Union import xml.etree.ElementTree as ET @@ -12,6 +13,26 @@ from app_be.services.StorageServiceInterface import StorageServiceInterface class MinioService(StorageServiceInterface): name = "MinIO" + @staticmethod + def get_last_modified(filename: str): + """Get last modified time of file + + :param filename: filename + :return: Date of last modification or None + """ + try: + r = requests.get(settings.AWS_HOST + "?prefix="+filename) + root = ET.fromstring(r.content) + for child in root.iter('{http://s3.amazonaws.com/doc/2006-03-01/}Contents'): + if child.find('{http://s3.amazonaws.com/doc/2006-03-01/}Key').text == filename: + date_time_str = child.find('{http://s3.amazonaws.com/doc/2006-03-01/}LastModified').text + print(date_time_str) + date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ') + return date_time_obj + except: + return None + return None + @staticmethod def check() -> bool: print("Checking MinIO availability") diff --git a/middleware/app_be/services/mongodbservice.py b/middleware/app_be/services/mongodbservice.py index 5d951e2..a473f2c 100644 --- a/middleware/app_be/services/mongodbservice.py +++ b/middleware/app_be/services/mongodbservice.py @@ -108,6 +108,13 @@ class MongoDBService: print("Could not delete Metadata") return resp + @staticmethod + def replaceHash(identifier, decoded_image): + instance = MongoManager.getInstance() + db = instance.AIC + col = db.metadata + col.update_one({"identifier": identifier}, {"$set": {"sha512": create_sha512(decoded_image)}}) + @staticmethod def deleteAll(): instance = MongoManager.getInstance() diff --git a/middleware/app_be/urls.py b/middleware/app_be/urls.py index 13fa166..df54cfa 100644 --- a/middleware/app_be/urls.py +++ b/middleware/app_be/urls.py @@ -25,6 +25,7 @@ urlpatterns = [ url(r'^test/', TestApiClass.test_api), url(r'^image/get/all$', ImageEndpoint.image_api_get_all), url(r'^image/get/(?P[\w-]+)$', ImageEndpoint.image_api_get_single), + url(r'^image/get/(?P[\w-]+)/version/(?P[\w-]+)$', ImageEndpoint.image_api_get_single_version), url(r'^image/delete/all$', ImageEndpoint.image_api_delete_all), url(r'^image/delete/(?P[\w-]+)$', ImageEndpoint.image_api_delete), url(r'^image/post$', ImageEndpoint.image_api_post), diff --git a/middleware/app_be/views/rest_api.py b/middleware/app_be/views/rest_api.py index ec91197..3162aa2 100644 --- a/middleware/app_be/views/rest_api.py +++ b/middleware/app_be/views/rest_api.py @@ -1,5 +1,6 @@ import json import logging +from datetime import datetime from app_be.services.dropboxservice import DropboxService from app_be.services.hashservice import create_sha512 @@ -36,10 +37,20 @@ class ImageEndpoint: return JsonResponse({'Result': 'success1'}, safe=False) + @staticmethod + @api_view(['GET']) + def image_api_get_single_version(request, identifier, version): + logger.debug('Image GET single with version call: {}'.format(request)) + return ImageEndpoint.get_image(identifier+'_'+version) + @staticmethod @api_view(['GET']) def image_api_get_single(request, identifier): logger.debug('Image GET single call: {}'.format(request)) + return ImageEndpoint.get_image(identifier) + + @staticmethod + def get_image(identifier): # get metadata from MongoDB metadata = MongoDBService.getSingle(identifier) @@ -58,7 +69,6 @@ class ImageEndpoint: # check image existence and correctness for each service + retrieve valid image if possible for service in ImageEndpoint.storageServiceList: logger.debug('Checking recovery for service ' + service.name) - print('Checking recovery for service ' + service.name) service_image_bytes = service.read_file(metadata['filename']) if service_image_bytes is not None: recovery_has_image[service.name] = True @@ -73,27 +83,39 @@ class ImageEndpoint: recovery_has_image[service.name] = False recovery_hash_matches[service.name] = False - # TODO: after talking with tobias about updating => replace with older version if hash is wrong - # check if any service has the image saved if not ImageEndpoint.combine_boolean_dict_or(recovery_has_image): logger.debug('None of the storage services has the requested image saved') - MongoDBService.deleteSingle(identifier) - return JsonResponse({'Result': 'Error - image is not available on any storage service and was deleted', + return JsonResponse({'Result': 'Error - image is not available on any storage service', 'id': identifier}, status=404, safe=False) # check if any service has a valid version of the image saved if not ImageEndpoint.combine_boolean_dict_or(recovery_hash_matches) and valid_image_bytes is None: - logger.debug('None of the storage services has a valid image saved') - MongoDBService.deleteSingle(identifier) - return JsonResponse({'Result': 'Error - image is not available on any storage service and was deleted', - 'id': identifier}, status=404, safe=False) + logger.debug('None of the storage services has a valid image saved - assume oldest image is valid') + service_with_oldest = None + oldest_last_modified = datetime.now() + for service in ImageEndpoint.storageServiceList: + modified = service.get_last_modified(metadata['filename']) + if modified is not None and modified < oldest_last_modified: + oldest_last_modified = modified + service_with_oldest = service - # at this point we know at least one valid image is available - for service in ImageEndpoint.storageServiceList: - if not recovery_has_image[service.name] or not recovery_hash_matches[service.name]: - if not service.create_file(metadata['filename'], valid_image_bytes): - logger.error('Error duplicating file in service ' + service.name) + if service_with_oldest is None: # failsave if no image is available + return JsonResponse({'Result': 'Error - image is not available on any storage service', + 'id': identifier}, status=404, safe=False) + # replace image on every service with oldest available + valid_image_bytes = service_with_oldest.read_file(metadata['filename']) + MongoDBService.replaceHash(identifier, valid_image_bytes) + for service in ImageEndpoint.storageServiceList: + if service != service_with_oldest: + if not service.create_file(metadata['filename'], valid_image_bytes): + logger.error('Error recovering file in service ' + service.name) + else: + # at this point we know at least one valid image is available + for service in ImageEndpoint.storageServiceList: + if not recovery_has_image[service.name] or not recovery_hash_matches[service.name]: + if not service.create_file(metadata['filename'], valid_image_bytes): + logger.error('Error duplicating file in service ' + service.name) payload = { 'id': identifier,