Updated recovery logic to fall back on older version if no image-hash is valid

iotclient/iot_client.py

@@ -73,11 +73,13 @@ def update_image(identifier, image_path, metadata_payload):
     return True
 
 
-def get_image(identifier):
+def get_image(identifier, version=None):
     print("Getting image with identifier " + identifier)
 
     baseurl = "http://127.0.0.1:8000"
     get_url = "/image/get/" + identifier
+    if version is not None:
+        get_url = get_url + "/version/"+str(version)
 
     try:
         response = requests.get(baseurl + get_url)
@@ -248,6 +250,7 @@ while (command.lower() not in ["exit", "quit", "end"]):
         print("trigger - next image in line is sent to backend")
         print("update - next image in line is updated in backend")
         print("fetch - gets the next image from database if exists")
+        print("fetchversion - gets the next image with specified version from database if exists")
         print("fetchall - get all images")
         print("delete - delets the next image from db if exists")
         print("deleteall - delets all images from the service")
@@ -354,6 +357,25 @@ while (command.lower() not in ["exit", "quit", "end"]):
         get_image(filename[:-4])
         print_cursor()
 
+    if command.lower() == "fetchversion":
+        if (len(attributes) < 1):
+            print("Error: No version supplied")
+            continue
+        if not attributes[0].isnumeric():
+            print("Version is no number")
+            continue
+        version = int(attributes[0])
+        if metadata is None:
+            print("No metadata loaded")
+            continue
+        if image_folder is None:
+            print("No image folder selected")
+            continue
+        meta_payload = metadata[index]
+        filename = meta_payload['filename']
+        get_image(filename[:-4], version)
+        print_cursor()
+
     if command.lower() == "fetchall":
         if metadata is None:
             print("No metadata loaded")

middleware/app_be/services/StorageServiceInterface.py

@@ -2,6 +2,15 @@ class StorageServiceInterface:
 
     name = "StorageServiceInterface"
 
+    @staticmethod
+    def get_last_modified(filename: str):
+        """Get last modified time of file
+
+        :param filename: filename
+        :return: Date of last modification or None
+        """
+        pass
+
     @staticmethod
     def check() -> bool:
         pass

middleware/app_be/services/dropboxservice.py

@@ -9,6 +9,20 @@ from app_be.services.StorageServiceInterface import StorageServiceInterface
 class DropboxService(StorageServiceInterface):
     name = "Dropbox"
 
+    @staticmethod
+    def get_last_modified(filename: str):
+        """Get last modified time of file
+
+        :param filename: filename
+        :return: Date of last modification or None
+        """
+        try:
+            dbx = dropbox.Dropbox(settings.DROPBOX_OAUTH2_ACCESS_TOKEN)
+            return dbx.files_get_metadata(settings.DROPBOX_IMAGE_FOLDER + filename).server_modified
+        except:
+            return None
+        return None
+
     @staticmethod
     def check() -> bool:
         try:

middleware/app_be/services/minioservice.py

@@ -1,5 +1,6 @@
 import base64
 import hashlib
+from datetime import datetime
 from typing import Union
 
 import xml.etree.ElementTree as ET
@@ -12,6 +13,26 @@ from app_be.services.StorageServiceInterface import StorageServiceInterface
 class MinioService(StorageServiceInterface):
     name = "MinIO"
 
+    @staticmethod
+    def get_last_modified(filename: str):
+        """Get last modified time of file
+
+        :param filename: filename
+        :return: Date of last modification or None
+        """
+        try:
+            r = requests.get(settings.AWS_HOST + "?prefix="+filename)
+            root = ET.fromstring(r.content)
+            for child in root.iter('{http://s3.amazonaws.com/doc/2006-03-01/}Contents'):
+                if child.find('{http://s3.amazonaws.com/doc/2006-03-01/}Key').text == filename:
+                    date_time_str = child.find('{http://s3.amazonaws.com/doc/2006-03-01/}LastModified').text
+                    print(date_time_str)
+                    date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ')
+                    return date_time_obj
+        except:
+            return None
+        return None
+
     @staticmethod
     def check() -> bool:
         print("Checking MinIO availability")

middleware/app_be/services/mongodbservice.py

@@ -108,6 +108,13 @@ class MongoDBService:
             print("Could not delete Metadata")
         return resp
 
+    @staticmethod
+    def replaceHash(identifier, decoded_image):
+        instance = MongoManager.getInstance()
+        db = instance.AIC
+        col = db.metadata
+        col.update_one({"identifier": identifier}, {"$set": {"sha512": create_sha512(decoded_image)}})
+
     @staticmethod
     def deleteAll():
         instance = MongoManager.getInstance()

middleware/app_be/urls.py

@@ -25,6 +25,7 @@ urlpatterns = [
     url(r'^test/', TestApiClass.test_api),
     url(r'^image/get/all$', ImageEndpoint.image_api_get_all),
     url(r'^image/get/(?P<identifier>[\w-]+)$', ImageEndpoint.image_api_get_single),
+    url(r'^image/get/(?P<identifier>[\w-]+)/version/(?P<version>[\w-]+)$', ImageEndpoint.image_api_get_single_version),
     url(r'^image/delete/all$', ImageEndpoint.image_api_delete_all),
     url(r'^image/delete/(?P<identifier>[\w-]+)$', ImageEndpoint.image_api_delete),
     url(r'^image/post$', ImageEndpoint.image_api_post),

middleware/app_be/views/rest_api.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from datetime import datetime
 
 from app_be.services.dropboxservice import DropboxService
 from app_be.services.hashservice import create_sha512
@@ -36,10 +37,20 @@ class ImageEndpoint:
 
         return JsonResponse({'Result': 'success1'}, safe=False)
 
+    @staticmethod
+    @api_view(['GET'])
+    def image_api_get_single_version(request, identifier, version):
+        logger.debug('Image GET single with version call: {}'.format(request))
+        return ImageEndpoint.get_image(identifier+'_'+version)
+
     @staticmethod
     @api_view(['GET'])
     def image_api_get_single(request, identifier):
         logger.debug('Image GET single call: {}'.format(request))
+        return ImageEndpoint.get_image(identifier)
+
+    @staticmethod
+    def get_image(identifier):
 
         # get metadata from MongoDB
         metadata = MongoDBService.getSingle(identifier)
@@ -58,7 +69,6 @@ class ImageEndpoint:
         # check image existence and correctness for each service + retrieve valid image if possible
         for service in ImageEndpoint.storageServiceList:
             logger.debug('Checking recovery for service ' + service.name)
-            print('Checking recovery for service ' + service.name)
             service_image_bytes = service.read_file(metadata['filename'])
             if service_image_bytes is not None:
                 recovery_has_image[service.name] = True
@@ -73,27 +83,39 @@ class ImageEndpoint:
                 recovery_has_image[service.name] = False
                 recovery_hash_matches[service.name] = False
 
-        # TODO: after talking with tobias about updating => replace with older version if hash is wrong
-
         # check if any service has the image saved
         if not ImageEndpoint.combine_boolean_dict_or(recovery_has_image):
             logger.debug('None of the storage services has the requested image saved')
-            MongoDBService.deleteSingle(identifier)
+            return JsonResponse({'Result': 'Error - image is not available on any storage service',
-            return JsonResponse({'Result': 'Error - image is not available on any storage service and was deleted',
                                  'id': identifier}, status=404, safe=False)
 
         # check if any service has a valid version of the image saved
         if not ImageEndpoint.combine_boolean_dict_or(recovery_hash_matches) and valid_image_bytes is None:
-            logger.debug('None of the storage services has a valid image saved')
+            logger.debug('None of the storage services has a valid image saved - assume oldest image is valid')
-            MongoDBService.deleteSingle(identifier)
+            service_with_oldest = None
-            return JsonResponse({'Result': 'Error - image is not available on any storage service and was deleted',
+            oldest_last_modified = datetime.now()
-                                 'id': identifier}, status=404, safe=False)
+            for service in ImageEndpoint.storageServiceList:
+                modified = service.get_last_modified(metadata['filename'])
+                if modified is not None and modified < oldest_last_modified:
+                    oldest_last_modified = modified
+                    service_with_oldest = service
 
-        # at this point we know at least one valid image is available
+            if service_with_oldest is None: # failsave if no image is available
-        for service in ImageEndpoint.storageServiceList:
+                return JsonResponse({'Result': 'Error - image is not available on any storage service',
-            if not recovery_has_image[service.name] or not recovery_hash_matches[service.name]:
+                                     'id': identifier}, status=404, safe=False)
-                if not service.create_file(metadata['filename'], valid_image_bytes):
+            # replace image on every service with oldest available
-                    logger.error('Error duplicating file in service ' + service.name)
+            valid_image_bytes = service_with_oldest.read_file(metadata['filename'])
+            MongoDBService.replaceHash(identifier, valid_image_bytes)
+            for service in ImageEndpoint.storageServiceList:
+                if service != service_with_oldest:
+                    if not service.create_file(metadata['filename'], valid_image_bytes):
+                        logger.error('Error recovering file in service ' + service.name)
+        else:
+            # at this point we know at least one valid image is available
+            for service in ImageEndpoint.storageServiceList:
+                if not recovery_has_image[service.name] or not recovery_hash_matches[service.name]:
+                    if not service.create_file(metadata['filename'], valid_image_bytes):
+                        logger.error('Error duplicating file in service ' + service.name)
 
         payload = {
             'id': identifier,