zenon/federated-storage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated recovery logic to fall back on older version if no image-hash is valid

Browse Source
This commit is contained in:
Martin Schett 2021-01-09 21:47:03 +01:00
parent cd707226f6
commit 1ed0627853
7 changed files with 111 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
iotclient/iot_client.py
View File

@ -73,11 +73,13 @@ def update_image(identifier, image_path, metadata_payload):
return True
def get_image(identifier):
def get_image(identifier, version=None):
print("Getting image with identifier " + identifier)
baseurl = "http://127.0.0.1:8000"
get_url = "/image/get/" + identifier
if version is not None:
get_url = get_url + "/version/"+str(version)
try:
response = requests.get(baseurl + get_url)
@ -248,6 +250,7 @@ while (command.lower() not in ["exit", "quit", "end"]):
print("trigger - next image in line is sent to backend")
print("update - next image in line is updated in backend")
print("fetch - gets the next image from database if exists")
print("fetchversion - gets the next image with specified version from database if exists")
print("fetchall - get all images")
print("delete - delets the next image from db if exists")
print("deleteall - delets all images from the service")
@ -354,6 +357,25 @@ while (command.lower() not in ["exit", "quit", "end"]):
get_image(filename[:-4])
print_cursor()
if command.lower() == "fetchversion":
if (len(attributes) < 1):
print("Error: No version supplied")
continue
if not attributes[0].isnumeric():
print("Version is no number")
continue
version = int(attributes[0])
if metadata is None:
print("No metadata loaded")
continue
if image_folder is None:
print("No image folder selected")
continue
meta_payload = metadata[index]
filename = meta_payload['filename']
get_image(filename[:-4], version)
print_cursor()
if command.lower() == "fetchall":
if metadata is None:
print("No metadata loaded")

9
middleware/app_be/services/StorageServiceInterface.py
View File

@ -2,6 +2,15 @@ class StorageServiceInterface:
name = "StorageServiceInterface"
@staticmethod
def get_last_modified(filename: str):
"""Get last modified time of file
:param filename: filename
:return: Date of last modification or None
"""
pass
@staticmethod
def check() -> bool:
pass

14
middleware/app_be/services/dropboxservice.py
View File

@ -9,6 +9,20 @@ from app_be.services.StorageServiceInterface import StorageServiceInterface
class DropboxService(StorageServiceInterface):
name = "Dropbox"
@staticmethod
def get_last_modified(filename: str):
"""Get last modified time of file
:param filename: filename
:return: Date of last modification or None
"""
try:
dbx = dropbox.Dropbox(settings.DROPBOX_OAUTH2_ACCESS_TOKEN)
return dbx.files_get_metadata(settings.DROPBOX_IMAGE_FOLDER + filename).server_modified
except:
return None
return None
@staticmethod
def check() -> bool:
try:

21
middleware/app_be/services/minioservice.py
View File

@ -1,5 +1,6 @@
import base64
import hashlib
from datetime import datetime
from typing import Union
import xml.etree.ElementTree as ET
@ -12,6 +13,26 @@ from app_be.services.StorageServiceInterface import StorageServiceInterface
class MinioService(StorageServiceInterface):
name = "MinIO"
@staticmethod
def get_last_modified(filename: str):
"""Get last modified time of file
:param filename: filename
:return: Date of last modification or None
"""
try:
r = requests.get(settings.AWS_HOST + "?prefix="+filename)
root = ET.fromstring(r.content)
for child in root.iter('{http://s3.amazonaws.com/doc/2006-03-01/}Contents'):
if child.find('{http://s3.amazonaws.com/doc/2006-03-01/}Key').text == filename:
date_time_str = child.find('{http://s3.amazonaws.com/doc/2006-03-01/}LastModified').text
print(date_time_str)
date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ')
return date_time_obj
except:
return None
return None
@staticmethod
def check() -> bool:
print("Checking MinIO availability")

7
middleware/app_be/services/mongodbservice.py
View File

@ -108,6 +108,13 @@ class MongoDBService:
print("Could not delete Metadata")
return resp
@staticmethod
def replaceHash(identifier, decoded_image):
instance = MongoManager.getInstance()
db = instance.AIC
col = db.metadata
col.update_one({"identifier": identifier}, {"$set": {"sha512": create_sha512(decoded_image)}})
@staticmethod
def deleteAll():
instance = MongoManager.getInstance()

1
middleware/app_be/urls.py
View File

@ -25,6 +25,7 @@ urlpatterns = [
url(r'^test/', TestApiClass.test_api),
url(r'^image/get/all$', ImageEndpoint.image_api_get_all),
url(r'^image/get/(?P<identifier>[\w-]+)$', ImageEndpoint.image_api_get_single),
url(r'^image/get/(?P<identifier>[\w-]+)/version/(?P<version>[\w-]+)$', ImageEndpoint.image_api_get_single_version),
url(r'^image/delete/all$', ImageEndpoint.image_api_delete_all),
url(r'^image/delete/(?P<identifier>[\w-]+)$', ImageEndpoint.image_api_delete),
url(r'^image/post$', ImageEndpoint.image_api_post),

50
middleware/app_be/views/rest_api.py
View File

@ -1,5 +1,6 @@
import json
import logging
from datetime import datetime
from app_be.services.dropboxservice import DropboxService
from app_be.services.hashservice import create_sha512
@ -36,10 +37,20 @@ class ImageEndpoint:
return JsonResponse({'Result': 'success1'}, safe=False)
@staticmethod
@api_view(['GET'])
def image_api_get_single_version(request, identifier, version):
logger.debug('Image GET single with version call: {}'.format(request))
return ImageEndpoint.get_image(identifier+'_'+version)
@staticmethod
@api_view(['GET'])
def image_api_get_single(request, identifier):
logger.debug('Image GET single call: {}'.format(request))
return ImageEndpoint.get_image(identifier)
@staticmethod
def get_image(identifier):
# get metadata from MongoDB
metadata = MongoDBService.getSingle(identifier)
@ -58,7 +69,6 @@ class ImageEndpoint:
# check image existence and correctness for each service + retrieve valid image if possible
for service in ImageEndpoint.storageServiceList:
logger.debug('Checking recovery for service ' + service.name)
print('Checking recovery for service ' + service.name)
service_image_bytes = service.read_file(metadata['filename'])
if service_image_bytes is not None:
recovery_has_image[service.name] = True
@ -73,27 +83,39 @@ class ImageEndpoint:
recovery_has_image[service.name] = False
recovery_hash_matches[service.name] = False
# TODO: after talking with tobias about updating => replace with older version if hash is wrong
# check if any service has the image saved
if not ImageEndpoint.combine_boolean_dict_or(recovery_has_image):
logger.debug('None of the storage services has the requested image saved')
MongoDBService.deleteSingle(identifier)
return JsonResponse({'Result': 'Error - image is not available on any storage service and was deleted',
return JsonResponse({'Result': 'Error - image is not available on any storage service',
'id': identifier}, status=404, safe=False)
# check if any service has a valid version of the image saved
if not ImageEndpoint.combine_boolean_dict_or(recovery_hash_matches) and valid_image_bytes is None:
logger.debug('None of the storage services has a valid image saved')
MongoDBService.deleteSingle(identifier)
return JsonResponse({'Result': 'Error - image is not available on any storage service and was deleted',
'id': identifier}, status=404, safe=False)
logger.debug('None of the storage services has a valid image saved - assume oldest image is valid')
service_with_oldest = None
oldest_last_modified = datetime.now()
for service in ImageEndpoint.storageServiceList:
modified = service.get_last_modified(metadata['filename'])
if modified is not None and modified < oldest_last_modified:
oldest_last_modified = modified
service_with_oldest = service
# at this point we know at least one valid image is available
for service in ImageEndpoint.storageServiceList:
if not recovery_has_image[service.name] or not recovery_hash_matches[service.name]:
if not service.create_file(metadata['filename'], valid_image_bytes):
logger.error('Error duplicating file in service ' + service.name)
if service_with_oldest is None: # failsave if no image is available
return JsonResponse({'Result': 'Error - image is not available on any storage service',
'id': identifier}, status=404, safe=False)
# replace image on every service with oldest available
valid_image_bytes = service_with_oldest.read_file(metadata['filename'])
MongoDBService.replaceHash(identifier, valid_image_bytes)
for service in ImageEndpoint.storageServiceList:
if service != service_with_oldest:
if not service.create_file(metadata['filename'], valid_image_bytes):
logger.error('Error recovering file in service ' + service.name)
else:
# at this point we know at least one valid image is available
for service in ImageEndpoint.storageServiceList:
if not recovery_has_image[service.name] or not recovery_hash_matches[service.name]:
if not service.create_file(metadata['filename'], valid_image_bytes):
logger.error('Error duplicating file in service ' + service.name)
payload = {
'id': identifier,