zenon/federated-storage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rewrote storage services to implement new interface

Browse Source
This commit is contained in:
Martin Schett 2020-12-28 17:29:14 +01:00
parent 6d9f1a856d
commit 96b94f5c89
4 changed files with 112 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
middleware/app_be/services/StorageServiceInterface.py Normal file
View File

@ -0,0 +1,53 @@
class StorageServiceInterface:
name = "StorageServiceInterface"
@staticmethod
def check() -> bool:
pass
@staticmethod
def create_file(filename: str, file: bytes) -> bool:
"""Create / save (or overwrite) an file on the storage.
:param filename: filename
:param file: bytes representation of the file
:return: Whether file was successfully uploaded
"""
pass
@staticmethod
def read_file(filename: str) -> bytes:
"""Read file on the storage.
:param filename: filename
:return: bytes representation of the file or None on error
"""
pass
@staticmethod
def update_file(filename: str, file: bytes) -> bool:
"""Update (currently: overwrite) an file on the storage.
:param filename: filename of the file
:param file: bytes representation of the file
:return: Whether file was successfully uploaded
"""
pass
@staticmethod
def delete_file(filename: str) -> bool:
"""Delete an file from the storage.
:param filename: filename of the file
:return: Whether file was successfully deleted
"""
pass
@staticmethod
def delete_all() -> bool:
"""Delete every file from the storage.
:return: Whether the storage was successfully emptied
"""
pass

9
middleware/app_be/services/dropboxservice.py
View File

@ -2,8 +2,12 @@ import requests
from django.conf import settings
import dropbox
from app_be.services.StorageServiceInterface import StorageServiceInterface
class DropboxService:
class DropboxService(StorageServiceInterface):
name = "Dropbox"
@staticmethod
def check() -> bool:
@ -17,7 +21,6 @@ class DropboxService:
return False
return True
@staticmethod
def create_file(filename: str, file: bytes) -> bool:
"""Create / save (or overwrite) an file on the dropbox storage.
@ -89,4 +92,4 @@ class DropboxService:
except dropbox.exceptions.ApiError as e:
print(e)
return False
return True
return True

7
middleware/app_be/services/minioservice.py
View File

@ -6,7 +6,12 @@ import xml.etree.ElementTree as ET
import requests
from django.conf import settings
class MinioService:
from app_be.services.StorageServiceInterface import StorageServiceInterface
class MinioService(StorageServiceInterface):
name = "MinIO"
@staticmethod
def check() -> bool:

104
middleware/app_be/views/rest_api.py
View File

@ -22,6 +22,9 @@ class TestApiClass:
class ImageEndpoint:
storageServiceList = [DropboxService, MinioService]
@staticmethod
@api_view(['GET'])
def image_api_get_all(request):
@ -48,37 +51,27 @@ class ImageEndpoint:
# get stored SHA512 hash
stored_hash = metadata.get('sha512', '')
logger.debug('Sorted SHA512: {}'.format(stored_hash))
service_image_bytes = None
for service in ImageEndpoint.storageServiceList:
logger.debug('Checking hash for ' + service.name)
# get image bytes from Dropbox
service_image_bytes = service.read_file(metadata['filename'])
if service_image_bytes is None:
return JsonResponse({'Result': 'Error - could not find image in ' + service.name, 'id': identifier},
status=404, safe=False)
# get image bytes from Dropbox
dropbox_image_bytes = DropboxService.read_file(metadata['filename'])
if dropbox_image_bytes is None:
return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404,
safe=False)
# calc dropbox image hash
actual_dropbox_hash = create_sha512(dropbox_image_bytes)
logger.debug('Actual Dropbox SHA512: {}'.format(actual_dropbox_hash))
if stored_hash != actual_dropbox_hash:
return JsonResponse('Stored hash does not match generated one! '
'stored: {} actual: {}'.format(stored_hash, actual_dropbox_hash), status=400, safe=False)
# get image bytes from MinIO
minio_image_bytes = MinioService.read_file(metadata['filename'])
if minio_image_bytes is None:
return JsonResponse({'Result': 'Error - could not find image in minIO.', 'id': identifier}, status=404,
safe=False)
# calc minIO image hash
actual_minio_hash = create_sha512(minio_image_bytes)
logger.debug('Actual MinIO SHA512: {}'.format(actual_minio_hash))
if stored_hash != actual_minio_hash:
return JsonResponse('Stored hash does not match generated one! '
'stored: {} actual: {}'.format(stored_hash, actual_minio_hash), status=400, safe=False)
# calc dropbox image hash
actual_service_hash = create_sha512(service_image_bytes)
logger.debug('Actual '+service.name+' SHA512: {}'.format(actual_service_hash))
if stored_hash != actual_service_hash:
return JsonResponse('Stored hash in service "'+service.name+'"does not match generated one! '
'stored: {} actual: {}'.format(stored_hash, actual_service_hash),
status=400, safe=False)
payload = {
'id': identifier,
'metadata': metadata,
'image_data': WrapperService.wrap_file(dropbox_image_bytes)
'image_data': WrapperService.wrap_file(service_image_bytes)
}
return JsonResponse(payload, safe=False)
@ -98,12 +91,11 @@ class ImageEndpoint:
if not MongoDBService.createSingle(metadata, identifier, decoded_image):
print("Could not save metadata")
return JsonResponse({'Result': 'Error - could not upload to MongoDB', 'id': identifier, 'filename': filename},status=500,safe=False)
if not DropboxService.create_file(filename, decoded_image):
print("Could not save image to dropbox")
return JsonResponse({'Result': 'Error - could not upload to Dropbox', 'id': identifier, 'filename': filename},status=500, safe=False)
if not MinioService.create_file(filename, decoded_image):
print("Could not save image to minio")
return JsonResponse({'Result': 'Error - could not upload to MinIO', 'id': identifier, 'filename': filename}, status=500, safe=False)
for service in ImageEndpoint.storageServiceList:
if not service.create_file(filename, decoded_image):
print("Could not save image to " + service.name)
return JsonResponse({'Result': 'Error - could not upload to ' + service.name, 'id': identifier, 'filename': filename},status=500, safe=False)
return JsonResponse({'id': identifier, 'filename': filename},safe=False)
@ -124,14 +116,10 @@ class ImageEndpoint:
resp = MongoDBService.deleteSingle(identifier)
print(resp)
if not DropboxService.delete_file(metadata['filename']):
print('Error deleting file in dropbox')
result_bool = False
if not MinioService.delete_file(metadata['filename']):
print('Error deleting file in minio')
result_bool = False
for service in ImageEndpoint.storageServiceList:
if not service.delete_file(metadata['filename']):
print('Error deleting file in ' + service.name)
result_bool = False
return JsonResponse({'Result': result_bool}, safe=False)
@ -145,13 +133,10 @@ class ImageEndpoint:
resp = MongoDBService.deleteAll()
print(resp)
if not DropboxService.delete_all():
print('Error deleting dropbox folder')
result_bool = False
if not MinioService.delete_all():
print('Error deleting minio folder')
result_bool = False
for service in ImageEndpoint.storageServiceList:
if not service.delete_all():
print('Error deleting ' + service.name + ' folder')
result_bool = False
return JsonResponse({'Result': result_bool}, safe=False)
@ -167,8 +152,9 @@ class ImageEndpoint:
return JsonResponse({'Result': 'Error - Could not find image to be updated',
'id': identifier}, status=404, safe=False)
DropboxService.delete_file(metadata['filename'])
# MinioService.delete_file(metadata['filename'])
for service in ImageEndpoint.storageServiceList:
service.delete_file(metadata['filename'])
MongoDBService.deleteSingle(identifier)
@ -182,20 +168,24 @@ class ImageEndpoint:
MongoDBService.createSingle(metadata, identifier, decoded_image)
if not DropboxService.create_file(filename, decoded_image):
print("Could not save image to dropbox")
if not MinioService.create_file(filename, decoded_image):
print("Could not save image to minio")
for service in ImageEndpoint.storageServiceList:
if not service.create_file(filename, decoded_image):
print("Could not save image to " + service.name)
return JsonResponse({'id': identifier, 'filename': filename},
safe=False)
class HealthEndpoint:
storageServiceList = [DropboxService, MinioService]
@staticmethod
@api_view(['GET'])
def check_systems(request):
logger.debug('Check health of all sub-systems')
mongo = MongoDBService.check()
dbx = DropboxService.check()
minio = MinioService.check()
response = {'MongoDB': MongoDBService.check()}
for service in HealthEndpoint.storageServiceList:
response[service.name] = service.check()
return JsonResponse({'MongoDB': mongo, 'Dropbox': dbx, 'MinIO': minio}, safe=False)
return JsonResponse(response, safe=False)