305 lines
12 KiB
Python
305 lines
12 KiB
Python
import json
|
|
import logging
|
|
from datetime import datetime
|
|
|
|
from app_be.services.dropboxservice import DropboxService
|
|
from app_be.services.hashservice import create_sha512
|
|
from app_be.services.minioservice import MinioService
|
|
from app_be.services.mongodbservice import MongoDBService
|
|
from app_be.services.wrapperservice import WrapperService
|
|
from django.http import HttpRequest
|
|
from django.http import JsonResponse
|
|
from rest_framework.decorators import api_view
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class TestApiClass:
|
|
@staticmethod
|
|
@api_view(['GET'])
|
|
def test_api(request):
|
|
logger.debug('Test api call: {}'.format(request))
|
|
return JsonResponse({'Result': 'success'}, safe=False)
|
|
|
|
|
|
class ImageEndpoint:
|
|
storageServiceList = [DropboxService, MinioService]
|
|
|
|
@staticmethod
|
|
@api_view(['GET'])
|
|
def image_api_get_all(request):
|
|
logger.debug('Image GET all call: {}'.format(request))
|
|
print(request)
|
|
|
|
metadata = MongoDBService.getAll()
|
|
print(metadata)
|
|
|
|
return JsonResponse(metadata, safe=False)
|
|
|
|
@staticmethod
|
|
@api_view(['GET'])
|
|
def image_api_get_single_version(request, identifier, version):
|
|
logger.debug('Image GET single with version call: {}'.format(request))
|
|
return ImageEndpoint.get_image(identifier + '_' + version)
|
|
|
|
@staticmethod
|
|
@api_view(['GET'])
|
|
def image_api_get_single(request, identifier):
|
|
logger.debug('Image GET single call: {}'.format(request))
|
|
return ImageEndpoint.get_image(identifier)
|
|
|
|
@staticmethod
|
|
@api_view(['GET'])
|
|
def image_api_get_status(request, identifier):
|
|
# get metadata from MongoDB
|
|
metadata = MongoDBService.getSingle(identifier)
|
|
if not metadata:
|
|
return JsonResponse({'valid': False, 'available': False})
|
|
|
|
# get stored SHA512 hash
|
|
stored_hash = metadata.get('sha512', '')
|
|
no_image = True
|
|
|
|
for service in ImageEndpoint.storageServiceList:
|
|
service_image_bytes = service.read_file(metadata['filename'])
|
|
if service_image_bytes is not None:
|
|
no_image = False
|
|
actual_service_hash = create_sha512(service_image_bytes)
|
|
if stored_hash != actual_service_hash:
|
|
return JsonResponse({'valid': False, 'available': True})
|
|
|
|
if no_image:
|
|
return JsonResponse({'valid': False, 'available': False})
|
|
|
|
return JsonResponse({'valid': True, 'available': True})
|
|
|
|
@staticmethod
|
|
def get_image(identifier):
|
|
|
|
# get metadata from MongoDB
|
|
metadata = MongoDBService.getSingle(identifier)
|
|
if not metadata:
|
|
return JsonResponse({'Result': 'Error - could not find any metadata in mongoDB.',
|
|
'id': identifier}, status=404, safe=False)
|
|
logger.debug('Mongo Meta: {}'.format(metadata))
|
|
|
|
# get stored SHA512 hash
|
|
stored_hash = metadata.get('sha512', '')
|
|
logger.debug('Sorted SHA512: {}'.format(stored_hash))
|
|
# recovery code
|
|
recovery_has_image = {}
|
|
recovery_hash_matches = {}
|
|
valid_image_bytes = None
|
|
# check image existence and correctness for each service + retrieve valid image if possible
|
|
for service in ImageEndpoint.storageServiceList:
|
|
logger.debug('Checking recovery for service ' + service.name)
|
|
service_image_bytes = service.read_file(metadata['filename'])
|
|
if service_image_bytes is not None:
|
|
recovery_has_image[service.name] = True
|
|
actual_service_hash = create_sha512(service_image_bytes)
|
|
if stored_hash == actual_service_hash:
|
|
recovery_hash_matches[service.name] = True
|
|
if valid_image_bytes is None:
|
|
valid_image_bytes = service_image_bytes
|
|
else:
|
|
recovery_hash_matches[service.name] = False
|
|
else:
|
|
recovery_has_image[service.name] = False
|
|
recovery_hash_matches[service.name] = False
|
|
|
|
# check if any service has the image saved
|
|
if not ImageEndpoint.combine_boolean_dict_or(recovery_has_image):
|
|
logger.debug('None of the storage services has the requested image saved')
|
|
return JsonResponse({'Result': 'Error - image is not available on any storage service',
|
|
'id': identifier}, status=404, safe=False)
|
|
|
|
# check if any service has a valid version of the image saved
|
|
if not ImageEndpoint.combine_boolean_dict_or(recovery_hash_matches) and valid_image_bytes is None:
|
|
logger.debug('None of the storage services has a valid image saved - assume oldest image is valid')
|
|
service_with_oldest = None
|
|
oldest_last_modified = datetime.now()
|
|
for service in ImageEndpoint.storageServiceList:
|
|
modified = service.get_last_modified(metadata['filename'])
|
|
if modified is not None and modified < oldest_last_modified:
|
|
oldest_last_modified = modified
|
|
service_with_oldest = service
|
|
|
|
if service_with_oldest is None: # failsave if no image is available
|
|
return JsonResponse({'Result': 'Error - image is not available on any storage service',
|
|
'id': identifier}, status=404, safe=False)
|
|
# replace image on every service with oldest available
|
|
valid_image_bytes = service_with_oldest.read_file(metadata['filename'])
|
|
MongoDBService.replaceHash(identifier, valid_image_bytes)
|
|
for service in ImageEndpoint.storageServiceList:
|
|
if service != service_with_oldest:
|
|
if not service.create_file(metadata['filename'], valid_image_bytes):
|
|
logger.error('Error recovering file in service ' + service.name)
|
|
else:
|
|
# at this point we know at least one valid image is available
|
|
for service in ImageEndpoint.storageServiceList:
|
|
if not recovery_has_image[service.name] or not recovery_hash_matches[service.name]:
|
|
if recovery_has_image[service.name]:
|
|
service.delete_file(metadata['filename'])
|
|
if not service.create_file(metadata['filename'], valid_image_bytes):
|
|
logger.error('Error duplicating file in service ' + service.name)
|
|
|
|
payload = {
|
|
'id': identifier,
|
|
'metadata': metadata,
|
|
'image_data': WrapperService.wrap_file(valid_image_bytes)
|
|
}
|
|
|
|
return JsonResponse(payload, safe=False)
|
|
|
|
@staticmethod
|
|
@api_view(['POST'])
|
|
def image_api_post(request: HttpRequest):
|
|
logger.debug('Image POST call: {}'.format(request))
|
|
|
|
try:
|
|
payload = json.loads(request.body)
|
|
except json.decoder.JSONDecodeError:
|
|
try:
|
|
payload = request.data.dict()
|
|
payload['metadata'] = json.loads(payload['metadata'])
|
|
except Exception as e:
|
|
return JsonResponse({'Result': 'Error - could not parse uploaded data',
|
|
'Error': str(e)}, status=500, safe=False)
|
|
|
|
b64encoded_image = payload['image_data']
|
|
identifier = payload['id']
|
|
metadata = payload['metadata']
|
|
metadata['version'] = 0
|
|
metadata['previous'] = ''
|
|
filename = payload['metadata']['filename']
|
|
|
|
decoded_image = WrapperService.unwrap_file(b64encoded_image)
|
|
|
|
try:
|
|
MongoDBService.createSingle(metadata, identifier, decoded_image)
|
|
except Exception as e:
|
|
print("Could not save metadata:", e)
|
|
return JsonResponse({'Result': 'Error - could not upload to MongoDB',
|
|
'Error': str(e), 'id': identifier, 'filename': filename}, status=500, safe=False)
|
|
|
|
for service in ImageEndpoint.storageServiceList:
|
|
if not service.create_file(filename, decoded_image):
|
|
print("Could not save image to " + service.name)
|
|
return JsonResponse(
|
|
{'Result': 'Error - could not upload to ' + service.name, 'id': identifier, 'filename': filename},
|
|
status=500, safe=False)
|
|
|
|
return JsonResponse({'id': identifier, 'filename': filename}, safe=False)
|
|
|
|
@staticmethod
|
|
@api_view(['DELETE'])
|
|
def image_api_delete(request: HttpRequest, identifier):
|
|
|
|
logger.debug('Image DELETE single call: {}'.format(request))
|
|
result_bool = True
|
|
|
|
# get metadata from MongoDB
|
|
metadata = MongoDBService.getSingle(identifier)
|
|
|
|
if not metadata:
|
|
return JsonResponse({'Result': 'Error - could not find any metadata in mongoDB.',
|
|
'id': identifier}, status=404, safe=False)
|
|
|
|
resp = MongoDBService.deleteSingle(identifier)
|
|
print(resp)
|
|
|
|
for service in ImageEndpoint.storageServiceList:
|
|
if not service.delete_file(metadata['filename']):
|
|
print('Error deleting file in ' + service.name)
|
|
result_bool = False
|
|
|
|
return JsonResponse({'Result': result_bool}, safe=False)
|
|
|
|
@staticmethod
|
|
@api_view(['DELETE'])
|
|
def image_api_delete_all(request: HttpRequest):
|
|
|
|
logger.debug('Image DELETE single call: {}'.format(request))
|
|
result_bool = True
|
|
|
|
resp = MongoDBService.deleteAll()
|
|
print(resp)
|
|
|
|
for service in ImageEndpoint.storageServiceList:
|
|
if not service.delete_all():
|
|
print('Error deleting ' + service.name + ' folder')
|
|
result_bool = False
|
|
|
|
return JsonResponse({'Result': result_bool}, safe=False)
|
|
|
|
@staticmethod
|
|
@api_view(['PUT'])
|
|
def image_api_update(request, identifier):
|
|
logger.debug('Image UPDATE single call: {}'.format(request))
|
|
|
|
try:
|
|
payload = json.loads(request.body)
|
|
except json.decoder.JSONDecodeError:
|
|
try:
|
|
payload = request.data.dict()
|
|
payload['metadata'] = json.loads(payload['metadata'])
|
|
except Exception as e:
|
|
return JsonResponse({'Result': 'Error - could not parse uploaded data',
|
|
'Error': str(e)}, status=500, safe=False)
|
|
|
|
b64encoded_image = payload['image_data']
|
|
# identifier = identifier
|
|
metadata = payload['metadata']
|
|
filename = payload['metadata']['filename']
|
|
|
|
# get metadata from MongoDB
|
|
mongodb_metadata = MongoDBService.getSingle(identifier)
|
|
if not mongodb_metadata:
|
|
return JsonResponse({'Result': 'Error - Could not find image to be updated',
|
|
'id': identifier}, status=404, safe=False)
|
|
|
|
metadata['version'] = mongodb_metadata['version']
|
|
filename = mongodb_metadata['filename']
|
|
|
|
decoded_image = WrapperService.unwrap_file(b64encoded_image)
|
|
|
|
MongoDBService.updateSingle(identifier, decoded_image, metadata)
|
|
|
|
for service in ImageEndpoint.storageServiceList:
|
|
orig_new_name = identifier + '_' + str(metadata['version']) + '.jpg'
|
|
if not service.move_file(identifier, orig_new_name):
|
|
print("Could not move file from {} to {}".format(identifier, orig_new_name))
|
|
if not service.create_file(filename, decoded_image):
|
|
print("Could not save updated image to " + service.name)
|
|
|
|
return JsonResponse({'id': identifier, 'filename': filename},
|
|
safe=False)
|
|
|
|
@staticmethod
|
|
def combine_boolean_dict_and(d: dict) -> bool:
|
|
result = True
|
|
for entry in d:
|
|
result = (result and d[entry])
|
|
return result
|
|
|
|
@staticmethod
|
|
def combine_boolean_dict_or(d: dict) -> bool:
|
|
result = False
|
|
for entry in d:
|
|
result = (result or d[entry])
|
|
return result
|
|
|
|
|
|
class HealthEndpoint:
|
|
storageServiceList = [DropboxService, MinioService]
|
|
|
|
@staticmethod
|
|
@api_view(['GET'])
|
|
def check_systems(request):
|
|
logger.debug('Check health of all sub-systems')
|
|
response = {'MongoDB': MongoDBService.check()}
|
|
for service in HealthEndpoint.storageServiceList:
|
|
response[service.name] = service.check()
|
|
|
|
return JsonResponse(response, safe=False)
|