zenon/federated-storage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added create / compare hashes and notify on miss-match for MinIO backups;

Browse Source
This commit is contained in:
Marco Zeisler 2020-12-06 16:48:50 +01:00
parent 7eb728968a
commit 776c144f3f
3 changed files with 57 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
middleware/.run/IoTClient.run.xml Normal file
View File

@ -0,0 +1,24 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="IoTClient" type="PythonConfigurationType" factoryName="Python">
<module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/iotclient" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/iotclient/iot_client.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
</component>

6
middleware/app_be/services/minioservice.py
View File

@ -1,3 +1,5 @@
from typing import Union
import requests
from django.conf import settings
@ -31,11 +33,11 @@ class MinioService:
:param filename: filename
:return: bytes representation of the file or None on error
"""
file_bytes: bytes
file_bytes: Union[bytes, None] = None
try:
url = settings.AWS_HOST + filename
response = requests.get(url, stream=True)
file_bytes=response.content
except:
print("Error downloading file from minio")
return file_bytes
return file_bytes

52
middleware/app_be/views/rest_api.py
View File

@ -1,16 +1,13 @@
import hashlib
import logging
import json
import logging
from app_be.services.dropboxservice import DropboxService
from app_be.services.mongodbservice import MongoDBService
from app_be.services.minioservice import MinioService
from app_be.services.hashservice import create_sha512
from app_be.services.minioservice import MinioService
from app_be.services.mongodbservice import MongoDBService
from app_be.services.wrapperservice import WrapperService
from app_be.views.mongo_db import MongoManager
from django.http import JsonResponse
from django.http import HttpRequest
from django.http import JsonResponse
from rest_framework.decorators import api_view
logger = logging.getLogger(__name__)
@ -31,7 +28,6 @@ class ImageEndpoint:
logger.debug('Image GET all call: {}'.format(request))
print(request)
metadata = MongoDBService.getAll()
print(metadata)
@ -42,30 +38,42 @@ class ImageEndpoint:
def image_api_get_single(request, identifier):
logger.debug('Image GET single call: {}'.format(request))
# get metadata from MongoDB
metadata = MongoDBService.getSingle(identifier)
print(metadata)
if not metadata:
return JsonResponse({'Result': 'Error - could not find any metadata in mongoDB.',
'id': identifier}, status=404, safe=False)
logger.debug('Mongo Meta: {}'.format(metadata))
# get stored SHA512 hash
stored_hash = metadata.get('sha512', '')
logger.debug('Sorted SHA512: {}'.format(stored_hash))
# get image bytes from Dropbox
dropbox_image_bytes = DropboxService.read_file(metadata['filename'])
#minio_image_bytes = MinioService.read_file(metadata['filename'])
if dropbox_image_bytes is None:
return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404,
safe=False)
stored_hash = metadata.get('sha512', '')
print(stored_hash)
# calc dropbox image hash
actual_dropbox_hash = create_sha512(dropbox_image_bytes)
logger.debug('Actual Dropbox SHA512: {}'.format(actual_dropbox_hash))
if stored_hash != actual_dropbox_hash:
return JsonResponse('Stored hash does not match generated one! '
'stored: {} actual: {}'.format(stored_hash, actual_dropbox_hash), safe=False)
# TODO - check hash of MinIO image, too
# actual_minio_hash = '' # create_sha512(minio_image_bytes)
# if stored_hash != actual_minio_hash:
# return JsonResponse('Stored hash does not match generated one! '
# 'stored: {} actual: {}'.format(stored_hash, actual_minio_hash), safe=False)
# get image bytes from MinIO
minio_image_bytes = MinioService.read_file(metadata['filename'])
if minio_image_bytes is None:
return JsonResponse({'Result': 'Error - could not find image in minIO.', 'id': identifier}, status=404,
safe=False)
# calc minIO image hash
actual_minio_hash = create_sha512(minio_image_bytes)
logger.debug('Actual MinIO SHA512: {}'.format(actual_minio_hash))
if stored_hash != actual_minio_hash:
return JsonResponse('Stored hash does not match generated one! '
'stored: {} actual: {}'.format(stored_hash, actual_minio_hash), safe=False)
payload = {
'id': identifier,
@ -87,7 +95,7 @@ class ImageEndpoint:
decoded_image = WrapperService.unwrap_file(b64encoded_image)
MongoDBService.createSingle(metadata,identifier,decoded_image)
MongoDBService.createSingle(metadata, identifier, decoded_image)
if not DropboxService.create_file(filename, decoded_image):
print("Could not save image to dropbox")
@ -102,9 +110,7 @@ class ImageEndpoint:
logger.debug('Image DELETE single call: {}'.format(request))
resp = MongoDBService.deleteSingle(identifier)
print(resp)
return JsonResponse({'Result': 'success1'}, safe=False)