zenon/federated-storage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into '3-minio_api'

Browse Source 
# Conflicts:
#   middleware/app_be/views/rest_api.py
This commit is contained in:
Martin Weick 2020-12-02 10:30:44 +01:00
commit 5dc0ccbde4
10 changed files with 131 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
iotclient/iot_client.py
View File

@ -5,12 +5,15 @@ import json
import time
from datetime import datetime
def print_response(response):
print("Code: " + str(response.status_code) + "; Body: " + str(response.json()))
def unwrap_file(file_str: str) -> bytes:
return base64.b64decode(file_str)
def send_image(identifier, image_path, metadata_payload):
print("Sending image with identifier " + identifier)
if not os.path.isfile(image_path):
@ -54,16 +57,38 @@ def get_image(identifier):
b64encoded_image = unwrap_file(payload['image_data'])
goal_folder_name = "images_fetch"
path = "../../" + goal_folder_name + "/" + payload['metadata']['filename']
goal_folder_path = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[:-3]) + os.path.sep + goal_folder_name
goal_folder_path = os.path.sep.join(
os.path.abspath(__file__).split(os.path.sep)[:-3]) + os.path.sep + goal_folder_name
if not os.path.isdir(goal_folder_path):
os.mkdir(goal_folder_path)
out = open(path, 'wb')
out.write(b64encoded_image)
out.close()
os.system(goal_folder_path + os.path.sep + payload['metadata']['filename'])
except os.error:
print("Error sending request")
except os.error as e:
print("Error sending request {}".format(e))
except KeyError as e:
print("Key error {}".format(e))
return True
def delete_image(identifier):
print("Deleting image with identifier " + identifier)
baseurl = "http://127.0.0.1:8000"
get_url = "/image/delete/" + identifier
try:
response = requests.delete(baseurl + get_url)
payload = response.json()
print(payload)
except os.error:
print("Error deleting request")
return True
def get_all():
print("Getting all images")
@ -78,6 +103,7 @@ def get_all():
print("Error sending request")
return True
metadata = None
index = 0
metadata_folder = "." + os.path.sep
@ -86,10 +112,10 @@ metadata_path = metadata_folder + os.path.sep + metadata_file
image_folder = "." + os.path.sep + "images"
if (os.path.isfile(metadata_path)):
print("Loading metadata to memory from " + metadata_path)
metadata = sorted(json.load(open(metadata_file, "r")), key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
metadata = sorted(json.load(open(metadata_file, "r")),
key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
else:
print("Default metadata_file not found.")
@ -127,6 +153,7 @@ while (command.lower() not in ["exit", "quit", "end"]):
print("trigger - next image in line is sent to backend")
print("fetch - gets the next image from database if exists")
print("fetchall - get all images")
print("delete - delets the next image from db if exists")
print("rapid <#amount> - next <amound> images in line are sent to backend in 2 second intervals")
print("skip [<#amount>] - skips the next <amount> number of images in line or 1 if no <amount> is given")
print("set <#index> - sets image on given <index> as next in line")
@ -163,7 +190,8 @@ while (command.lower() not in ["exit", "quit", "end"]):
print("Error: No file on path " + path)
continue
print("Loading metadata to memory from " + path)
metadata = sorted(json.load(open(metadata_file, "r")), key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
metadata = sorted(json.load(open(metadata_file, "r")),
key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
index = 0
if command.lower() == "imagefolder":
@ -211,7 +239,6 @@ while (command.lower() not in ["exit", "quit", "end"]):
get_image(filename[:-4])
index = index + 1
if command.lower() == "fetchall":
if metadata is None:
print("No metadata loaded")
@ -222,6 +249,18 @@ while (command.lower() not in ["exit", "quit", "end"]):
get_all()
index = index + 1
if command.lower() == "delete":
if metadata is None:
print("No metadata loaded")
continue
if image_folder is None:
print("No image folder selected")
continue
meta_payload = metadata[index]
filename = meta_payload['filename']
delete_image(filename[:-4])
index = index + 1
if command.lower() == "rapid":
if metadata is None:
print("No metadata loaded")
@ -277,7 +316,6 @@ while (command.lower() not in ["exit", "quit", "end"]):
continue
index = ind
if command.lower() == "select":
if metadata is None:
print("Please first select metadata")

4
middleware/.run/createsuperuser.run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="createsuperuser" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" />
<module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
@ -12,7 +12,7 @@
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="createsuperuser" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />

4
middleware/.run/makemigrations.run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="makemigrations" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" />
<module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
@ -12,7 +12,7 @@
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="makemigrations app_be" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />

4
middleware/.run/migrate.run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="migrate" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" />
<module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
@ -12,7 +12,7 @@
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="migrate" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />

6
middleware/.run/runserver.run.xml
View File

@ -1,18 +1,18 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="runserver" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" />
<module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/middleware" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="runserver" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />

7
middleware/app_be/services/hashservice.py Normal file
View File

@ -0,0 +1,7 @@
import hashlib
def create_sha512(b: bytes):
h = hashlib.sha512()
h.update(b)
return h.hexdigest()

2
middleware/app_be/services/wrapperservice.py
View File

@ -6,7 +6,7 @@ class WrapperService:
@staticmethod
def wrap_file(file_bytes: bytes) -> str:
file_encoded_b64 = base64.b64encode(file_bytes)
return file_encoded_b64.decode('utf-8')
return file_encoded_b64.decode()
@staticmethod
def unwrap_file(file_str: str) -> bytes:

1
middleware/app_be/urls.py
View File

@ -25,6 +25,7 @@ urlpatterns = [
url(r'^test/', TestApiClass.test_api),
url(r'^image/get/all$', ImageEndpoint.image_api_get_all),
url(r'^image/get/(?P<identifier>[\w-]+)$', ImageEndpoint.image_api_get_single),
url(r'^image/delete/(?P<identifier>[\w-]+)$', ImageEndpoint.image_api_delete),
url(r'^image/post$', ImageEndpoint.image_api_post)
]

7
middleware/app_be/views/mongo_db.py
View File

@ -1,5 +1,6 @@
import pymongo
class MongoManager:
__instance = None
@ -16,4 +17,8 @@ class MongoManager:
MongoManager.__instance = pymongo.MongoClient('127.0.0.1', 27017)
db = MongoManager.__instance.AIC
coll = db.metadata
coll.create_index("filename",unique=True)
db.coll["identifier"]
db.coll["sha512"]
db.coll["location"]
coll.create_index("identifier", unique=True)
coll.create_index([("location", pymongo.GEO2D)])

52
middleware/app_be/views/rest_api.py
View File

@ -1,15 +1,14 @@
import hashlib
import logging
import json
import base64
from app_be.services.dropboxservice import DropboxService
from app_be.services.minioservice import MinioService
from app_be.services.hashservice import create_sha512
from app_be.services.wrapperservice import WrapperService
from app_be.views.mongo_db import MongoManager
from django.http import JsonResponse
from django.http import HttpRequest
from rest_framework.decorators import api_view
logger = logging.getLogger(__name__)
@ -44,7 +43,6 @@ class ImageEndpoint:
return JsonResponse({'Result': 'success1'}, safe=False)
@staticmethod
@api_view(['GET'])
def image_api_get_single(request, identifier):
@ -56,7 +54,7 @@ class ImageEndpoint:
col = db.metadata
metadata = None
try:
resp = col.find({"filename": identifier+".jpg"})
resp = col.find({"identifier": identifier})
metadata = resp[0]
except:
print("Could not find Metadata")
@ -68,8 +66,23 @@ class ImageEndpoint:
dropbox_image_bytes = DropboxService.read_file(metadata['filename'])
#minio_image_bytes = MinioService.read_file(metadata['filename'])
if dropbox_image_bytes is None:
return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404, safe=False)
return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404,
safe=False)
stored_hash = metadata.get('sha512', '')
actual_dropbox_hash = create_sha512(dropbox_image_bytes)
if stored_hash != actual_dropbox_hash:
return JsonResponse('Stored hash does not match generated one! '
'stored: {} actual: {}'.format(stored_hash, actual_dropbox_hash), safe=False)
# TODO - check hash of MinIO image, too
# actual_minio_hash = '' # create_sha512(minio_image_bytes)
# if stored_hash != actual_minio_hash:
# return JsonResponse('Stored hash does not match generated one! '
# 'stored: {} actual: {}'.format(stored_hash, actual_minio_hash), safe=False)
payload = {
'id': identifier,
@ -94,15 +107,38 @@ class ImageEndpoint:
db = instance.AIC
col = db.metadata
decoded_image = WrapperService.unwrap_file(b64encoded_image)
metadata['sha512'] = create_sha512(decoded_image)
try:
resp = col.insert_one(metadata)
metadata['identifier'] = identifier
metadata['location'] = [metadata['longitude'], metadata['latitude']]
col.insert_one(metadata)
except:
print("Could not insert Metadata")
decoded_image = WrapperService.unwrap_file(b64encoded_image)
if not DropboxService.create_file(filename, decoded_image):
print("Could not save image to dropbox")
if not MinioService.create_file(filename, decoded_image):
print("Could not save image to minio")
return JsonResponse({'id': identifier, 'filename': filename},
safe=False) # 'metadata': metadata response beinhaltet ObjectId welche nicht serializable is?
@staticmethod
@api_view(['DELETE'])
def image_api_delete(request: HttpRequest, identifier):
logger.debug('Image DELETE single call: {}'.format(request))
instance = MongoManager.getInstance()
db = instance.AIC
col = db.metadata
try:
col.delete_one({"identifier": identifier})
except:
print("Could not delete Metadata")
return JsonResponse({'Result': 'success1'}, safe=False)