zenon/federated-storage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into '3-minio_api'

Browse Source 
# Conflicts:
#   middleware/app_be/views/rest_api.py
This commit is contained in:
Martin Weick 2020-12-02 10:30:44 +01:00
commit 5dc0ccbde4
10 changed files with 131 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
iotclient/iot_client.py
View File

@ -5,12 +5,15 @@ import json
import time import time
from datetime import datetime from datetime import datetime
def print_response(response): def print_response(response):
print("Code: " + str(response.status_code) + "; Body: " + str(response.json())) print("Code: " + str(response.status_code) + "; Body: " + str(response.json()))
def unwrap_file(file_str: str) -> bytes: def unwrap_file(file_str: str) -> bytes:
return base64.b64decode(file_str) return base64.b64decode(file_str)
def send_image(identifier, image_path, metadata_payload): def send_image(identifier, image_path, metadata_payload):
print("Sending image with identifier " + identifier) print("Sending image with identifier " + identifier)
if not os.path.isfile(image_path): if not os.path.isfile(image_path):
@ -41,10 +44,10 @@ def send_image(identifier, image_path, metadata_payload):
def get_image(identifier): def get_image(identifier):
print("Getting image with identifier "+identifier) print("Getting image with identifier " + identifier)
baseurl = "http://127.0.0.1:8000" baseurl = "http://127.0.0.1:8000"
get_url = "/image/get/"+identifier get_url = "/image/get/" + identifier
try: try:
response = requests.get(baseurl + get_url) response = requests.get(baseurl + get_url)
@ -53,17 +56,39 @@ def get_image(identifier):
print(payload['metadata']) print(payload['metadata'])
b64encoded_image = unwrap_file(payload['image_data']) b64encoded_image = unwrap_file(payload['image_data'])
goal_folder_name = "images_fetch" goal_folder_name = "images_fetch"
path = "../../"+goal_folder_name+"/"+payload['metadata']['filename'] path = "../../" + goal_folder_name + "/" + payload['metadata']['filename']
goal_folder_path = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[:-3]) + os.path.sep + goal_folder_name
goal_folder_path = os.path.sep.join(
os.path.abspath(__file__).split(os.path.sep)[:-3]) + os.path.sep + goal_folder_name
if not os.path.isdir(goal_folder_path):
os.mkdir(goal_folder_path)
out = open(path, 'wb') out = open(path, 'wb')
out.write(b64encoded_image) out.write(b64encoded_image)
out.close() out.close()
os.system(goal_folder_path + os.path.sep + payload['metadata']['filename']) os.system(goal_folder_path + os.path.sep + payload['metadata']['filename'])
except os.error: except os.error as e:
print("Error sending request") print("Error sending request {}".format(e))
except KeyError as e:
print("Key error {}".format(e))
return True return True
def delete_image(identifier):
print("Deleting image with identifier " + identifier)
baseurl = "http://127.0.0.1:8000"
get_url = "/image/delete/" + identifier
try:
response = requests.delete(baseurl + get_url)
payload = response.json()
print(payload)
except os.error:
print("Error deleting request")
return True
def get_all(): def get_all():
print("Getting all images") print("Getting all images")
@ -78,6 +103,7 @@ def get_all():
print("Error sending request") print("Error sending request")
return True return True
metadata = None metadata = None
index = 0 index = 0
metadata_folder = "." + os.path.sep metadata_folder = "." + os.path.sep
@ -86,22 +112,22 @@ metadata_path = metadata_folder + os.path.sep + metadata_file
image_folder = "." + os.path.sep + "images" image_folder = "." + os.path.sep + "images"
if (os.path.isfile(metadata_path)): if (os.path.isfile(metadata_path)):
print("Loading metadata to memory from " + metadata_path) print("Loading metadata to memory from " + metadata_path)
metadata = sorted(json.load(open(metadata_file, "r")), key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)')) metadata = sorted(json.load(open(metadata_file, "r")),
key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
else: else:
print("Default metadata_file not found.") print("Default metadata_file not found.")
if (os.path.isdir(image_folder)): if (os.path.isdir(image_folder)):
print("Image folder set to " + image_folder) print("Image folder set to " + image_folder)
else: else:
print("Default image folder not found.") print("Default image folder not found.")
#main input loop # main input loop
command = "dummy" command = "dummy"
while (command.lower() not in ["exit", "quit", "end"]): while (command.lower() not in ["exit", "quit", "end"]):
try: try:
command = input("Please enter command: ") command = input("Please enter command: ")
except: except:
@ -127,25 +153,26 @@ while (command.lower() not in ["exit", "quit", "end"]):
print("trigger - next image in line is sent to backend") print("trigger - next image in line is sent to backend")
print("fetch - gets the next image from database if exists") print("fetch - gets the next image from database if exists")
print("fetchall - get all images") print("fetchall - get all images")
print("delete - delets the next image from db if exists")
print("rapid <#amount> - next <amound> images in line are sent to backend in 2 second intervals") print("rapid <#amount> - next <amound> images in line are sent to backend in 2 second intervals")
print("skip [<#amount>] - skips the next <amount> number of images in line or 1 if no <amount> is given") print("skip [<#amount>] - skips the next <amount> number of images in line or 1 if no <amount> is given")
print("set <#index> - sets image on given <index> as next in line") print("set <#index> - sets image on given <index> as next in line")
print("select |<identifier> - select image with given <identifier> as next in line") print("select |<identifier> - select image with given <identifier> as next in line")
print() print()
print() print()
if command.lower() == "dir": if command.lower() == "dir":
print("Current image folder is " + image_folder) print("Current image folder is " + image_folder)
if command.lower() == "len": if command.lower() == "len":
if metadata is None: if metadata is None:
print("No metadata selected") print("No metadata selected")
else: else:
print("Length of metadata list " + str(len(metadata))) print("Length of metadata list " + str(len(metadata)))
if command.lower() == "index": if command.lower() == "index":
print("Current lineup index is " + str(index)) print("Current lineup index is " + str(index))
if command.lower() == "info": if command.lower() == "info":
if metadata is None: if metadata is None:
print("No metadata selected") print("No metadata selected")
@ -163,7 +190,8 @@ while (command.lower() not in ["exit", "quit", "end"]):
print("Error: No file on path " + path) print("Error: No file on path " + path)
continue continue
print("Loading metadata to memory from " + path) print("Loading metadata to memory from " + path)
metadata = sorted(json.load(open(metadata_file, "r")), key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)')) metadata = sorted(json.load(open(metadata_file, "r")),
key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
index = 0 index = 0
if command.lower() == "imagefolder": if command.lower() == "imagefolder":
@ -173,13 +201,13 @@ while (command.lower() not in ["exit", "quit", "end"]):
path = attributes[0] path = attributes[0]
if not os.path.isdir(path): if not os.path.isdir(path):
print("Error: Path is no valid directory " + path) print("Error: Path is no valid directory " + path)
continue continue
print("New image folder selected " + path) print("New image folder selected " + path)
if command.lower() == "show": if command.lower() == "show":
if metadata is None: if metadata is None:
print("No metadata loaded") print("No metadata loaded")
continue continue
if image_folder is None: if image_folder is None:
print("No image folder selected") print("No image folder selected")
continue continue
@ -190,7 +218,7 @@ while (command.lower() not in ["exit", "quit", "end"]):
if command.lower() == "trigger": if command.lower() == "trigger":
if metadata is None: if metadata is None:
print("No metadata loaded") print("No metadata loaded")
continue continue
if image_folder is None: if image_folder is None:
print("No image folder selected") print("No image folder selected")
continue continue
@ -211,7 +239,6 @@ while (command.lower() not in ["exit", "quit", "end"]):
get_image(filename[:-4]) get_image(filename[:-4])
index = index + 1 index = index + 1
if command.lower() == "fetchall": if command.lower() == "fetchall":
if metadata is None: if metadata is None:
print("No metadata loaded") print("No metadata loaded")
@ -222,6 +249,18 @@ while (command.lower() not in ["exit", "quit", "end"]):
get_all() get_all()
index = index + 1 index = index + 1
if command.lower() == "delete":
if metadata is None:
print("No metadata loaded")
continue
if image_folder is None:
print("No image folder selected")
continue
meta_payload = metadata[index]
filename = meta_payload['filename']
delete_image(filename[:-4])
index = index + 1
if command.lower() == "rapid": if command.lower() == "rapid":
if metadata is None: if metadata is None:
print("No metadata loaded") print("No metadata loaded")
@ -236,7 +275,7 @@ while (command.lower() not in ["exit", "quit", "end"]):
print("Error: amount is no number") print("Error: amount is no number")
continue continue
amount = int(attributes[0]) amount = int(attributes[0])
for i in range(0,amount): for i in range(0, amount):
meta_payload = metadata[index] meta_payload = metadata[index]
filename = meta_payload['filename'] filename = meta_payload['filename']
send_image(filename[:-4], image_folder + os.path.sep + filename, meta_payload) send_image(filename[:-4], image_folder + os.path.sep + filename, meta_payload)
@ -277,7 +316,6 @@ while (command.lower() not in ["exit", "quit", "end"]):
continue continue
index = ind index = ind
if command.lower() == "select": if command.lower() == "select":
if metadata is None: if metadata is None:
print("Please first select metadata") print("Please first select metadata")
@ -296,4 +334,4 @@ while (command.lower() not in ["exit", "quit", "end"]):
if not found: if not found:
print("Could not find image by identifier " + identifier) print("Could not find image by identifier " + identifier)
quit("Bye") quit("Bye")

4
middleware/.run/createsuperuser.run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager"> <component name="ProjectRunConfigurationManager">
<configuration default="false" name="createsuperuser" type="PythonConfigurationType" factoryName="Python"> <configuration default="false" name="createsuperuser" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" /> <module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" /> <option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" /> <option name="PARENT_ENVS" value="true" />
<envs> <envs>
@ -12,7 +12,7 @@
<option name="ADD_CONTENT_ROOTS" value="true" /> <option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" /> <option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" /> <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" /> <option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="createsuperuser" /> <option name="PARAMETERS" value="createsuperuser" />
<option name="SHOW_COMMAND_LINE" value="false" /> <option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" /> <option name="EMULATE_TERMINAL" value="false" />

4
middleware/.run/makemigrations.run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager"> <component name="ProjectRunConfigurationManager">
<configuration default="false" name="makemigrations" type="PythonConfigurationType" factoryName="Python"> <configuration default="false" name="makemigrations" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" /> <module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" /> <option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" /> <option name="PARENT_ENVS" value="true" />
<envs> <envs>
@ -12,7 +12,7 @@
<option name="ADD_CONTENT_ROOTS" value="true" /> <option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" /> <option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" /> <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" /> <option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="makemigrations app_be" /> <option name="PARAMETERS" value="makemigrations app_be" />
<option name="SHOW_COMMAND_LINE" value="false" /> <option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" /> <option name="EMULATE_TERMINAL" value="false" />

4
middleware/.run/migrate.run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager"> <component name="ProjectRunConfigurationManager">
<configuration default="false" name="migrate" type="PythonConfigurationType" factoryName="Python"> <configuration default="false" name="migrate" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" /> <module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" /> <option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" /> <option name="PARENT_ENVS" value="true" />
<envs> <envs>
@ -12,7 +12,7 @@
<option name="ADD_CONTENT_ROOTS" value="true" /> <option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" /> <option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" /> <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" /> <option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="migrate" /> <option name="PARAMETERS" value="migrate" />
<option name="SHOW_COMMAND_LINE" value="false" /> <option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" /> <option name="EMULATE_TERMINAL" value="false" />

6
middleware/.run/runserver.run.xml
View File

@ -1,18 +1,18 @@
<component name="ProjectRunConfigurationManager"> <component name="ProjectRunConfigurationManager">
<configuration default="false" name="runserver" type="PythonConfigurationType" factoryName="Python"> <configuration default="false" name="runserver" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" /> <module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" /> <option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" /> <option name="PARENT_ENVS" value="true" />
<envs> <envs>
<env name="PYTHONUNBUFFERED" value="1" /> <env name="PYTHONUNBUFFERED" value="1" />
</envs> </envs>
<option name="SDK_HOME" value="" /> <option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" /> <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/middleware" />
<option name="IS_MODULE_SDK" value="true" /> <option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" /> <option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" /> <option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" /> <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" /> <option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="runserver" /> <option name="PARAMETERS" value="runserver" />
<option name="SHOW_COMMAND_LINE" value="false" /> <option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" /> <option name="EMULATE_TERMINAL" value="false" />

7
middleware/app_be/services/hashservice.py Normal file
View File

@ -0,0 +1,7 @@
import hashlib
def create_sha512(b: bytes):
h = hashlib.sha512()
h.update(b)
return h.hexdigest()

2
middleware/app_be/services/wrapperservice.py
View File

@ -6,7 +6,7 @@ class WrapperService:
@staticmethod @staticmethod
def wrap_file(file_bytes: bytes) -> str: def wrap_file(file_bytes: bytes) -> str:
file_encoded_b64 = base64.b64encode(file_bytes) file_encoded_b64 = base64.b64encode(file_bytes)
return file_encoded_b64.decode('utf-8') return file_encoded_b64.decode()
@staticmethod @staticmethod
def unwrap_file(file_str: str) -> bytes: def unwrap_file(file_str: str) -> bytes:

1
middleware/app_be/urls.py
View File

@ -25,6 +25,7 @@ urlpatterns = [
url(r'^test/', TestApiClass.test_api), url(r'^test/', TestApiClass.test_api),
url(r'^image/get/all$', ImageEndpoint.image_api_get_all), url(r'^image/get/all$', ImageEndpoint.image_api_get_all),
url(r'^image/get/(?P<identifier>[\w-]+)$', ImageEndpoint.image_api_get_single), url(r'^image/get/(?P<identifier>[\w-]+)$', ImageEndpoint.image_api_get_single),
url(r'^image/delete/(?P<identifier>[\w-]+)$', ImageEndpoint.image_api_delete),
url(r'^image/post$', ImageEndpoint.image_api_post) url(r'^image/post$', ImageEndpoint.image_api_post)
] ]

7
middleware/app_be/views/mongo_db.py
View File

@ -1,5 +1,6 @@
import pymongo import pymongo
class MongoManager: class MongoManager:
__instance = None __instance = None
@ -16,4 +17,8 @@ class MongoManager:
MongoManager.__instance = pymongo.MongoClient('127.0.0.1', 27017) MongoManager.__instance = pymongo.MongoClient('127.0.0.1', 27017)
db = MongoManager.__instance.AIC db = MongoManager.__instance.AIC
coll = db.metadata coll = db.metadata
coll.create_index("filename",unique=True) db.coll["identifier"]
db.coll["sha512"]
db.coll["location"]
coll.create_index("identifier", unique=True)
coll.create_index([("location", pymongo.GEO2D)])

52
middleware/app_be/views/rest_api.py
View File

@ -1,15 +1,14 @@
import hashlib
import logging import logging
import json import json
import base64
from app_be.services.dropboxservice import DropboxService from app_be.services.dropboxservice import DropboxService
from app_be.services.minioservice import MinioService from app_be.services.minioservice import MinioService
from app_be.services.hashservice import create_sha512
from app_be.services.wrapperservice import WrapperService from app_be.services.wrapperservice import WrapperService
from app_be.views.mongo_db import MongoManager from app_be.views.mongo_db import MongoManager
from django.http import JsonResponse from django.http import JsonResponse
from django.http import HttpRequest from django.http import HttpRequest
from rest_framework.decorators import api_view from rest_framework.decorators import api_view
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -44,7 +43,6 @@ class ImageEndpoint:
return JsonResponse({'Result': 'success1'}, safe=False) return JsonResponse({'Result': 'success1'}, safe=False)
@staticmethod @staticmethod
@api_view(['GET']) @api_view(['GET'])
def image_api_get_single(request, identifier): def image_api_get_single(request, identifier):
@ -56,7 +54,7 @@ class ImageEndpoint:
col = db.metadata col = db.metadata
metadata = None metadata = None
try: try:
resp = col.find({"filename": identifier+".jpg"}) resp = col.find({"identifier": identifier})
metadata = resp[0] metadata = resp[0]
except: except:
print("Could not find Metadata") print("Could not find Metadata")
@ -68,8 +66,23 @@ class ImageEndpoint:
dropbox_image_bytes = DropboxService.read_file(metadata['filename']) dropbox_image_bytes = DropboxService.read_file(metadata['filename'])
#minio_image_bytes = MinioService.read_file(metadata['filename']) #minio_image_bytes = MinioService.read_file(metadata['filename'])
if dropbox_image_bytes is None: if dropbox_image_bytes is None:
return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404, safe=False) return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404,
safe=False)
stored_hash = metadata.get('sha512', '')
actual_dropbox_hash = create_sha512(dropbox_image_bytes)
if stored_hash != actual_dropbox_hash:
return JsonResponse('Stored hash does not match generated one! '
'stored: {} actual: {}'.format(stored_hash, actual_dropbox_hash), safe=False)
# TODO - check hash of MinIO image, too
# actual_minio_hash = '' # create_sha512(minio_image_bytes)
# if stored_hash != actual_minio_hash:
# return JsonResponse('Stored hash does not match generated one! '
# 'stored: {} actual: {}'.format(stored_hash, actual_minio_hash), safe=False)
payload = { payload = {
'id': identifier, 'id': identifier,
@ -94,15 +107,38 @@ class ImageEndpoint:
db = instance.AIC db = instance.AIC
col = db.metadata col = db.metadata
decoded_image = WrapperService.unwrap_file(b64encoded_image)
metadata['sha512'] = create_sha512(decoded_image)
try: try:
resp = col.insert_one(metadata) metadata['identifier'] = identifier
metadata['location'] = [metadata['longitude'], metadata['latitude']]
col.insert_one(metadata)
except: except:
print("Could not insert Metadata") print("Could not insert Metadata")
decoded_image = WrapperService.unwrap_file(b64encoded_image)
if not DropboxService.create_file(filename, decoded_image): if not DropboxService.create_file(filename, decoded_image):
print("Could not save image to dropbox") print("Could not save image to dropbox")
if not MinioService.create_file(filename, decoded_image): if not MinioService.create_file(filename, decoded_image):
print("Could not save image to minio") print("Could not save image to minio")
return JsonResponse({'id': identifier, 'filename': filename}, return JsonResponse({'id': identifier, 'filename': filename},
safe=False) # 'metadata': metadata response beinhaltet ObjectId welche nicht serializable is? safe=False) # 'metadata': metadata response beinhaltet ObjectId welche nicht serializable is?
@staticmethod
@api_view(['DELETE'])
def image_api_delete(request: HttpRequest, identifier):
logger.debug('Image DELETE single call: {}'.format(request))
instance = MongoManager.getInstance()
db = instance.AIC
col = db.metadata
try:
col.delete_one({"identifier": identifier})
except:
print("Could not delete Metadata")
return JsonResponse({'Result': 'success1'}, safe=False)