zenon/federated-storage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into 4-mongodb_api

Browse Source 
# Conflicts:
#	middleware/app_be/views/rest_api.py
This commit is contained in:
Manuel Hude 2020-12-06 14:24:01 +01:00
commit e6f54951e6
13 changed files with 154 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
iotclient/iot_client.py
View File

@ -5,12 +5,15 @@ import json
import time
from datetime import datetime
def print_response(response):
print("Code: " + str(response.status_code) + "; Body: " + str(response.json()))
def unwrap_file(file_str: str) -> bytes:
return base64.b64decode(file_str)
def send_image(identifier, image_path, metadata_payload):
print("Sending image with identifier " + identifier)
if not os.path.isfile(image_path):
@ -41,10 +44,10 @@ def send_image(identifier, image_path, metadata_payload):
def get_image(identifier):
print("Getting image with identifier "+identifier)
print("Getting image with identifier " + identifier)
baseurl = "http://127.0.0.1:8000"
get_url = "/image/get/"+identifier
get_url = "/image/get/" + identifier
try:
response = requests.get(baseurl + get_url)
@ -53,9 +56,10 @@ def get_image(identifier):
print(payload['metadata'])
b64encoded_image = unwrap_file(payload['image_data'])
goal_folder_name = "images_fetch"
path = "../../"+goal_folder_name+"/"+payload['metadata']['filename']
path = "../../" + goal_folder_name + "/" + payload['metadata']['filename']
goal_folder_path = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[:-3]) + os.path.sep + goal_folder_name
goal_folder_path = os.path.sep.join(
os.path.abspath(__file__).split(os.path.sep)[:-3]) + os.path.sep + goal_folder_name
if not os.path.isdir(goal_folder_path):
os.mkdir(goal_folder_path)
@ -64,16 +68,18 @@ def get_image(identifier):
out.write(b64encoded_image)
out.close()
os.system(goal_folder_path + os.path.sep + payload['metadata']['filename'])
except os.error:
print("Error sending request")
except os.error as e:
print("Error sending request {}".format(e))
except KeyError as e:
print("Key error {}".format(e))
return True
def delete_image(identifier):
print("Deleting image with identifier "+identifier)
print("Deleting image with identifier " + identifier)
baseurl = "http://127.0.0.1:8000"
get_url = "/image/delete/"+identifier
get_url = "/image/delete/" + identifier
try:
response = requests.delete(baseurl + get_url)
@ -84,7 +90,6 @@ def delete_image(identifier):
return True
def get_all():
print("Getting all images")
@ -98,6 +103,7 @@ def get_all():
print("Error sending request")
return True
metadata = None
index = 0
metadata_folder = "." + os.path.sep
@ -106,10 +112,10 @@ metadata_path = metadata_folder + os.path.sep + metadata_file
image_folder = "." + os.path.sep + "images"
if (os.path.isfile(metadata_path)):
print("Loading metadata to memory from " + metadata_path)
metadata = sorted(json.load(open(metadata_file, "r")), key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
metadata = sorted(json.load(open(metadata_file, "r")),
key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
else:
print("Default metadata_file not found.")
@ -118,7 +124,7 @@ if (os.path.isdir(image_folder)):
else:
print("Default image folder not found.")
#main input loop
# main input loop
command = "dummy"
while (command.lower() not in ["exit", "quit", "end"]):
@ -184,7 +190,8 @@ while (command.lower() not in ["exit", "quit", "end"]):
print("Error: No file on path " + path)
continue
print("Loading metadata to memory from " + path)
metadata = sorted(json.load(open(metadata_file, "r")), key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
metadata = sorted(json.load(open(metadata_file, "r")),
key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
index = 0
if command.lower() == "imagefolder":
@ -232,7 +239,6 @@ while (command.lower() not in ["exit", "quit", "end"]):
get_image(filename[:-4])
index = index + 1
if command.lower() == "fetchall":
if metadata is None:
print("No metadata loaded")
@ -269,7 +275,7 @@ while (command.lower() not in ["exit", "quit", "end"]):
print("Error: amount is no number")
continue
amount = int(attributes[0])
for i in range(0,amount):
for i in range(0, amount):
meta_payload = metadata[index]
filename = meta_payload['filename']
send_image(filename[:-4], image_folder + os.path.sep + filename, meta_payload)
@ -310,7 +316,6 @@ while (command.lower() not in ["exit", "quit", "end"]):
continue
index = ind
if command.lower() == "select":
if metadata is None:
print("Please first select metadata")

4
middleware/.run/createsuperuser.run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="createsuperuser" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" />
<module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
@ -12,7 +12,7 @@
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="createsuperuser" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />

4
middleware/.run/makemigrations.run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="makemigrations" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" />
<module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
@ -12,7 +12,7 @@
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="makemigrations app_be" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />

4
middleware/.run/migrate.run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="migrate" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" />
<module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
@ -12,7 +12,7 @@
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="migrate" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />

6
middleware/.run/runserver.run.xml
View File

@ -1,18 +1,18 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="runserver" type="PythonConfigurationType" factoryName="Python">
<module name="middleware" />
<module name="AIC" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/middleware" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/manage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/middleware/manage.py" />
<option name="PARAMETERS" value="runserver" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />

7
middleware/app_be/services/hashservice.py Normal file
View File

@ -0,0 +1,7 @@
import hashlib
def create_sha512(b: bytes):
h = hashlib.sha512()
h.update(b)
return h.hexdigest()

41
middleware/app_be/services/minioservice.py Normal file
View File

@ -0,0 +1,41 @@
import requests
from django.conf import settings
class MinioService:
@staticmethod
def create_file(filename: str, file: bytes) -> bool:
"""Create / save (or overwrite) an file on the minio storage.
:param filename: filename
:param file: bytes representation of the file
:return: Whether file was successfully uploaded
"""
try:
url= settings.AWS_HOST + filename
headers = {'Content-Type': 'binary/octet-stream'}
r = requests.put(url,data=file,headers=headers)
if r.status_code == 200:
print("Successfully uploaded a file!")
else:
print("Something went wrong")
except:
return False
return True
@staticmethod
def read_file(filename: str) -> bytes:
"""Read file on the minio storage.
:param filename: filename
:return: bytes representation of the file or None on error
"""
file_bytes: bytes
try:
url = settings.AWS_HOST + filename
response = requests.get(url, stream=True)
file_bytes=response.content
except:
print("Error downloading file from minio")
return file_bytes

2
middleware/app_be/services/wrapperservice.py
View File

@ -6,7 +6,7 @@ class WrapperService:
@staticmethod
def wrap_file(file_bytes: bytes) -> str:
file_encoded_b64 = base64.b64encode(file_bytes)
return file_encoded_b64.decode('utf-8')
return file_encoded_b64.decode()
@staticmethod
def unwrap_file(file_str: str) -> bytes:

2
middleware/app_be/settings.py
View File

@ -220,3 +220,5 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = 104857600
DROPBOX_OAUTH2_ACCESS_TOKEN = 'SDt1aqMQg7EAAAAAAAAAARV4CNnOSTjYLc05W2YAxIArG93DnaK9Si9VbwE-aBbQ'
DROPBOX_IMAGE_FOLDER = '/Apps/AIC Federated Storage Infrastructure/'
AWS_HOST= 'https://aic-fun-bucket-2020.s3.amazonaws.com/'

4
middleware/app_be/views/mongo_db.py
View File

@ -1,5 +1,6 @@
import pymongo
class MongoManager:
__instance = None
@ -17,6 +18,7 @@ class MongoManager:
db = MongoManager.__instance.AIC
coll = db.metadata
db.coll["identifier"]
db.coll["sha512"]
db.coll["location"]
coll.create_index("identifier",unique=True)
coll.create_index("identifier", unique=True)
coll.create_index([("location", pymongo.GEO2D)])

31
middleware/app_be/views/rest_api.py
View File

@ -1,14 +1,16 @@
import hashlib
import logging
import json
import base64
from app_be.services.dropboxservice import DropboxService
from app_be.services.mongodbservice import MongoDBService
from app_be.services.minioservice import MinioService
from app_be.services.hashservice import create_sha512
from app_be.services.wrapperservice import WrapperService
from app_be.views.mongo_db import MongoManager
from django.http import JsonResponse
from django.http import HttpRequest
from rest_framework.decorators import api_view
logger = logging.getLogger(__name__)
@ -35,7 +37,6 @@ class ImageEndpoint:
return JsonResponse({'Result': 'success1'}, safe=False)
@staticmethod
@api_view(['GET'])
def image_api_get_single(request, identifier):
@ -46,9 +47,24 @@ class ImageEndpoint:
print(metadata)
dropbox_image_bytes = DropboxService.read_file(metadata['filename'])
#minio_image_bytes = MinioService.read_file(metadata['filename'])
if dropbox_image_bytes is None:
return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404, safe=False)
return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404,
safe=False)
stored_hash = metadata.get('sha512', '')
actual_dropbox_hash = create_sha512(dropbox_image_bytes)
if stored_hash != actual_dropbox_hash:
return JsonResponse('Stored hash does not match generated one! '
'stored: {} actual: {}'.format(stored_hash, actual_dropbox_hash), safe=False)
# TODO - check hash of MinIO image, too
# actual_minio_hash = '' # create_sha512(minio_image_bytes)
# if stored_hash != actual_minio_hash:
# return JsonResponse('Stored hash does not match generated one! '
# 'stored: {} actual: {}'.format(stored_hash, actual_minio_hash), safe=False)
payload = {
'id': identifier,
@ -69,12 +85,15 @@ class ImageEndpoint:
filename = payload['metadata']['filename']
decoded_image = WrapperService.unwrap_file(b64encoded_image)
metadata['sha512'] = create_sha512(decoded_image)
MongoDBService.createSingle(metadata,identifier)
decoded_image = WrapperService.unwrap_file(b64encoded_image)
if not DropboxService.create_file(filename, decoded_image):
print("Could not save image to dropbox")
if not MinioService.create_file(filename, decoded_image):
print("Could not save image to minio")
return JsonResponse({'id': identifier, 'filename': filename},
safe=False) # 'metadata': metadata response beinhaltet ObjectId welche nicht serializable is?

BIN
minio_upload/mpv-shot0001.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 360 KiB

31
minio_upload/upload.py Normal file
View File

@ -0,0 +1,31 @@
import requests
import shutil
host = 'https://aic-fun-bucket-2020.s3.amazonaws.com/'
remote_filename = 'testing2.jpg'
url = host + remote_filename
local_filename = 'mpv-shot0001.jpg'
headers = {'Content-Type': 'image/jpeg'}
# PUT image named 'mpv-shot0001.jpg' to 'testing2.jpg'
r = requests.put(url, data=open(local_filename, 'rb'), headers=headers)
if r.status_code == 200:
print("Successfully uploaded a file!")
else:
print("Something went wrong")
exit()
r = requests.get(url, stream=True)
if r.status_code == 200:
with open('file_downloaded.jpg', 'wb') as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, f)
print("Successfully downloaded the file and saved it in 'file_downloaded.jpg'")
else:
print("Something went wrong while downloading the file!")
exit()