From e1a1cb25fc42c892535c40e91f8ee4b5c295b740 Mon Sep 17 00:00:00 2001 From: Manuel Hude Date: Mon, 30 Nov 2020 12:16:52 +0100 Subject: [PATCH 01/10] added GEODATA functionality --- middleware/app_be/views/mongo_db.py | 3 ++- middleware/app_be/views/rest_api.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/middleware/app_be/views/mongo_db.py b/middleware/app_be/views/mongo_db.py index 5980813..01a2b4b 100644 --- a/middleware/app_be/views/mongo_db.py +++ b/middleware/app_be/views/mongo_db.py @@ -16,4 +16,5 @@ class MongoManager: MongoManager.__instance = pymongo.MongoClient('127.0.0.1', 27017) db = MongoManager.__instance.AIC coll = db.metadata - coll.create_index("filename",unique=True) \ No newline at end of file + coll.create_index("filename",unique=True) + coll.create_index([("loc", pymongo.GEO2D)]) \ No newline at end of file diff --git a/middleware/app_be/views/rest_api.py b/middleware/app_be/views/rest_api.py index 54fc50b..4378c9b 100644 --- a/middleware/app_be/views/rest_api.py +++ b/middleware/app_be/views/rest_api.py @@ -94,7 +94,9 @@ class ImageEndpoint: col = db.metadata try: - resp = col.insert_one(metadata) + metadata['location'] = [metadata['longitude'],metadata['latitude']] + col.insert_one(metadata) + except: print("Could not insert Metadata") From 2e58c64cedc9c7501f5088aa26000479a880e42e Mon Sep 17 00:00:00 2001 From: Manuel Hude Date: Mon, 30 Nov 2020 12:47:32 +0100 Subject: [PATCH 02/10] new identifier and added creation folder logic --- iotclient/iot_client.py | 7 ++++++- middleware/app_be/views/mongo_db.py | 6 ++++-- middleware/app_be/views/rest_api.py | 3 ++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/iotclient/iot_client.py b/iotclient/iot_client.py index 29aa747..eba5fce 100644 --- a/iotclient/iot_client.py +++ b/iotclient/iot_client.py @@ -54,7 +54,12 @@ def get_image(identifier): b64encoded_image = unwrap_file(payload['image_data']) goal_folder_name = "images_fetch" path = "../../"+goal_folder_name+"/"+payload['metadata']['filename'] - goal_folder_path = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[:-3]) + os.path.sep + goal_folder_name + + goal_folder_path = os.path.sep.join(os.path.abspath(__file__).split(os.path.sep)[:-3]) + os.path.sep + goal_folder_name + + if not os.path.isdir(goal_folder_path): + os.mkdir(goal_folder_path) + out = open(path, 'wb') out.write(b64encoded_image) out.close() diff --git a/middleware/app_be/views/mongo_db.py b/middleware/app_be/views/mongo_db.py index 01a2b4b..3326c1f 100644 --- a/middleware/app_be/views/mongo_db.py +++ b/middleware/app_be/views/mongo_db.py @@ -16,5 +16,7 @@ class MongoManager: MongoManager.__instance = pymongo.MongoClient('127.0.0.1', 27017) db = MongoManager.__instance.AIC coll = db.metadata - coll.create_index("filename",unique=True) - coll.create_index([("loc", pymongo.GEO2D)]) \ No newline at end of file + db.coll["identifier"] + db.coll["location"] + coll.create_index("identifier",unique=True) + coll.create_index([("location", pymongo.GEO2D)]) \ No newline at end of file diff --git a/middleware/app_be/views/rest_api.py b/middleware/app_be/views/rest_api.py index 4378c9b..41616f0 100644 --- a/middleware/app_be/views/rest_api.py +++ b/middleware/app_be/views/rest_api.py @@ -55,7 +55,7 @@ class ImageEndpoint: col = db.metadata metadata = None try: - resp = col.find({"filename": identifier+".jpg"}) + resp = col.find({"identifier": identifier}) metadata = resp[0] except: print("Could not find Metadata") @@ -94,6 +94,7 @@ class ImageEndpoint: col = db.metadata try: + metadata['identifier'] = identifier metadata['location'] = [metadata['longitude'],metadata['latitude']] col.insert_one(metadata) From d7e2bf651b62a2a9174e2a766557052c17439047 Mon Sep 17 00:00:00 2001 From: Manuel Hude Date: Mon, 30 Nov 2020 14:23:32 +0100 Subject: [PATCH 03/10] delete functionality for metadata --- iotclient/iot_client.py | 28 ++++++++++++++++++++++++++++ middleware/app_be/urls.py | 1 + middleware/app_be/views/rest_api.py | 18 ++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/iotclient/iot_client.py b/iotclient/iot_client.py index eba5fce..d33050d 100644 --- a/iotclient/iot_client.py +++ b/iotclient/iot_client.py @@ -69,6 +69,21 @@ def get_image(identifier): return True +def delete_image(identifier): + print("Deleting image with identifier "+identifier) + + baseurl = "http://127.0.0.1:8000" + get_url = "/image/delete/"+identifier + + try: + response = requests.delete(baseurl + get_url) + payload = response.json() + print(payload) + except os.error: + print("Error deleting request") + return True + + def get_all(): print("Getting all images") @@ -132,6 +147,7 @@ while (command.lower() not in ["exit", "quit", "end"]): print("trigger - next image in line is sent to backend") print("fetch - gets the next image from database if exists") print("fetchall - get all images") + print("delete - delets the next image from db if exists") print("rapid <#amount> - next images in line are sent to backend in 2 second intervals") print("skip [<#amount>] - skips the next number of images in line or 1 if no is given") print("set <#index> - sets image on given as next in line") @@ -227,6 +243,18 @@ while (command.lower() not in ["exit", "quit", "end"]): get_all() index = index + 1 + if command.lower() == "delete": + if metadata is None: + print("No metadata loaded") + continue + if image_folder is None: + print("No image folder selected") + continue + meta_payload = metadata[index] + filename = meta_payload['filename'] + delete_image(filename[:-4]) + index = index + 1 + if command.lower() == "rapid": if metadata is None: print("No metadata loaded") diff --git a/middleware/app_be/urls.py b/middleware/app_be/urls.py index b727930..1896baa 100644 --- a/middleware/app_be/urls.py +++ b/middleware/app_be/urls.py @@ -25,6 +25,7 @@ urlpatterns = [ url(r'^test/', TestApiClass.test_api), url(r'^image/get/all$', ImageEndpoint.image_api_get_all), url(r'^image/get/(?P[\w-]+)$', ImageEndpoint.image_api_get_single), + url(r'^image/delete/(?P[\w-]+)$', ImageEndpoint.image_api_delete), url(r'^image/post$', ImageEndpoint.image_api_post) ] diff --git a/middleware/app_be/views/rest_api.py b/middleware/app_be/views/rest_api.py index 41616f0..b65a0b0 100644 --- a/middleware/app_be/views/rest_api.py +++ b/middleware/app_be/views/rest_api.py @@ -107,3 +107,21 @@ class ImageEndpoint: return JsonResponse({'id': identifier, 'filename': filename}, safe=False) # 'metadata': metadata response beinhaltet ObjectId welche nicht serializable is? + + @staticmethod + @api_view(['DELETE']) + def image_api_delete(request: HttpRequest, identifier): + + logger.debug('Image DELETE single call: {}'.format(request)) + + instance = MongoManager.getInstance() + + db = instance.AIC + col = db.metadata + + try: + col.delete_one({"identifier": identifier}) + except: + print("Could not delete Metadata") + + return JsonResponse({'Result': 'success1'}, safe=False) \ No newline at end of file From 19bb2c01fb931c3db639a629c864eef73bb92607 Mon Sep 17 00:00:00 2001 From: Marco Zeisler Date: Mon, 30 Nov 2020 18:51:46 +0100 Subject: [PATCH 04/10] add image sha512 hash to mongo on post; --- middleware/app_be/views/mongo_db.py | 6 ++++-- middleware/app_be/views/rest_api.py | 17 +++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/middleware/app_be/views/mongo_db.py b/middleware/app_be/views/mongo_db.py index 3326c1f..ce5c8e8 100644 --- a/middleware/app_be/views/mongo_db.py +++ b/middleware/app_be/views/mongo_db.py @@ -1,5 +1,6 @@ import pymongo + class MongoManager: __instance = None @@ -17,6 +18,7 @@ class MongoManager: db = MongoManager.__instance.AIC coll = db.metadata db.coll["identifier"] + db.coll["sha512"] db.coll["location"] - coll.create_index("identifier",unique=True) - coll.create_index([("location", pymongo.GEO2D)]) \ No newline at end of file + coll.create_index("identifier", unique=True) + coll.create_index([("location", pymongo.GEO2D)]) diff --git a/middleware/app_be/views/rest_api.py b/middleware/app_be/views/rest_api.py index b65a0b0..414ec7b 100644 --- a/middleware/app_be/views/rest_api.py +++ b/middleware/app_be/views/rest_api.py @@ -1,6 +1,6 @@ +import hashlib import logging import json -import base64 from app_be.services.dropboxservice import DropboxService from app_be.services.wrapperservice import WrapperService @@ -43,7 +43,6 @@ class ImageEndpoint: return JsonResponse({'Result': 'success1'}, safe=False) - @staticmethod @api_view(['GET']) def image_api_get_single(request, identifier): @@ -68,7 +67,8 @@ class ImageEndpoint: dropbox_image_bytes = DropboxService.read_file(metadata['filename']) if dropbox_image_bytes is None: - return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404, safe=False) + return JsonResponse({'Result': 'Error - could not find image in dropbox.', 'id': identifier}, status=404, + safe=False) payload = { 'id': identifier, @@ -93,15 +93,20 @@ class ImageEndpoint: db = instance.AIC col = db.metadata + decoded_image = WrapperService.unwrap_file(b64encoded_image) + h = hashlib.sha512() + h.update(decoded_image) + metadata['sha512'] = h.hexdigest() + try: metadata['identifier'] = identifier - metadata['location'] = [metadata['longitude'],metadata['latitude']] + metadata['location'] = [metadata['longitude'], metadata['latitude']] col.insert_one(metadata) except: print("Could not insert Metadata") - decoded_image = WrapperService.unwrap_file(b64encoded_image) + if not DropboxService.create_file(filename, decoded_image): print("Could not save image to dropbox") @@ -124,4 +129,4 @@ class ImageEndpoint: except: print("Could not delete Metadata") - return JsonResponse({'Result': 'success1'}, safe=False) \ No newline at end of file + return JsonResponse({'Result': 'success1'}, safe=False) From 3ea437c67095d7c891234ddb30c0cad191a3d167 Mon Sep 17 00:00:00 2001 From: Marco Zeisler Date: Mon, 30 Nov 2020 18:52:04 +0100 Subject: [PATCH 05/10] remove redundant utf-8; --- middleware/app_be/services/wrapperservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/app_be/services/wrapperservice.py b/middleware/app_be/services/wrapperservice.py index bae55f6..8bddd3a 100644 --- a/middleware/app_be/services/wrapperservice.py +++ b/middleware/app_be/services/wrapperservice.py @@ -6,7 +6,7 @@ class WrapperService: @staticmethod def wrap_file(file_bytes: bytes) -> str: file_encoded_b64 = base64.b64encode(file_bytes) - return file_encoded_b64.decode('utf-8') + return file_encoded_b64.decode() @staticmethod def unwrap_file(file_str: str) -> bytes: From 8748611fa6d548d0bee26a10bc0177a34ea29651 Mon Sep 17 00:00:00 2001 From: Marco Zeisler Date: Mon, 30 Nov 2020 18:57:57 +0100 Subject: [PATCH 06/10] update; --- middleware/.run/createsuperuser.run.xml | 4 ++-- middleware/.run/makemigrations.run.xml | 4 ++-- middleware/.run/migrate.run.xml | 4 ++-- middleware/.run/runserver.run.xml | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/middleware/.run/createsuperuser.run.xml b/middleware/.run/createsuperuser.run.xml index 5fc8ae4..b5b43dd 100644 --- a/middleware/.run/createsuperuser.run.xml +++ b/middleware/.run/createsuperuser.run.xml @@ -1,6 +1,6 @@ - +