Merge branch '4-mongodb_api'

This commit is contained in:
Manuel Hude 2020-11-28 20:35:41 +01:00
commit 3fcceb91a1
29 changed files with 601 additions and 12 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 74 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 160 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 183 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 182 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 82 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 108 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 122 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 79 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 66 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 89 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 144 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 151 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 46 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 182 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 165 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 80 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 75 KiB

287
iotclient/iot_client.py Normal file
View File

@ -0,0 +1,287 @@
import os
import base64
import requests
import json
import time
from datetime import datetime
def print_response(response):
print("Code: " + str(response.status_code) + "; Body: " + str(response.json()))
def send_image(identifier, image_path, metadata_payload):
print("Sending image with identifier " + identifier)
if not os.path.isfile(image_path):
print("No image found at path " + image_path)
return False
file_encoded_b64 = None
with open(image_path, "rb") as file_to_send:
file_encoded_b64 = base64.b64encode(file_to_send.read())
if file_encoded_b64 is None:
print("Error reading file")
return False
file_payload = file_encoded_b64.decode('utf-8')
baseurl = "http://127.0.0.1:8000"
post_url = "/image/post"
body = {
'id': identifier,
'metadata': metadata_payload,
'image_data': file_payload
}
try:
response = requests.post(baseurl + post_url, json=body)
print_response(response)
except os.error:
print("Error sending request")
return True
def get_image(identifier):
print("Getting image with identifier "+identifier)
baseurl = "http://127.0.0.1:8000"
get_url = "/image/get?id="+identifier
try:
response = requests.get(baseurl + get_url)
print_response(response)
except os.error:
print("Error sending request")
return True
def get_all():
print("Getting all images")
baseurl = "http://127.0.0.1:8000"
get_url = "/image/get"
try:
response = requests.get(baseurl + get_url)
print_response(response)
except os.error:
print("Error sending request")
return True
metadata = None
index = 0
metadata_folder = "." + os.path.sep
metadata_file = "metadata_short" + ".json"
metadata_path = metadata_folder + os.path.sep + metadata_file
image_folder = "." + os.path.sep + "images"
if (os.path.isfile(metadata_path)):
print("Loading metadata to memory from " + metadata_path)
metadata = sorted(json.load(open(metadata_file, "r")), key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
else:
print("Default metadata_file not found.")
if (os.path.isdir(image_folder)):
print("Image folder set to " + image_folder)
else:
print("Default image folder not found.")
#main input loop
command = "dummy"
while (command.lower() not in ["exit", "quit", "end"]):
try:
command = input("Please enter command: ")
except:
print()
print("Error while reading keyboard input")
break
command_split = command.split(" ")
command = command_split[0]
attributes = command_split[1:]
if command.lower() == "help" or command == "?":
print()
print("List of commands:")
print("exit - exits the program")
print("help - lists all commands")
print("dir - list current selected image folder")
print("len - length of current metadata set")
print("index - display index of current lined up image")
print("info - display info of current lined up image")
print("metadata <filepath> - load new metadata file from path")
print("imagefolder <path> - selected new image folder")
print("show - opens image next in line in default os program")
print("trigger - next image in line is sent to backend")
print("fetch - gets the next image from database if exists")
print("fetchall - get all images")
print("rapid <#amount> - next <amound> images in line are sent to backend in 2 second intervals")
print("skip [<#amount>] - skips the next <amount> number of images in line or 1 if no <amount> is given")
print("set <#index> - sets image on given <index> as next in line")
print("select |<identifier> - select image with given <identifier> as next in line")
print()
print()
if command.lower() == "dir":
print("Current image folder is " + image_folder)
if command.lower() == "len":
if metadata is None:
print("No metadata selected")
else:
print("Length of metadata list " + str(len(metadata)))
if command.lower() == "index":
print("Current lineup index is " + str(index))
if command.lower() == "info":
if metadata is None:
print("No metadata selected")
else:
if (index < len(metadata)):
print("Metadata on index " + str(index))
print(metadata[index])
if command.lower() == "metadata":
if (len(attributes) < 1):
print("Error: No path supplied")
continue
path = attributes[0]
if not os.path.isfile(path):
print("Error: No file on path " + path)
continue
print("Loading metadata to memory from " + path)
metadata = sorted(json.load(open(metadata_file, "r")), key=lambda k: datetime.strptime(k['datetime'], '%d-%b-%Y (%H:%M:%S.%f)'))
index = 0
if command.lower() == "imagefolder":
if (len(attributes) < 1):
print("Error: No path supplied")
continue
path = attributes[0]
if not os.path.isdir(path):
print("Error: Path is no valid directory " + path)
continue
print("New image folder selected " + path)
if command.lower() == "show":
if metadata is None:
print("No metadata loaded")
continue
if image_folder is None:
print("No image folder selected")
continue
meta_payload = metadata[index]
filename = meta_payload['filename']
os.system(image_folder + os.path.sep + filename)
if command.lower() == "trigger":
if metadata is None:
print("No metadata loaded")
continue
if image_folder is None:
print("No image folder selected")
continue
meta_payload = metadata[index]
filename = meta_payload['filename']
send_image(filename[:-4], image_folder + os.path.sep + filename, meta_payload)
index = index + 1
if command.lower() == "fetch":
if metadata is None:
print("No metadata loaded")
continue
if image_folder is None:
print("No image folder selected")
continue
meta_payload = metadata[index]
filename = meta_payload['filename']
get_image(filename[:-4])
index = index + 1
if command.lower() == "fetchall":
if metadata is None:
print("No metadata loaded")
continue
if image_folder is None:
print("No image folder selected")
continue
get_all()
index = index + 1
if command.lower() == "rapid":
if metadata is None:
print("No metadata loaded")
continue
if image_folder is None:
print("No image folder selected")
continue
if (len(attributes) < 1):
print("Error: No amount supplied")
continue
if not attributes[0].isnumeric():
print("Error: amount is no number")
continue
amount = int(attributes[0])
for i in range(0,amount):
meta_payload = metadata[index]
filename = meta_payload['filename']
send_image(filename[:-4], image_folder + os.path.sep + filename, meta_payload)
index = index + 1
if index >= len(metadata) or index < 0:
index = 0
try:
time.sleep(2)
except:
print("Error on sleep")
if command.lower() == "skip":
if metadata is None:
print("Please first select metadata")
continue
if (len(attributes) < 1):
index = index + 1
else:
amount = attributes[0]
if amount.isnumeric():
index = index + int(amount)
if index >= len(metadata) or index < 0:
index = 0
if command.lower() == "set":
if metadata is None:
print("Please first select metadata")
continue
if (len(attributes) < 1):
print("Error: No index supplied")
continue
if not attributes[0].isnumeric():
print("Index is no number")
continue
ind = int(attributes[0])
if ind >= len(metadata) or ind < 0:
print("Index outside length of metadata")
continue
index = ind
if command.lower() == "select":
if metadata is None:
print("Please first select metadata")
continue
if (len(attributes) < 1):
print("Error: No identifier supplied")
continue
identifier = attributes[0]
found = False
for i, elem in enumerate(metadata):
if elem['filename'][:-4] == identifier:
index = i
print("Found image at index " + str(i))
found = True
break
if not found:
print("Could not find image by identifier " + identifier)
quit("Bye")

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,230 @@
[
{
"place_ident":"Macon Inc7B",
"name":"MACON INCIDENTAL 7B",
"seq_id":"6f9e088a-2e32-11e9-ab9e-dca9047ef277",
"longitude":-114.57247057156248,
"datetime":"19-Apr-2019 (00:27:53.000000)",
"frame_num":2,
"seq_num_frames":3,
"latitude":43.14244151487849,
"filename":"bbe75cb2-2c0f-11e9-bcad-06f10d5896c4.jpg",
"device_id":"87781718-e693-471e-aaca-aed81d77cc0c"
},
{
"place_ident":"031909WC03",
"name":"HILL PASTURE 3",
"seq_id":"6f85d617-2e32-11e9-bde0-dca9047ef277",
"longitude":-116.00210150434704,
"datetime":"14-Apr-2019 (21:10:49.000000)",
"frame_num":3,
"seq_num_frames":3,
"latitude":42.55575800290714,
"filename":"9f0c5c33-2c02-11e9-bcad-06f10d5896c4.jpg",
"device_id":"2e473b72-1ac3-4b4e-9e53-b45043cbed97"
},
{
"place_ident":"ticeska06",
"name":"TICESKA 6",
"seq_id":"6fd7dc35-2e32-11e9-a6bc-dca9047ef277",
"longitude":-115.03066519770705,
"datetime":"16-Apr-2019 (05:19:20.000000)",
"frame_num":5,
"seq_num_frames":5,
"latitude":42.896085756920634,
"filename":"442f65cc-2c1d-11e9-bcad-06f10d5896c4.jpg",
"device_id":"2771a46b-f4c9-408d-aaf5-eb0de438fff0"
},
{
"place_ident":"mammoth02",
"name":"MAMMOTH CAVE 2",
"seq_id":"6fc9a8c7-2e32-11e9-a3de-dca9047ef277",
"longitude":-114.44452539296739,
"datetime":"18-Apr-2019 (14:20:52.000000)",
"frame_num":4,
"seq_num_frames":10,
"latitude":43.05439627180452,
"filename":"bac2c873-2c1b-11e9-bcad-06f10d5896c4.jpg",
"device_id":"71edb5c2-8eb2-4c60-a016-16e1f050f47c"
},
{
"place_ident":"grasres02",
"name":"GRASMERE RESERVOIR 5",
"seq_id":"6faaa217-2e32-11e9-b389-dca9047ef277",
"longitude":-115.97287500807049,
"datetime":"17-Apr-2019 (16:11:15.000000)",
"frame_num":3,
"seq_num_frames":3,
"latitude":42.34494360401168,
"filename":"cfea1276-2c16-11e9-bcad-06f10d5896c4.jpg",
"device_id":"a3838526-0a32-4da5-9a06-9d25946fdbe6"
},
{
"place_ident":"031909GRAS02",
"name":"GRASMERE 2",
"seq_id":"6fe8bc23-2e32-11e9-ae7a-dca9047ef277",
"longitude":-115.92689233439906,
"datetime":"15-Apr-2019 (10:34:33.000000)",
"frame_num":4,
"seq_num_frames":5,
"latitude":42.41793686970267,
"filename":"fa92e5e0-2c21-11e9-bcad-06f10d5896c4.jpg",
"device_id":"4d0ac2cc-ad27-490e-98dd-36a505f45ed7"
},
{
"place_ident":"ticeska06",
"name":"TICESKA 6",
"seq_id":"6fc5e51c-2e32-11e9-a5ef-dca9047ef277",
"longitude":-115.03066519770705,
"datetime":"18-Apr-2019 (22:51:02.000000)",
"frame_num":1,
"seq_num_frames":1,
"latitude":42.896085756920634,
"filename":"c6136c00-2c1d-11e9-bcad-06f10d5896c4.jpg",
"device_id":"2771a46b-f4c9-408d-aaf5-eb0de438fff0"
},
{
"place_ident":"031709CRSE05",
"name":"CRATER RINGS SE 5",
"seq_id":"6ec97eb3-2e32-11e9-bb0a-dca9047ef277",
"longitude":-115.75646888788206,
"datetime":"17-Apr-2019 (02:07:28.000000)",
"frame_num":1,
"seq_num_frames":1,
"latitude":42.97209410431162,
"filename":"6d44c156-2bf9-11e9-bcad-06f10d5896c4.jpg",
"device_id":"88129de7-f3e5-4c09-9c12-ffb293464a37"
},
{
"place_ident":"ticeska02",
"name":"TICESKA 8",
"seq_id":"6fdd0f07-2e32-11e9-bc87-dca9047ef277",
"longitude":-115.0486112620767,
"datetime":"17-Apr-2019 (09:11:41.000000)",
"frame_num":5,
"seq_num_frames":10,
"latitude":42.898455408098776,
"filename":"3c156ef8-2c1e-11e9-bcad-06f10d5896c4.jpg",
"device_id":"9fbc13ff-f5dd-4564-98f1-2307f1528050"
},
{
"place_ident":"031909GRAS01",
"name":"GRASMERE 1",
"seq_id":"6faad6a8-2e32-11e9-a081-dca9047ef277",
"longitude":-115.9043718414795,
"datetime":"13-Apr-2019 (22:21:47.000000)",
"frame_num":2,
"seq_num_frames":11,
"latitude":42.38461654217346,
"filename":"1d60c980-2c17-11e9-bcad-06f10d5896c4.jpg",
"device_id":"b3f129b8-59f2-458f-bf2f-f0c1af0032d3"
},
{
"place_ident":"031809HR04",
"name":"HOLE IN ROCK 4",
"seq_id":"6f75abb0-2e32-11e9-bc23-dca9047ef277",
"longitude":-116.03169405245525,
"datetime":"22-Apr-2019 (23:12:04.000000)",
"frame_num":2,
"seq_num_frames":9,
"latitude":42.60268919509188,
"filename":"4af7020f-2bfa-11e9-bcad-06f10d5896c4.jpg",
"device_id":"8a5081ec-b22d-4322-8e2b-dc76d3ab4fa5"
},
{
"place_ident":"Macon Inc7C",
"name":"MACON INCIDENTAL 7C",
"seq_id":"6ec3afa8-2e32-11e9-b477-dca9047ef277",
"longitude":-114.58253565673935,
"datetime":"18-Apr-2019 (18:36:37.000000)",
"frame_num":1,
"seq_num_frames":10,
"latitude":43.01981809912367,
"filename":"e8da54f2-2bf6-11e9-bcad-06f10d5896c4.jpg",
"device_id":"e1253e22-776a-4924-aadc-ad7cac4c92da"
},
{
"place_ident":"grasres02",
"name":"GRASMERE RESERVOIR 5",
"seq_id":"6f8802eb-2e32-11e9-b65c-dca9047ef277",
"longitude":-115.97287500807049,
"datetime":"21-Apr-2019 (16:36:45.000000)",
"frame_num":9,
"seq_num_frames":12,
"latitude":42.34494360401168,
"filename":"78e255ae-2c06-11e9-bcad-06f10d5896c4.jpg",
"device_id":"a3838526-0a32-4da5-9a06-9d25946fdbe6"
},
{
"place_ident":"Macon Inc7C",
"name":"MACON INCIDENTAL 7C",
"seq_id":"6fe6eea6-2e32-11e9-bff9-dca9047ef277",
"longitude":-114.59497489371012,
"datetime":"21-Apr-2019 (22:30:03.000000)",
"frame_num":4,
"seq_num_frames":5,
"latitude":43.41433084319401,
"filename":"f0b71d84-2c20-11e9-bcad-06f10d5896c4.jpg",
"device_id":"8e5e7190-11fa-47eb-b510-6e31551a6c59"
},
{
"place_ident":"342Cc013",
"name":"MACON 12",
"seq_id":"6fe5fc94-2e32-11e9-ab68-dca9047ef277",
"longitude":-114.58298511215773,
"datetime":"13-Apr-2019 (10:14:32.000000)",
"frame_num":1,
"seq_num_frames":5,
"latitude":43.17674023998888,
"filename":"a3324d47-2c20-11e9-bcad-06f10d5896c4.jpg",
"device_id":"a2e5036c-8c37-41b4-af16-c9a4a3f35b68"
},
{
"place_ident":"Macon Inc7B",
"name":"MACON INCIDENTAL 7B",
"seq_id":"6fa481a6-2e32-11e9-a65f-dca9047ef277",
"longitude":-114.55869957980991,
"datetime":"20-Apr-2019 (14:48:28.000000)",
"frame_num":9,
"seq_num_frames":12,
"latitude":43.01003674972061,
"filename":"ec315c3a-2c11-11e9-bcad-06f10d5896c4.jpg",
"device_id":"4f3ddae4-290c-4104-8583-139ea81e46a0"
},
{
"place_ident":"342Cc016",
"name":"MACON 30",
"seq_id":"6fab05ba-2e32-11e9-bd08-dca9047ef277",
"longitude":-114.56313608407,
"datetime":"19-Apr-2019 (07:34:29.000000)",
"frame_num":2,
"seq_num_frames":2,
"latitude":43.141335018744,
"filename":"54442cb4-2c17-11e9-bcad-06f10d5896c4.jpg",
"device_id":"41944840-499f-46c7-9c78-865a6ecf6be2"
},
{
"place_ident":"031909WC05",
"name":"HILL PASTURE 5",
"seq_id":"6f96857d-2e32-11e9-86e9-dca9047ef277",
"longitude":-116.03815543598549,
"datetime":"18-Apr-2019 (21:20:36.000000)",
"frame_num":11,
"seq_num_frames":12,
"latitude":42.55736106335515,
"filename":"e8ed1de3-2c0a-11e9-bcad-06f10d5896c4.jpg",
"device_id":"b47480ff-630b-48c0-8988-5c7936d68b8a"
},
{
"place_ident":"031909GRAS02",
"name":"GRASMERE 2",
"seq_id":"6f7ff98a-2e32-11e9-8663-dca9047ef277",
"longitude":-115.92689233439906,
"datetime":"17-Apr-2019 (00:37:55.000000)",
"frame_num":4,
"seq_num_frames":6,
"latitude":42.41793686970267,
"filename":"fcdcbf3c-2bfd-11e9-bcad-06f10d5896c4.jpg",
"device_id":"4d0ac2cc-ad27-490e-98dd-36a505f45ed7"
}
]

0
logfile Normal file
View File

View File

@ -1,3 +1,5 @@
from django.db import models
# Create your models here.

View File

@ -11,12 +11,10 @@ import os
# set the websocket routing module location here
ASGI_APPLICATION = 'app_be.routing.application'
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
@ -37,7 +35,7 @@ INSTALLED_APPS = [
'django.contrib.messages',
'django.contrib.staticfiles',
'channels',
'rest_framework'
'rest_framework',
]
MIDDLEWARE = [
@ -49,6 +47,7 @@ MIDDLEWARE = [
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
"""
@ -67,7 +66,6 @@ REST_FRAMEWORK = {
)
}
# configuration of jason web token authentication
JWT_AUTH = {
'JWT_VERIFY': True,
@ -117,15 +115,16 @@ TEMPLATES = [
WSGI_APPLICATION = 'app_be.wsgi.application'
# Database
# https://docs.djangoproject.com/en/2.0/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
AUTH_PASSWORD_VALIDATORS = [
@ -219,4 +218,4 @@ LOGGING = {
# increase the maximum upload size of files
DATA_UPLOAD_MAX_MEMORY_SIZE = 104857600
DROPBOX_OAUTH2_ACCESS_TOKEN = 'sl.AmBKKaIkaH0ZKuk5i60EbN8dIspDugFK2k4ixfMkr3BZ7ZJ0yF5YAj-69mWKO-G_K-ctk_Ai4a6BeOXEZ_kZH4waUUlPhnV3QBe4QfGYN5PQcs-9SwQ5BWQfeQSKw9jkybbV9mjOdQ4'
DROPBOX_OAUTH2_ACCESS_TOKEN = 'SDt1aqMQg7EAAAAAAAAAARV4CNnOSTjYLc05W2YAxIArG93DnaK9Si9VbwE-aBbQ'

View File

@ -24,7 +24,7 @@ urlpatterns = [
path('admin/', admin.site.urls),
url(r'^test/', TestApiClass.test_api),
url(r'^image/get$', ImageEndpoint.image_api_get_all),
url(r'^image/get/(?P<id>[0-9]+)$', ImageEndpoint.image_api_get_single),
url(r'^image/get(?P<id>[-|a-zA-Z0-9]+)$', ImageEndpoint.image_api_get_single),
url(r'^image/post$', ImageEndpoint.image_api_post)
]

View File

@ -0,0 +1,19 @@
import pymongo
class MongoManager:
__instance = None
@staticmethod
def getInstance():
if MongoManager.__instance == None:
MongoManager()
return MongoManager.__instance
def __init__(self):
if MongoManager.__instance != None:
raise Exception("This class is a singleton!")
else:
MongoManager.__instance = pymongo.MongoClient('127.0.0.1', 27017)
db = MongoManager.__instance.AIC
coll = db.metadata
coll.create_index("filename",unique=True)

View File

@ -2,6 +2,9 @@ import logging
import json
import base64
import dropbox
import pymongo
from app_be.views.mongo_db import MongoManager
from django.http import JsonResponse
from django.http import HttpRequest
@ -25,19 +28,67 @@ class ImageEndpoint:
@api_view(['GET'])
def image_api_get_all(request):
logger.debug('Image GET all call: {}'.format(request))
instance = MongoManager.getInstance()
db = instance.AIC
col = db.metadata
print("hier")
try:
for resp in col.find():
print(resp)
except:
print("Could not find Metadata")
return JsonResponse({'Result': 'success1'}, safe=False)
@staticmethod
@api_view(['GET'])
def image_api_get_single(request, id):
logger.debug('Image GET single call: {}'.format(request))
print("im single")
instance = MongoManager.getInstance()
db = instance.AIC
col = db.metadata
try:
resp = col.find({"filename": id+".jpg"})
print(resp[0]);
except:
print("Could not find Metadata")
return JsonResponse({'Result': 'success1', 'id': id}, safe=False)
@staticmethod
@api_view(['POST'])
def image_api_post(request: HttpRequest):
logger.debug('Image POST call: {}'.format(request))
data = json.loads(request.body.decode('utf-8'))
payload = json.loads(request.body.decode('utf-8'))
b64encoded_image = payload['image_data']
filename = payload['id']
metadata = payload['metadata']
instance = MongoManager.getInstance()
db = instance.AIC
col = db.metadata
try:
resp = col.insert_one(metadata)
except:
print("Could not insert Metadata")
decoded_image = base64.b64decode(b64encoded_image)
dbx = dropbox.Dropbox(settings.DROPBOX_OAUTH2_ACCESS_TOKEN)
dbx.files_upload(base64.b64decode(data['image_data']),'/Apps/AIC Federated Storage Infrastructure/test_image_1.jpg')
return JsonResponse({'Result': 'success2', 'received file': data['filename']}, safe=False)
dbx.files_upload(decoded_image, '/Apps/AIC Federated Storage Infrastructure/' + filename)
return JsonResponse({'Result': 'success2', 'received file': filename},
safe=False) # 'metadata': metadata response beinhaltet ObjectId welche nicht serializable is?

View File

@ -18,8 +18,8 @@ setup(
'channels_redis==3.2.0',
'whitenoise==5.2.0',
'Pillow==8.0.1',
'dropbox==10.10.0'
'dropbox==10.10.0',
'pymongo=3.11.1',
],
license='BSD License', # example license
description='DESCRIPTION'