Upload json tagging data, update upload/download functions to use singular data_access library and configuration.

.gitignore

@@ -118,3 +118,6 @@ tag/*.csv
 
 # TF exported graph files
 .pb
+
+# VSCode
+.vscode/*.json

functions/pipeline/download/__init__.py

@@ -2,39 +2,54 @@
 
 import azure.functions as func
 import json
-import os
 
-from ..shared import db_access as DB_Access
-from ..shared import vott_json_parser as vott_json_parser
+from ..shared.vott_parser import create_starting_vott_json
+from ..shared.db_provider import get_postgres_provider
+from ..shared.db_access import ImageTagDataAccess
+
 
 def main(req: func.HttpRequest) -> func.HttpResponse:
     logging.info('Python HTTP trigger function processed a request.')
 
-    imageCount = req.params.get('imageCount')
+    image_count = int(req.params.get('imageCount'))
+    user_id = int(req.params.get('userId'))
+
     # setup response object
     headers = {
         "content-type": "application/json"
     }
-    if not imageCount:
+    if not user_id:
+        return func.HttpResponse(
+            status_code=401,
+            headers=headers,
+            body=json.dumps({"error": "invalid userId given or omitted"})
+        )
+    elif not image_count:
         return func.HttpResponse(
             status_code=400,
             headers=headers,
             body=json.dumps({"error": "image count not specified"})
         )
     else:
-        # setup response object
-        connection = DB_Access.get_connection()
-        # TODO: images need more meaningful data than just download urls
-        image_urls = DB_Access.get_images_for_tagging(connection, imageCount)
+        try:
+            # DB configuration
+            data_access = ImageTagDataAccess(get_postgres_provider())
 
-        # TODO: Build vott json
-        vott_json = vott_json_parser.create_starting_json(image_urls)
+            image_urls = list(data_access.get_new_images(image_count, user_id))
 
-        return_body_json = {"imageUrls": image_urls, "vottJson": vott_json}
+            # TODO: Populate starting json with tags, if any exist... (precomputed or retagging?)
+            vott_json = create_starting_vott_json(image_urls)
 
-        content = json.dumps(return_body_json)
-        return func.HttpResponse(
-            status_code=200, 
-            headers=headers, 
-            body=content
-        )
+            return_body_json = {"imageUrls": image_urls, "vottJson": vott_json}
+
+            content = json.dumps(return_body_json)
+            return func.HttpResponse(
+                status_code=200,
+                headers=headers,
+                body=content
+            )
+        except Exception as e:
+            return func.HttpResponse(
+                "exception:" + str(e),
+                status_code=500
+            )

functions/pipeline/onboarding/__init__.py

@@ -1,18 +1,15 @@
 import os
 import logging
 import azure.functions as func
-from ..shared import db_access_v2 as DB_Access_V2
-from azure.storage.blob import BlockBlobService, ContentSettings
+
+from ..shared.db_provider import get_postgres_provider
+from ..shared.db_access import ImageTagDataAccess, ImageInfo
+from azure.storage.blob import BlockBlobService
 
 # TODO: User id as param to function - holding off until further discussion
 # regarding whether user ID should be generated/looked up by the CLI or
 # from within this function
 
-default_db_host = ""
-default_db_name = ""
-default_db_user = ""
-default_db_pass = ""
-
 def main(req: func.HttpRequest) -> func.HttpResponse:
     logging.info('Python HTTP trigger function processed a request.')
 
@@ -41,19 +38,17 @@ def main(req: func.HttpRequest) -> func.HttpResponse:
         # Create ImageInfo object (def in db_access.py)
         # Note: For testing, default image height/width are set to 50x50
         # TODO: Figure out where actual height/width need to come from
-        image = DB_Access_V2.ImageInfo(original_filename, url, 50, 50)
+        image = ImageInfo(original_filename, url, 50, 50)
         # Append image object to the list
         image_object_list.append(image)
 
     # TODO: Wrap db access section in try/catch, send an appropriate http response in the event of an error
     logging.info("Now connecting to database...")
-    db_config = DB_Access_V2.DatabaseInfo(os.getenv('DB_HOST', default_db_host), os.getenv('DB_NAME', default_db_name), os.getenv('DB_USER', default_db_user), os.getenv('DB_PASS', default_db_pass))
-    data_access = DB_Access_V2.ImageTagDataAccess(DB_Access_V2.PostGresProvider(db_config))
+    data_access = ImageTagDataAccess(get_postgres_provider())
     logging.info("Connected.")
 
     # Create user id
-    user_id = data_access.create_user(DB_Access_V2.getpass.getuser())
-    logging.info("The user id for '{0}' is {1}".format(DB_Access_V2.getpass.getuser(),user_id))
+    user_id = data_access.create_user("testuser")  # TODO: remove this hardcoding, should be passed in the request.
 
     # Add new images to the database, and retrieve a dictionary ImageId's mapped to ImageUrl's
     image_id_url_map = data_access.add_new_images(image_object_list,user_id)

functions/pipeline/onboarding/onboarding-client.py

@@ -1,41 +1,39 @@
-import requests
-import json
-import pg8000
-
-# The following mock client imitates the CLI during the onboarding scenario for new images.
-# The expectation is that the CLI uploads images to a temporary blob store, then gets a list 
-# of URLs to those images and passes the list to an HTTP trigger function in the format of
-# a JSON string.  The HTTP trigger function creates rows in the database for the images,
-# retrieves the ImageId's for them, and then copies the images, each renamed as "ImageId.extension",
-# into a permanent blob storage container.  The HTTP function returns the list of URLs to
-# the images in permanent blob storage.
-
-print("\nTest client for CLI Onboarding scenario")
-print('-' * 40)
-
-# functionURL = "https://onboardinghttptrigger.azurewebsites.net/api/onboarding?code=lI1zl4IhiHcOcxTS85RsE7yZJXeNRxnr7tXSO1SrLWdpiN0W6hT3Jw=="
-functionURL = "http://localhost:7071/api/onboarding"
-# Sean's function URL:
-# functionURL = "https://onboardinghttptrigger.azurewebsites.net/api/onboarding?code=lI1zl4IhiHcOcxTS85RsE7yZJXeNRxnr7tXSO1SrLWdpiN0W6hT3Jw=="
-# functionURL = "https://abrig-linux-func.azurewebsites.net/api/onboarding"
-
-urlList = { "imageUrls": ["https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies1.jpg",
-                         "https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies2.jpg",
-                         "https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies3.jpg"] }
-
-headers = {"Content-Type": "application/json"}
-
-print("Now executing POST request to onboard images...to:")
-print("Function URL: " + functionURL)
-print("Headers:")
-for key, value in headers.items():
-    print("\t" + key + ": " + value)
-response = requests.post(url=functionURL, headers=headers, json=urlList)
-print("Completed POST request.")
-
-raw_response = response.text
-response_array = raw_response.split(", ")
-response_output = "\n".join(response_array)
-
-print(f"Response status code: {response.status_code}")
-print(f"Response string: {response_output}")
+# import requests
+#
+# # The following mock client imitates the CLI during the onboarding scenario for new images.
+# # The expectation is that the CLI uploads images to a temporary blob store, then gets a list
+# # of URLs to those images and passes the list to an HTTP trigger function in the format of
+# # a JSON string.  The HTTP trigger function creates rows in the database for the images,
+# # retrieves the ImageId's for them, and then copies the images, each renamed as "ImageId.extension",
+# # into a permanent blob storage container.  The HTTP function returns the list of URLs to
+# # the images in permanent blob storage.
+#
+# print("\nTest client for CLI Onboarding scenario")
+# print('-' * 40)
+#
+# # functionURL = "https://onboardinghttptrigger.azurewebsites.net/api/onboarding?code=lI1zl4IhiHcOcxTS85RsE7yZJXeNRxnr7tXSO1SrLWdpiN0W6hT3Jw=="
+# functionURL = "http://localhost:7071/api/onboarding"
+# # Sean's function URL:
+# # functionURL = "https://onboardinghttptrigger.azurewebsites.net/api/onboarding?code=lI1zl4IhiHcOcxTS85RsE7yZJXeNRxnr7tXSO1SrLWdpiN0W6hT3Jw=="
+# # functionURL = "https://abrig-linux-func.azurewebsites.net/api/onboarding"
+#
+# urlList = { "imageUrls": ["https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies1.jpg",
+#                          "https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies2.jpg",
+#                          "https://akaonboardingstorage.blob.core.windows.net/aka-temp-source-container/puppies3.jpg"] }
+#
+# headers = {"Content-Type": "application/json"}
+#
+# print("Now executing POST request to onboard images...to:")
+# print("Function URL: " + functionURL)
+# print("Headers:")
+# for key, value in headers.items():
+#     print("\t" + key + ": " + value)
+# response = requests.post(url=functionURL, headers=headers, json=urlList)
+# print("Completed POST request.")
+#
+# raw_response = response.text
+# response_array = raw_response.split(", ")
+# response_output = "\n".join(response_array)
+#
+# print(f"Response status code: {response.status_code}")
+# print(f"Response string: {response_output}")

functions/pipeline/shared/db_access/__init__.py

@@ -0,0 +1 @@
+from .db_access_v2 import ImageTagDataAccess, ImageTag, ImageInfo, ImageTagState

functions/pipeline/shared/db_access_v2.py → functions/pipeline/shared/db_access/db_access_v2.py

@@ -6,7 +6,8 @@
 from enum import IntEnum, unique
 import getpass
 import itertools
-from .db_provider import DatabaseInfo, PostGresProvider
+
+from ..db_provider import DatabaseInfo, PostGresProvider
 
 @unique
 class ImageTagState(IntEnum):
@@ -76,11 +77,10 @@ def get_new_images(self, number_of_images, user_id):
             conn = self._db_provider.get_connection()
             try:
                 cursor = conn.cursor()
-                # TODO: Should we add TagStateId = INCOMPLETE_TAG also for fetching images?
                 query = ("SELECT b.ImageId, b.ImageLocation, a.TagStateId FROM Image_Tagging_State a "
-                        "JOIN Image_Info b ON a.ImageId = b.ImageId WHERE a.TagStateId = 1 order by "
+                        "JOIN Image_Info b ON a.ImageId = b.ImageId WHERE a.TagStateId IN ({1}, {2}) order by "
                         "a.createddtim DESC limit {0}")
-                cursor.execute(query.format(number_of_images))
+                cursor.execute(query.format(number_of_images, ImageTagState.READY_TO_TAG, ImageTagState.INCOMPLETE_TAG))
                 for row in cursor:
                     print('Image Id: {0} \t\tImage Name: {1} \t\tTag State: {2}'.format(row[0], row[1], row[2]))
                     selected_images_to_tag[str(row[0])] = str(row[1])

functions/pipeline/shared/test_db_access_v2.py → functions/pipeline/shared/db_access/test_db_access_v2.py

@@ -2,7 +2,7 @@
 from unittest.mock import patch 
 from unittest.mock import Mock
 
-from db_access_v2 import(
+from .db_access_v2 import(
     ImageTagDataAccess,
     ArgumentException,
     ImageTagState,

functions/pipeline/shared/db_provider/__init__.py

@@ -0,0 +1 @@
+from .db_provider import DatabaseInfo, DBProvider, PostGresProvider, get_postgres_provider