Add SP list trigger based on lastModified

python-lib/sharepoint_client.py

@@ -7,6 +7,7 @@
 import time
 import json
 import re
+import calendar
 
 from xml.etree.ElementTree import Element, tostring
 from xml.dom import minidom
@@ -421,6 +422,40 @@ def get_list_items(self, list_title, params=None):
         self.assert_response_ok(response, calling_method="get_list_items")
         return response.json().get("ListData", {})
 
+    def get_list_last_modified(self, list_title, params=None):
+        params = params or {}
+        data = {
+            "parameters": {
+                "__metadata": {
+                    "type": "SP.RenderListDataParameters"
+                },
+                "RenderOptions": SharePointConstants.RENDER_OPTIONS,
+                "AllowMultipleValueFilterForTaxonomyFields": True,
+                "AddRequiredFields": True
+            }
+        }
+        headers = DSSConstants.JSON_HEADERS
+        response = self.session.post(
+            self.get_list_data_as_stream(list_title),
+            params=params,
+            headers=headers,
+            json=data
+        )
+        self.assert_response_ok(response, calling_method="get_list_last_modified")
+        last_item_modified_time = response.json().get("lastItemModifiedTime")
+        last_item_modified_time_epoch = 0
+        if last_item_modified_time:
+            try:
+                last_item_modified_time_epoch = calendar.timegm(
+                    time.strptime(last_item_modified_time, "%Y-%m-%d %H:%M:%SZ")
+                ) * 1000
+            except Exception as error:
+                logger.error("Could not convert lastItemModifiedTime ({}): {}".format(
+                    last_item_modified_time,
+                    error
+                ))
+        return last_item_modified_time_epoch
+
     def create_list(self, list_name):
         headers = DSSConstants.JSON_HEADERS
         data = {

python-triggers/check-list-modified/trigger.json

@@ -0,0 +1,183 @@
+{
+    "meta": {
+        "label": "Check modified list",
+        "description": "Check that a SharePoint list has been modified",
+        "icon": "icon-cloud"
+    },
+    "params": [
+        {
+            "name": "auth_type",
+            "label": "Type of authentication",
+            "type": "SELECT",
+            "selectChoices": [
+                {
+                    "value": "login",
+                    "label": "User name / password (deprecated)"
+                },
+                {
+                    "value": "oauth",
+                    "label": "Azure Single Sign On"
+                },
+                {
+                    "value": "site-app-permissions",
+                    "label": "Site App Permissions"
+                },
+                {
+                    "value": "app-certificate",
+                    "label": "Certificates"
+                },
+                {
+                    "value": "app-username-password",
+                    "label": "User name / password"
+                }
+            ]
+        },
+        {
+            "name": "sharepoint_oauth",
+            "label": "Azure preset",
+            "type": "PRESET",
+            "parameterSetId": "oauth-login",
+            "visibilityCondition": "model.auth_type == 'oauth'"
+        },
+        {
+            "name": "sharepoint_sharepy",
+            "label": "SharePoint preset",
+            "type": "PRESET",
+            "parameterSetId": "sharepoint-login",
+            "visibilityCondition": "model.auth_type == 'login'"
+        },
+        {
+            "name": "site_app_permissions",
+            "label": "Site App preset",
+            "type": "PRESET",
+            "parameterSetId": "site-app-permissions",
+            "visibilityCondition": "model.auth_type == 'site-app-permissions'"
+        },
+        {
+            "name": "app_certificate",
+            "label": "Certificates",
+            "type": "PRESET",
+            "parameterSetId": "app-certificate",
+            "visibilityCondition": "model.auth_type == 'app-certificate'"
+        },
+        {
+            "name": "app_username_password",
+            "label": "App username password",
+            "type": "PRESET",
+            "parameterSetId": "app-username-password",
+            "visibilityCondition": "model.auth_type == 'app-username-password'"
+        },
+        {
+            "name": "sharepoint_list_title",
+            "label": "List title",
+            "defaultValue":  "DSS_${projectKey}_",
+            "description": "",
+            "type": "STRING",
+            "mandatory": true
+        },
+        {
+            "name": "expand_lookup",
+            "label": "Expand lookup fields",
+            "description": "",
+            "type": "BOOLEAN",
+            "defaultValue": false,
+            "visibilityCondition": false,
+            "mandatory": true
+        },
+        {
+            "name": "metadata_to_retrieve",
+            "label": "Metadata to retrieve",
+            "type": "MULTISELECT",
+            "visibilityCondition": false,
+            "selectChoices": [
+                {
+                    "value": "ID",
+                    "label": "Item's ID"
+                },
+                {
+                    "value": "Created",
+                    "label": "Creation date"
+                },
+                {
+                    "value": "Modified",
+                    "label": "Modification date"
+                },
+                {
+                    "value": "Author",
+                    "label": "Created by"
+                },
+                {
+                    "value": "Editor",
+                    "label": "Modified by"
+                }
+            ]
+        },
+        {
+            "name": "advanced_parameters",
+            "label": "Show advanced parameters",
+            "description": "",
+            "type": "BOOLEAN",
+            "defaultValue": false
+        },
+        {
+            "name": "sharepoint_site_overwrite",
+            "label": "Site path preset overwrite",
+            "type": "STRING",
+            "description": "sites/site_name/subsite...",
+            "visibilityCondition": "model.advanced_parameters == true"
+        },
+        {
+            "name": "sharepoint_list_view_title",
+            "label": "View name",
+            "description": "Read data from a specific view",
+            "type": "STRING",
+            "defaultValue": "",
+            "visibilityCondition": "model.advanced_parameters == true"
+        },
+        {
+            "name": "write_mode",
+            "label": "Write mode",
+            "type": "SELECT",
+            "defaultValue": "create",
+            "selectChoices": [
+                {
+                    "value": "create",
+                    "label": "Create a new list"
+                },
+                {
+                    "value": "append",
+                    "label": "Append to existing list"
+                }
+            ],
+            "visibilityCondition": false
+        },
+        {
+            "name": "max_workers",
+            "label": "Max nb of workers (write mode only)",
+            "description": "More workers will speed writing but also randomize items order",
+            "visibilityCondition": "model.advanced_parameters == true",
+            "type": "INT",
+            "defaultValue": 1,
+            "minI": 1,
+            "maxI": 5
+        },
+        {
+            "name": "batch_size",
+            "label": "Batch size (write mode only)",
+            "description": "Number of items writen per batch",
+            "visibilityCondition": "model.advanced_parameters == true",
+            "type": "INT",
+            "defaultValue": 100,
+            "minI": 1,
+            "maxI": 100
+        },
+        {
+            "name": "attempt_session_reset_on_403",
+            "label": "Attempt session reset",
+            "description": "Slow, refer to documentation",
+            "type": "BOOLEAN",
+            "defaultValue": false,
+            "visibilityCondition": "model.advanced_parameters == true"
+        }
+    ]
+}

python-triggers/check-list-modified/trigger.py

@@ -0,0 +1,75 @@
+import time
+import logging
+from dataiku.customtrigger import get_plugin_config
+from dataiku.scenario import Trigger
+from sharepoint_client import SharePointClient
+from dss_constants import DSSConstants
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO,
+                    format='sharepoint-online plugin %(levelname)s - %(message)s')
+
+
+class ProjectVariable():
+    def __init__(self, variable_name, variable_type=None, default_value=None):
+        from dataiku import Project
+        self.variable_name = variable_name
+        self.project = Project()
+        self.variable_type = variable_type or "standard"
+        self.default_value = default_value
+
+    def is_not_set(self):
+        project_variables = self.project.get_variables()
+        project_variable = project_variables.get(self.variable_type, {}).get(self.variable_name)
+        if project_variable is None:
+            return True
+        return False
+
+    def get_value(self):
+        project_variables = self.project.get_variables()
+        project_variable = project_variables.get(self.variable_type, {}).get(self.variable_name, self.default_value)
+        return project_variable
+
+    def set_value(self, value):
+        project_variables = self.project.get_variables()
+        project_variables[self.variable_type][self.variable_name] = value
+        self.project.set_variables(project_variables)
+
+
+def pretty_epoch(epoch_time):
+    return time.strftime('%Y-%m-%d %H:%M:%S%z', time.gmtime(epoch_time/1000))
+
+
+logger.info('SharePoint Online plugin list trigger v{}'.format(DSSConstants.PLUGIN_VERSION))
+plugin_config = get_plugin_config()
+config = plugin_config.get("config", {})
+sharepoint_list_title = config.get("sharepoint_list_title")
+
+trigger = Trigger()
+project_variable_name = "sharepoint-online-list-trigger_{}".format(sharepoint_list_title)
+last_modified = ProjectVariable(project_variable_name, default_value=0)
+client = SharePointClient(config)
+remote_file_last_modified_epoch = client.get_list_last_modified(
+    sharepoint_list_title
+)
+last_modified_epoch = last_modified.get_value()
+logger.info("Trigger.{}.lastLocalTime: {} ({})".format(
+        project_variable_name,
+        last_modified_epoch,
+        pretty_epoch(last_modified_epoch)
+    )
+)
+logger.info("Trigger.{}.remoteTime: {} ({})".format(
+        project_variable_name,
+        remote_file_last_modified_epoch,
+        pretty_epoch(remote_file_last_modified_epoch)
+    )
+)
+if last_modified.is_not_set() or (remote_file_last_modified_epoch > last_modified_epoch):
+    logger.info("remote epoch {} > local epoch {}, firing the trigger".format(remote_file_last_modified_epoch, last_modified_epoch))
+    remote_file_last_modified_epoch = int(time.time()) * 1000
+    last_modified.set_value(remote_file_last_modified_epoch)
+    trigger.fire()
+else:
+    logger.info("Remote spreadsheet has not been modified")