Triggers

.flake8

@@ -14,6 +14,8 @@ per-file-ignores =
     # supress some docstring requirements in tests
     tests/unit_tests/*.py: D
     tests/unit_tests/**/*.py: D
+    tests/it_tests/*.py: D
+    tests/it_tests/**/*.py: D
     # these files are from OTEL so should use OTEL license.
     */deep/api/types.py: NCF102
     */deep/api/resource/__init__.py: NCF102

.github/workflows/it_tests.yaml

@@ -0,0 +1,26 @@
+name: IT Tests
+
+on:
+    workflow_dispatch:
+    schedule:
+        - cron: '0 0 * * *'
+
+jobs:
+    itTests:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v3
+
+            - run: |
+                  echo "${{ github.event_name }}"
+                  NEW_COMMIT_COUNT=$(git log --oneline --since '24 hours ago' | wc -l)
+                  echo "$NEW_COMMIT_COUNT"
+                  echo "CHANGE_COUNT=$NEW_COMMIT_COUNT" >> $GITHUB_ENV
+
+            - name: Setup Python # Set Python version
+              uses: actions/setup-python@v4
+              with:
+                  python-version: 3.11
+            - name: Run IT Tests
+              if: ${{ github.event.event_name == 'workflow_dispatch' || env.CHANGE_COUNT > 0 }}
+              run: make it-tests

.github/workflows/on_push.yaml

@@ -44,7 +44,6 @@ jobs:
           make coverage
 
   tests:
-
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -62,7 +61,6 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r dev-requirements.txt
-          pip install .
       - name: Test with pytest
         run: pytest tests/unit_tests --doctest-modules --junitxml=junit/test-results-${{ matrix.python-version }}.xml
       - name: Upload pytest test results

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 - **[CHANGE]**: change(build): add doc string check to flake8 [#14](https://github.com/intergral/deep/pull/14) [@Umaaz](https://github.com/Umaaz)
 - **[FEATURE]**: feat(logging): initial implementation of log points [#3](https://github.com/intergral/deep/pull/3) [@Umaaz](https://github.com/Umaaz)
+- **[ENHANCEMENT]**: enhancement(trigger): change tracepoint handling to use triggers [#16](https://github.com/intergral/deep/pull/16) [@Umaaz](https://github.com/Umaaz)
 - **[BUGFIX]**: feat(api): add api function to register tracepoint directly [#8](https://github.com/intergral/deep/pull/8) [@Umaaz](https://github.com/Umaaz)
 
 # 1.0.1 (22/06/2023)

Makefile

@@ -18,9 +18,13 @@ endif
 test:
 	pytest tests/unit_tests
 
+.PHONY: it-test
+it-test:
+	pytest tests/it_tests
+
 .PHONY: coverage
 coverage:
-	pytest tests/unit_tests --cov=deep  --cov-report term --cov-fail-under=77 --cov-report html --cov-branch
+	pytest tests/unit_tests --cov=deep  --cov-report term --cov-fail-under=82 --cov-report html --cov-branch
 
 .PHONY: lint
 lint:
@@ -60,4 +64,7 @@ docs:
 
 .PHONY: clean
 clean:
-	rm -Rf _site docs/apidocs .pytest_cache test/.pytest_cache
+	rm -Rf _site docs/apidocs .pytest_cache test/.pytest_cache
+
+.PHONY: precommit
+precommit: lint tests coverage

deep-python-client.iml

@@ -8,6 +8,7 @@
       <sourceFolder url="file://$MODULE_DIR$/examples/simple-app/src" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/tests" isTestSource="true" />
+      <excludeFolder url="file://$MODULE_DIR$/htmlcov" />
       <excludeFolder url="file://$MODULE_DIR$/venv/lib/python3.10/site-packages/deep" />
     </content>
     <orderEntry type="inheritedJdk" />

dev/test-server/src/test_server/__init__.py

@@ -0,0 +1,17 @@
+#       Copyright (C) 2023  Intergral GmbH
+#
+#      This program is free software: you can redistribute it and/or modify
+#      it under the terms of the GNU Affero General Public License as published by
+#      the Free Software Foundation, either version 3 of the License, or
+#      (at your option) any later version.
+#
+#      This program is distributed in the hope that it will be useful,
+#      but WITHOUT ANY WARRANTY; without even the implied warranty of
+#      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#      GNU Affero General Public License for more details.
+#
+#      You should have received a copy of the GNU Affero General Public License
+#      along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+"""This is a basic example of setting up a GRPC server to consume Deep protobuf messages."""

docs/config/config.md

@@ -4,15 +4,15 @@ A list of the possible config values for the deep python agent. They can be set
 
 Note: When setting as environment variable prefix the key with 'DEEP_'. e.g. DEEP_SERVICE_URL
 
-| Key                   | Default    | Description                                                                                                                                              |
-|-----------------------|------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|
-| SERVICE_URL           | deep:43315 | The url (hostname:port) of the deep service to connect to.                                                                                               |
-| SERVICE_SECURE        | True       | Can be set to False if the service doesn't support secure connections.                                                                                   |
-| LOGGING_CONF          | None       | Can be used to override the python logging config used by the agent.                                                                                     |
-| POLL_TIMER            | 10         | The time (in seconds) of the interval between polls.                                                                                                     |
-| SERVICE_AUTH_PROVIDER | None       | The auth provider to use, each provider can have their own config, see available [auth providers](../auth/providers.md) for details.                     |
-| IN_APP_INCLUDE        | None       | A string of comma (,) seperated values that indicate a package is part of the app.                                                                       |
-| IN_APP_EXCLUDE        | None       | A string of comma (,) seperated values that indicate a package is not part of the app.                                                                   |
+| Key                   | Default    | Description                                                                                                                                                |
+|-----------------------|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| SERVICE_URL           | deep:43315 | The url (hostname:port) of the deep service to connect to.                                                                                                 |
+| SERVICE_SECURE        | True       | Can be set to False if the service doesn't support secure connections.                                                                                     |
+| LOGGING_CONF          | None       | Can be used to override the python logging config used by the agent.                                                                                       |
+| POLL_TIMER            | 10         | The time (in seconds) of the interval between polls.                                                                                                       |
+| SERVICE_AUTH_PROVIDER | None       | The auth provider to use, each provider can have their own config, see available [auth providers](../auth/providers.md) for details.                       |
+| IN_APP_INCLUDE        | None       | A string of comma (,) seperated values that indicate a package is part of the app.                                                                         |
+| IN_APP_EXCLUDE        | None       | A string of comma (,) seperated values that indicate a package is not part of the app.                                                                     |
 | APP_ROOT              | Calculated | This is the root folder in which the application is running. If not set it is calculated as the directory in which the file that calls `Deep.start` is in. |
 
 

docs/index.md

@@ -2,7 +2,7 @@
 This is the python client for Deep, a dynamic monitor and debugging tool.
 
 # Getting started
-You will need to have a running version of the [DEEP server](#) to connect this client to.
+You will need to have a running version of the [DEEP server](https://github.com/intergral/deep) to connect this client to.
 
 ## Install Agent
 To install the python agent just add the dependency 'deep-agent' to your project.
@@ -12,7 +12,7 @@ pip install deep-agent
 ```
 
 ## Setup
-Once installed you need to setup the agent. At the earliest part of the code you should add the following code:
+Once installed you need to set up the agent. At the earliest part of the code you should add the following code:
 
 ```python
 import deep

examples/simple-app-metrics/src/simple-app/base_test.py

@@ -0,0 +1,43 @@
+#       Copyright (C) 2023  Intergral GmbH
+#
+#      This program is free software: you can redistribute it and/or modify
+#      it under the terms of the GNU Affero General Public License as published by
+#      the Free Software Foundation, either version 3 of the License, or
+#      (at your option) any later version.
+#
+#      This program is distributed in the hope that it will be useful,
+#      but WITHOUT ANY WARRANTY; without even the implied warranty of
+#      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#      GNU Affero General Public License for more details.
+#
+#      You should have received a copy of the GNU Affero General Public License
+#      along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""A simple test object for examples."""
+
+import random
+import uuid
+
+
+class BaseTest:
+    """A basic test that is used in examples."""
+
+    def new_id(self):
+        """Create new id."""
+        return str(uuid.uuid4())
+
+    def next_max(self):
+        """Create new random max."""
+        return random.randint(1, 101)
+
+    def make_char_count_map(self, in_str):
+        """Create char count map."""
+        res = {}
+
+        for i in range(0, len(in_str)):
+            c = in_str[i]
+            if c not in res:
+                res[c] = 0
+            else:
+                res[c] = res[c] + 1
+        return res

examples/simple-app-metrics/src/simple-app/main.py

@@ -0,0 +1,91 @@
+#       Copyright (C) 2024  Intergral GmbH
+#
+#      This program is free software: you can redistribute it and/or modify
+#      it under the terms of the GNU Affero General Public License as published by
+#      the Free Software Foundation, either version 3 of the License, or
+#      (at your option) any later version.
+#
+#      This program is distributed in the hope that it will be useful,
+#      but WITHOUT ANY WARRANTY; without even the implied warranty of
+#      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#      GNU Affero General Public License for more details.
+#
+#      You should have received a copy of the GNU Affero General Public License
+#      along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+#      This program is free software: you can redistribute it and/or modify
+#      it under the terms of the GNU Affero General Public License as published by
+#      the Free Software Foundation, either version 3 of the License, or
+#      (at your option) any later version.
+#
+#      This program is distributed in the hope that it will be useful,
+#      but WITHOUT ANY WARRANTY; without even the implied warranty of
+#      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#      GNU Affero General Public License for more details.
+
+"""Simple example showing usage with prometheus metrics."""
+
+import signal
+import time
+
+from prometheus_client import Summary, start_http_server
+
+import deep
+from simple_test import SimpleTest
+
+
+class GracefulKiller:
+    """Ensure clean shutdown."""
+
+    kill_now = False
+
+    def __init__(self):
+        """Crate new killer."""
+        signal.signal(signal.SIGINT, self.exit_gracefully)
+        signal.signal(signal.SIGTERM, self.exit_gracefully)
+
+    def exit_gracefully(self, *args):
+        """Exit example."""
+        self.kill_now = True
+
+
+def main():
+    """Run the example."""
+    killer = GracefulKiller()
+    ts = SimpleTest("This is a test")
+    while not killer.kill_now:
+        try:
+            ts.message(ts.new_id())
+        except BaseException as e:
+            print(e)
+            ts.reset()
+
+        time.sleep(0.1)
+
+
+# Create a metric to track time spent and requests made.
+REQUEST_TIME = Summary('request_processing_seconds', 'Time spent processing request')
+
+
+# Decorate function with metric.
+@REQUEST_TIME.time()
+def process_request(t):
+    """
+    Sleep.
+
+    A dummy function that takes some time.
+    """
+    time.sleep(t)
+
+
+if __name__ == '__main__':
+    start_http_server(8000)
+    d = deep.start({
+        'SERVICE_URL': 'localhost:43315',
+        'SERVICE_SECURE': 'False',
+    })
+
+    d.register_tracepoint("simple_test.py", 31)
+
+    print("app running")
+    main()

examples/simple-app-metrics/src/simple-app/simple_test.py

@@ -0,0 +1,73 @@
+#       Copyright (C) 2023  Intergral GmbH
+#
+#      This program is free software: you can redistribute it and/or modify
+#      it under the terms of the GNU Affero General Public License as published by
+#      the Free Software Foundation, either version 3 of the License, or
+#      (at your option) any later version.
+#
+#      This program is distributed in the hope that it will be useful,
+#      but WITHOUT ANY WARRANTY; without even the implied warranty of
+#      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#      GNU Affero General Public License for more details.
+#
+#      You should have received a copy of the GNU Affero General Public License
+#      along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""A simple test object for examples."""
+
+import time
+
+from base_test import BaseTest
+
+
+class SimpleTest(BaseTest):
+    """A basic test that is used in examples."""
+
+    def __init__(self, test_name):
+        """Create new test object."""
+        super().__init__()
+        self._started_at = round(time.time() * 1000)
+        self.__cnt = 0
+        self.char_counter = {}
+        self.test_name = test_name
+        self.max_executions = self.next_max()
+
+    def message(self, uuid):
+        """Print message to console."""
+        print("%s:%s" % (self.__cnt, uuid))
+        self.__cnt += 1
+        self.check_end(self.__cnt, self.max_executions)
+
+        info = self.make_char_count_map(uuid)
+        self.merge(self.char_counter, info)
+        if self.__cnt % 30 == 0:
+            self.dump()
+
+    def merge(self, char_counter, new_info):
+        """Merge captured data."""
+        for key in new_info:
+            new_val = new_info[key]
+
+            if key not in char_counter:
+                char_counter[key] = new_val
+            else:
+                char_counter[key] = new_val + char_counter[key]
+
+    def dump(self):
+        """Dump message to console."""
+        print(self.char_counter)
+        self.char_counter = {}
+
+    def check_end(self, value, max_executions):
+        """Check if we are at end."""
+        if value > max_executions:
+            raise Exception("Hit max executions %s %s " % (value, max_executions))
+
+    def __str__(self) -> str:
+        """Represent this as a string."""
+        return self.__class__.__name__ + ":" + self.test_name + ":" + str(self._started_at)
+
+    def reset(self):
+        """Reset the count."""
+        self.__cnt = 0
+        self.max_executions = self.next_max()