DLPX-94248 address drgn merge conflict with upstream

.pre-commit-config.yaml

@@ -0,0 +1,39 @@
+exclude: ^contrib/
+repos:
+-   repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+    -   id: isort
+        name: isort (python)
+-   repo: https://github.com/psf/black
+    rev: 24.8.0
+    hooks:
+    -   id: black
+        exclude: ^docs/exts/details\.py$
+-   repo: https://github.com/pycqa/flake8
+    rev: 7.1.2
+    hooks:
+    -   id: flake8
+#-   repo: https://github.com/pre-commit/mirrors-mypy
+#    rev: v1.14.1
+#    hooks:
+#    -   id: mypy
+#        args: [--show-error-codes, --strict, --no-warn-return-any]
+#        files: ^(drgn/.*\.py|_drgn.pyi|_drgn_util/.*\.py|tools/.*\.py|vmtest/.*\.py)$
+#        additional_dependencies: [aiohttp, uritemplate]
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    -   id: trailing-whitespace
+        exclude_types: [diff]
+    -   id: end-of-file-fixer
+        exclude_types: [diff]
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: debug-statements
+    -   id: check-merge-conflict
+-   repo: https://github.com/netromdk/vermin
+    rev: v1.6.0
+    hooks:
+    -   id: vermin
+        args: ['-t=3.8-', '--violations', '--eval-annotations']

drgn/commands/_builtin/__init__.py

@@ -9,11 +9,14 @@
 import argparse
 import importlib
 import pkgutil
+import re
 import subprocess
+import sys
+import traceback
 from typing import Any, Dict
 
 from drgn import Program, execscript
-from drgn.commands import argument, command, custom_command
+from drgn.commands import _shell_command, argument, command, custom_command
 
 # Import all submodules, recursively.
 for _module_info in pkgutil.walk_packages(__path__, __name__ + "."):
@@ -37,6 +40,82 @@ def _cmd_sh(prog: Program, name: str, args: str, **kwargs: Any) -> int:
         return subprocess.call(["sh", "-i"])
 
 
+@custom_command(
+    description="execute a python statement and allow shell redirection",
+    usage="**py** [*command*]",
+    long_description="""
+    Execute the given code, up to the first shell redirection or pipeline
+    statement, as Python code.
+
+    For each occurrence of a pipeline operator (``|``) or any redirection
+    operator (``<``, ``>``, ``<<``, ``>>``), attempt to parse the preceding text
+    as Python code. If the preceding text is syntactically valid code, then
+    interpret the remainder of the command as shell redirections or pipelines,
+    and execute the Python code with those redirections and pipelines applied.
+
+    The operators above can be used in syntactically valid Python. This means
+    you need to be careful when using this function, and ensure that you wrap
+    their uses with parentheses.
+
+    For example, consider the command: ``%py field | MY_FLAG | grep foo``. While
+    the intent here may be to execute the Python code ``field | MY_FLAG`` and
+    pass its result to ``grep``, that is not what will happen. The portion of
+    text prior to the first ``|`` is valid Python, so it will be executed, and
+    its output piped to the shell pipeline ``MY_FLAG | grep foo``. Instead,
+    running ``%py (field | MY_FLAG) | grep foo`` ensures that ``field |
+    MY_FLAG`` gets piped to ``grep foo``, because ``(field`` on its own is not
+    valid Python syntax.
+    """,
+)
+def _cmd_py(
+    prog: Program,
+    name: str,
+    args: str,
+    *,
+    globals: Dict[str, Any],
+    **kwargs: Any,
+) -> None:
+
+    def print_exc() -> None:
+        # When printing a traceback, we should not print our own stack frame, as
+        # that would confuse the user. Unfortunately the traceback objects are
+        # linked lists and there's no functionality to drop the last N frames of
+        # a traceback while printing.
+        _, _, tb = sys.exc_info()
+        count = 0
+        while tb:
+            count += 1
+            tb = tb.tb_next
+        traceback.print_exc(limit=1 - count)
+
+    for match in re.finditer(r"[|<>]", args):
+        try:
+            pos = match.start()
+            code = compile(args[:pos], "<input>", "single")
+            break
+        except SyntaxError:
+            pass
+    else:
+        # Fallback for no match: compile all the code as a "single" statement so
+        # exec() still prints out the result. At this point, a syntax error
+        # should be formatted just like a standard Python exception.
+        try:
+            pos = len(args)
+            code = compile(args, "<input>", "single")
+        except SyntaxError:
+            print_exc()
+            return
+
+    with _shell_command(args[pos:]):
+        try:
+            exec(code, globals)
+        except (Exception, KeyboardInterrupt):
+            # Any exception should be formatted just as the interpreter would.
+            # This includes keyboard interrupts, but not things like
+            # SystemExit or GeneratorExit.
+            print_exc()
+
+
 @command(
     description="run a drgn script",
     long_description="""

drgn/commands/_builtin/crash/ptov.py

@@ -0,0 +1,114 @@
+# Copyright (c) 2025, Kylin Software, Inc. and affiliates.
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+import argparse
+from typing import Any
+
+from drgn import Object, Program
+from drgn.commands import argument, drgn_argument
+from drgn.commands.crash import CrashDrgnCodeBuilder, crash_command, parse_cpuspec
+from drgn.helpers.common.format import print_table
+from drgn.helpers.linux.mm import phys_to_virt
+from drgn.helpers.linux.percpu import per_cpu_ptr
+
+
+@crash_command(
+    description="physical or per-CPU to virtual",
+    long_description="""This command translates a hexadecimal physical address into a
+                        kernel virtual address. Alternatively, a hexadecimal per-cpu
+                        offset and cpu specifier will be translated into kernel virtual
+                        addresses for each cpu specified.""",
+    arguments=(
+        argument(
+            "address",
+            metavar="address|offset:cpuspec",
+            nargs="+",
+            help="hexadecimal physical address or hexadecimal per-CPU offset and CPU specifier",
+        ),
+        drgn_argument,
+    ),
+)
+def _crash_cmd_ptov(
+    prog: Program, name: str, args: argparse.Namespace, **kwargs: Any
+) -> None:
+    if args.drgn:
+        # Create a single builder for all addresses
+        builder = CrashDrgnCodeBuilder(prog)
+        physical_addresses = []
+        per_cpu_offsets = []
+
+        for address in args.address:
+            if ":" in address:
+                # Add imports only once
+                builder.add_from_import("drgn", "Object")
+                builder.add_from_import("drgn.helpers.linux.percpu", "per_cpu_ptr")
+                builder.add_from_import(
+                    "drgn.helpers.linux.cpumask", "for_each_possible_cpu"
+                )
+                # Parse the cpuspec in the actual command code
+                offset_str, cpu_spec = address.split(":", 1)
+                offset = int(offset_str, 16)
+                per_cpu_offsets.append((offset, parse_cpuspec(cpu_spec)))
+            else:
+                # Add imports only once
+                builder.add_from_import("drgn.helpers.linux.mm", "phys_to_virt")
+                physical_addresses.append(int(address, 16))
+
+        # Generate code for physical addresses
+        if physical_addresses:
+            builder.append("addresses = [")
+            builder.append(", ".join(f"0x{addr:x}" for addr in physical_addresses))
+            builder.append("]\n")
+            builder.append("for address in addresses:\n")
+            builder.append("    virt = phys_to_virt(address)\n")
+
+        # Generate code for per-CPU offsets
+        for offset, cpuspec in per_cpu_offsets:
+            builder.append(f"\noffset = {offset:#x}\n")
+            builder.append_cpuspec(
+                cpuspec,
+                """
+    virt = per_cpu_ptr(Object(prog, 'void *', offset), cpu)
+                """,
+            )
+
+        # Print the generated code once at the end
+        builder.print()
+        return
+
+    # Handle direct execution without --drgn
+    for i, address in enumerate(args.address):
+        if i > 0:
+            print()  # Add a blank line between outputs for multiple addresses
+
+        if ":" in address:
+            # Handle per-CPU offset case
+            offset_str, cpu_spec = address.split(":", 1)
+            offset = int(offset_str, 16)
+
+            # Parse CPU specifier using parse_cpuspec
+            cpus = parse_cpuspec(cpu_spec)
+
+            # Print offset information
+            print(f"PER-CPU OFFSET: {offset:x}")  # Directly print offset information
+
+            # Prepare data for print_table()
+            rows = [("  CPU", "  VIRTUAL")]  # Add CPU and VIRTUAL header
+            ptr = Object(prog, "void *", offset)  # Changed type to "void *"
+            for cpu in cpus.cpus(prog):
+                virt = per_cpu_ptr(ptr, cpu)
+                rows.append((f"  [{cpu}]", f"{virt.value_():016x}"))
+
+            # Print the table
+            print_table(rows)
+        else:
+            # Handle physical address case
+            phys = int(address, 16)
+            virt = phys_to_virt(prog, phys)
+            virt_int = virt.value_()
+
+            # Prepare data for print_table()
+            rows = [("VIRTUAL", "PHYSICAL"), (f"{virt_int:016x}", f"{phys:x}")]
+
+            # Print the table
+            print_table(rows)

drgn/helpers/linux/sched.py

@@ -24,11 +24,14 @@
 
 __all__ = (
     "cpu_curr",
+    "cpu_rq",
     "get_task_state",
     "idle_task",
     "loadavg",
     "task_cpu",
     "task_on_cpu",
+    "task_rq",
+    "task_since_last_arrival_ns",
     "task_state_to_char",
     "task_thread_info",
 )
@@ -189,10 +192,12 @@ def task_rq(task: Object) -> Object:
 
 def task_since_last_arrival_ns(task: Object) -> int:
     """
-    Get the number of nanoseconds since a task last started running.
+    Get the difference between the runqueue timestamp when a task last started
+    running and the current runqueue timestamp.
 
-    Assuming that time slices are short, this is approximately the time that
-    the task has been in its current status (running, queued, or blocked).
+    This is approximately the time that the task has been in its current status
+    (running, queued, or blocked). However, if a CPU is either idle or running
+    the same task for a long time, then the timestamps will not be accurate.
 
     This is only supported if the kernel was compiled with
     ``CONFIG_SCHEDSTATS`` or ``CONFIG_TASK_DELAY_ACCT``.

tests/commands/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+from drgn.commands import DEFAULT_COMMAND_NAMESPACE
+from tests import TestCase
+
+
+class CommandTestCase(TestCase):
+
+    @staticmethod
+    def run_command(source, **kwargs):
+        return DEFAULT_COMMAND_NAMESPACE.run(None, source, globals={}, **kwargs)

tests/commands/test_builtin.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+import contextlib
+import os
+from pathlib import Path
+import tempfile
+
+import drgn.commands._builtin  # noqa: F401
+from tests.commands import CommandTestCase
+
+
+@contextlib.contextmanager
+def temporary_working_directory():
+    old_working_directory = os.getcwd()
+    with tempfile.TemporaryDirectory() as f:
+        try:
+            os.chdir(f)
+            yield f
+        finally:
+            os.chdir(old_working_directory)
+
+
+class RedirectedFile:
+    def __init__(self, f):
+        self.tempfile = f
+        self.value = None
+
+
+@contextlib.contextmanager
+def redirect(stdout=False, stderr=False):
+    # To redirect stdout for commands, we need a real file descriptor, not just
+    # a StringIO
+    with contextlib.ExitStack() as stack:
+        f = stack.enter_context(tempfile.TemporaryFile("w+t"))
+        if stdout:
+            stack.enter_context(contextlib.redirect_stdout(f))
+        if stderr:
+            stack.enter_context(contextlib.redirect_stderr(f))
+        redir = RedirectedFile(f)
+        try:
+            yield redir
+        finally:
+            f.seek(0)
+            redir.value = f.read()
+
+
+class TestPyCommand(CommandTestCase):
+
+    def test_py_redirect(self):
+        with temporary_working_directory() as temp_dir:
+            path = Path(temp_dir) / "6"
+            self.run_command("py var = 5; var > 6")
+            self.assertEqual(path.read_text(), "5\n")
+
+    def test_py_paren_avoid_redirect(self):
+        self.run_command("py var = 5; (var > 6)")
+        with redirect(stdout=True) as f:
+            self.run_command("py var = 5; (var > 6)")
+        self.assertEqual(f.value, "False\n")
+
+    def test_py_pipe(self):
+        with redirect(stdout=True) as f:
+            self.run_command("py echo = 5; 2 | echo + 5")
+        self.assertEqual(f.value, "+ 5\n")
+
+    def test_py_avoid_pipe(self):
+        with redirect(stdout=True) as f:
+            self.run_command("py echo = 5; (2 | (echo + 5))")
+        self.assertEqual(f.value, "10\n")
+
+    def test_py_chooses_first_pipe(self):
+        with redirect(stdout=True) as f:
+            # If the first | is used to separate the Python from the pipeline
+            # (the expected behavior), then we'll get the value 5 written into
+            # the "echo" command, which will ignore that and write "+ 6" through
+            # the cat process to stdout. If the second | is used to separate the
+            # Python from the pipeline, then we'll get "15" written into the cat
+            # process. If none of the | were interpreted as a pipeline operator,
+            # then the statement would output 31.
+            self.run_command("py echo = 5; cat = 16; 5 | echo + 6 | cat")
+        self.assertEqual("+ 6\n", f.value)
+
+    def test_py_traceback_on_syntax_error(self):
+        with redirect(stderr=True) as f:
+            self.run_command("py a +")
+        # SyntaxError does not print the "Traceback" header. Rather than trying
+        # to assert too much about the format of the traceback, just assert that
+        # the incorrect code is shown, as it would be for a traceback.
+        self.assertTrue("a +" in f.value)
+        self.assertTrue("SyntaxError" in f.value)
+
+    def test_py_traceback_on_exception(self):
+        with redirect(stderr=True) as f:
+            self.run_command("py raise Exception('text')")
+        self.assertTrue(f.value.startswith("Traceback"))
+        self.assertTrue("Exception" in f.value)