Detect installed packages in the selected environment

Author: karthiknadig

package.json

@@ -1526,10 +1526,8 @@
                 ],
                 "configuration": "./languages/pip-requirements.json",
                 "filenamePatterns": [
-                    "**/*-requirements.{txt, in}",
-                    "**/*-constraints.txt",
-                    "**/requirements-*.{txt, in}",
-                    "**/constraints-*.txt",
+                    "**/*requirements*.{txt, in}",
+                    "**/*constraints*.txt",
                     "**/requirements/*.{txt,in}",
                     "**/constraints/*.txt"
                 ],
@@ -1733,7 +1731,7 @@
                 {
                     "group": "Python",
                     "command": "python.createEnvironment-button",
-                    "when": "showCreateEnvButton && resourceLangId == pip-requirements && !virtualWorkspace && shellExecutionSupported && !inDiffEditor"
+                    "when": "showCreateEnvButton && resourceLangId == pip-requirements && !virtualWorkspace && shellExecutionSupported && !inDiffEditor && pipDepsNotInstalled"
                 },
                 {
                     "group": "Python",

pythonFiles/installed_check.py

@@ -0,0 +1,72 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import argparse
+import json
+import os
+import pathlib
+import sys
+from typing import Optional, Sequence
+
+LIB_ROOT = pathlib.Path(__file__).parent / "lib" / "python"
+sys.path.insert(0, os.fspath(LIB_ROOT))
+
+from importlib_metadata import metadata
+from packaging.requirements import Requirement
+
+
+def parse_args(argv: Optional[Sequence[str]] = None):
+    if argv is None:
+        argv = sys.argv[1:]
+    parser = argparse.ArgumentParser(
+        description="Check for installed packages against requirements"
+    )
+    parser.add_argument(
+        "REQUIREMENTS", type=str, help="Path to requirements.[txt, in]", nargs="+"
+    )
+
+    return parser.parse_args(argv)
+
+
+def parse_requirements(line: str) -> Optional[Requirement]:
+    try:
+        req = Requirement(line.strip("\\"))
+        if req.marker is None:
+            return req
+        elif req.marker.evaluate():
+            return req
+    except:
+        return None
+
+
+def main():
+    args = parse_args()
+
+    diagnostics = []
+    for req_file in args.REQUIREMENTS:
+        req_file = pathlib.Path(req_file)
+        if req_file.exists():
+            lines = req_file.read_text(encoding="utf-8").splitlines()
+            for n, line in enumerate(lines):
+                if line.startswith(("#", "-", " ")) or line == "":
+                    continue
+
+                req = parse_requirements(line)
+                if req:
+                    try:
+                        # Check if package is installed
+                        metadata(req.name)
+                    except:
+                        diagnostics.append(
+                            {
+                                "line": n,
+                                "package": req.name,
+                                "code": "not-installed",
+                                "severity": 3,
+                            }
+                        )
+    print(json.dumps(diagnostics, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

requirements.in

@@ -8,3 +8,7 @@ typing-extensions==4.5.0
 
 # Fallback env creator for debian
 microvenv
+
+# Checker for installed packages
+packaging
+importlib_metadata

requirements.txt

@@ -4,11 +4,25 @@
 #
 #    pip-compile --generate-hashes requirements.in
 #
+importlib-metadata==6.6.0 \
+    --hash=sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed \
+    --hash=sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705
+    # via -r requirements.in
 microvenv==2023.2.0 \
     --hash=sha256:5b46296d6a65992946da504bd9e724a5becf5c256091f2f9383e5b4e9f567f23 \
     --hash=sha256:a07e88a8fb5ee90219b86dd90095cb5646462d45d30285ea3b1a3c7cf33616d3
     # via -r requirements.in
+packaging==23.1 \
+    --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
+    --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
+    # via -r requirements.in
 typing-extensions==4.5.0 \
     --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \
     --hash=sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4
-    # via -r requirements.in
+    # via
+    #   -r requirements.in
+    #   importlib-metadata
+zipp==3.15.0 \
+    --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \
+    --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556
+    # via importlib-metadata

src/client/common/process/internal/scripts/index.ts

@@ -149,3 +149,8 @@ export function createCondaScript(): string {
     const script = path.join(SCRIPTS_DIR, 'create_conda.py');
     return script;
 }
+
+export function installedCheckScript(): string {
+    const script = path.join(SCRIPTS_DIR, 'installed_check.py');
+    return script;
+}

src/client/common/vscodeApis/languageApis.ts

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License
+
+import { DiagnosticChangeEvent, DiagnosticCollection, Disposable, languages } from 'vscode';
+
+export function createDiagnosticCollection(name: string): DiagnosticCollection {
+    return languages.createDiagnosticCollection(name);
+}
+
+export function onDidChangeDiagnostics(handler: (e: DiagnosticChangeEvent) => void): Disposable {
+    return languages.onDidChangeDiagnostics(handler);
+}

src/client/common/vscodeApis/windowApis.ts

@@ -77,6 +77,10 @@ export function getActiveTextEditor(): TextEditor | undefined {
     return activeTextEditor;
 }
 
+export function onDidChangeActiveTextEditor(handler: (e: TextEditor | undefined) => void): Disposable {
+    return window.onDidChangeActiveTextEditor(handler);
+}
+
 export enum MultiStepAction {
     Back = 'Back',
     Cancel = 'Cancel',

src/client/common/vscodeApis/workspaceApis.ts

@@ -36,12 +36,12 @@ export function findFiles(
     return vscode.workspace.findFiles(include, exclude, maxResults, token);
 }
 
-export function onDidSaveTextDocument(
-    listener: (e: vscode.TextDocument) => unknown,
-    thisArgs?: unknown,
-    disposables?: vscode.Disposable[],
-): vscode.Disposable {
-    return vscode.workspace.onDidSaveTextDocument(listener, thisArgs, disposables);
+export function onDidCloseTextDocument(handler: (e: vscode.TextDocument) => unknown): vscode.Disposable {
+    return vscode.workspace.onDidCloseTextDocument(handler);
+}
+
+export function onDidSaveTextDocument(handler: (e: vscode.TextDocument) => unknown): vscode.Disposable {
+    return vscode.workspace.onDidSaveTextDocument(handler);
 }
 
 export function getOpenTextDocuments(): readonly vscode.TextDocument[] {

src/client/extensionActivation.ts

@@ -55,6 +55,7 @@ import { registerCreateEnvironmentFeatures } from './pythonEnvironments/creation
 import { IInterpreterQuickPick } from './interpreter/configuration/types';
 import { registerInstallFormatterPrompt } from './providers/prompts/installFormatterPrompt';
 import { registerCreateEnvButtonFeatures } from './pythonEnvironments/creation/createEnvButtonContext';
+import { registerInstalledPackagesChecking } from './pythonEnvironments/creation/installedPackagesDiagnostic';
 
 export async function activateComponents(
     // `ext` is passed to any extra activation funcs.
@@ -99,6 +100,7 @@ export function activateFeatures(ext: ExtensionState, _components: Components):
     const pathUtils = ext.legacyIOC.serviceContainer.get<IPathUtils>(IPathUtils);
     registerCreateEnvironmentFeatures(ext.disposables, interpreterQuickPick, interpreterPathService, pathUtils);
     registerCreateEnvButtonFeatures(ext.disposables);
+    registerInstalledPackagesChecking(interpreterPathService, ext.disposables);
 }
 
 /// //////////////////////////

src/client/pythonEnvironments/creation/installedPackagesDiagnostic.ts

@@ -0,0 +1,122 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License
+
+import { Diagnostic, DiagnosticCollection, DiagnosticSeverity, l10n, Range, TextDocument, Uri } from 'vscode';
+import { installedCheckScript } from '../../common/process/internal/scripts';
+import { plainExec } from '../../common/process/rawProcessApis';
+import { IDisposableRegistry, IInterpreterPathService } from '../../common/types';
+import { executeCommand } from '../../common/vscodeApis/commandApis';
+import { createDiagnosticCollection, onDidChangeDiagnostics } from '../../common/vscodeApis/languageApis';
+import { getActiveTextEditor, onDidChangeActiveTextEditor } from '../../common/vscodeApis/windowApis';
+import {
+    getOpenTextDocuments,
+    onDidCloseTextDocument,
+    onDidOpenTextDocument,
+    onDidSaveTextDocument,
+} from '../../common/vscodeApis/workspaceApis';
+import { traceVerbose } from '../../logging';
+
+interface PackageDiagnostic {
+    package: string;
+    line: number;
+    code: string;
+    severity: DiagnosticSeverity;
+}
+
+const SOURCE = 'Python-Ext';
+const PIP_DEPS_NOT_INSTALLED_KEY = 'pipDepsNotInstalled';
+
+async function getPipRequirementsDiagnostics(
+    interpreterPathService: IInterpreterPathService,
+    doc: TextDocument,
+): Promise<Diagnostic[]> {
+    const interpreter = interpreterPathService.get(doc.uri);
+    const result = await plainExec(interpreter, [installedCheckScript(), doc.uri.fsPath]);
+    traceVerbose('Installed packages check result:\n', result.stdout);
+    let diagnostics: Diagnostic[] = [];
+    try {
+        const raw = JSON.parse(result.stdout) as PackageDiagnostic[];
+        diagnostics = raw.map((item) => {
+            const d = new Diagnostic(
+                new Range(item.line, 0, item.line, item.package.length),
+                l10n.t(`Package \`${item.package}\` is not installed in the selected environment.`),
+                item.severity,
+            );
+            d.code = { value: item.code, target: Uri.parse(`https://pypi.org/p/${item.package}`) };
+            d.source = SOURCE;
+            return d;
+        });
+    } catch {
+        diagnostics = [];
+    }
+    return diagnostics;
+}
+
+async function setContextForActiveEditor(diagnosticCollection: DiagnosticCollection): Promise<void> {
+    const doc = getActiveTextEditor()?.document;
+    if (doc && doc.languageId === 'pip-requirements') {
+        const diagnostics = diagnosticCollection.get(doc.uri);
+        if (diagnostics && diagnostics.length > 0) {
+            traceVerbose(`Setting context for pip dependencies not installed: ${doc.uri.fsPath}`);
+            await executeCommand('setContext', PIP_DEPS_NOT_INSTALLED_KEY, true);
+            return;
+        }
+    }
+
+    // undefined here in the logs means no file was selected
+    traceVerbose(`Clearing context for pip dependencies not installed: ${doc?.uri.fsPath}`);
+    await executeCommand('setContext', PIP_DEPS_NOT_INSTALLED_KEY, false);
+}
+
+export function registerInstalledPackagesChecking(
+    interpreterPathService: IInterpreterPathService,
+    disposables: IDisposableRegistry,
+): void {
+    const diagnosticCollection = createDiagnosticCollection(SOURCE);
+
+    disposables.push(diagnosticCollection);
+    disposables.push(
+        onDidOpenTextDocument(async (e: TextDocument) => {
+            if (e.languageId === 'pip-requirements') {
+                const diagnostics = await getPipRequirementsDiagnostics(interpreterPathService, e);
+                if (diagnostics.length > 0) {
+                    diagnosticCollection.set(e.uri, diagnostics);
+                } else if (diagnosticCollection.has(e.uri)) {
+                    diagnosticCollection.delete(e.uri);
+                }
+            }
+        }),
+        onDidSaveTextDocument(async (e: TextDocument) => {
+            if (e.languageId === 'pip-requirements') {
+                const diagnostics = await getPipRequirementsDiagnostics(interpreterPathService, e);
+                if (diagnostics.length > 0) {
+                    diagnosticCollection.set(e.uri, diagnostics);
+                } else if (diagnosticCollection.has(e.uri)) {
+                    diagnosticCollection.delete(e.uri);
+                }
+            }
+        }),
+        onDidCloseTextDocument((e: TextDocument) => {
+            if (diagnosticCollection.has(e.uri)) {
+                diagnosticCollection.delete(e.uri);
+            }
+        }),
+        onDidChangeDiagnostics(async () => {
+            await setContextForActiveEditor(diagnosticCollection);
+        }),
+        onDidChangeActiveTextEditor(async () => {
+            await setContextForActiveEditor(diagnosticCollection);
+        }),
+    );
+
+    getOpenTextDocuments().forEach(async (doc: TextDocument) => {
+        if (doc.languageId === 'pip-requirements') {
+            const diagnostics = await getPipRequirementsDiagnostics(interpreterPathService, doc);
+            if (diagnostics.length > 0) {
+                diagnosticCollection.set(doc.uri, diagnostics);
+            } else if (diagnosticCollection.has(doc.uri)) {
+                diagnosticCollection.delete(doc.uri);
+            }
+        }
+    });
+}