CM-53929 Add full support of .cycodeignore for repository and commit range scans (#351)

Author: DmytroLynda

cycode/cli/apps/report/sbom/path/path_command.py

@@ -12,6 +12,7 @@
 from cycode.cli.files_collector.zip_documents import zip_documents
 from cycode.cli.utils.get_api_client import get_report_cycode_client
 from cycode.cli.utils.progress_bar import SbomReportProgressBarSection
+from cycode.cli.utils.scan_utils import is_cycodeignore_allowed_by_scan_config
 from cycode.cli.utils.sentry import add_breadcrumb
 
 
@@ -37,7 +38,11 @@ def path_command(
 
     try:
         documents = get_relevant_documents(
-            progress_bar, SbomReportProgressBarSection.PREPARE_LOCAL_FILES, consts.SCA_SCAN_TYPE, (str(path),)
+            progress_bar,
+            SbomReportProgressBarSection.PREPARE_LOCAL_FILES,
+            consts.SCA_SCAN_TYPE,
+            (str(path),),
+            is_cycodeignore_allowed=is_cycodeignore_allowed_by_scan_config(ctx),
         )
         # TODO(MarshalX): combine perform_pre_scan_documents_actions with get_relevant_document.
         #  unhardcode usage of context in perform_pre_scan_documents_actions

cycode/cli/apps/scan/code_scanner.py

@@ -23,7 +23,11 @@
 from cycode.cli.models import CliError, Document, LocalScanResult
 from cycode.cli.utils.progress_bar import ScanProgressBarSection
 from cycode.cli.utils.scan_batch import run_parallel_batched_scan
-from cycode.cli.utils.scan_utils import generate_unique_scan_id, set_issue_detected_by_scan_results
+from cycode.cli.utils.scan_utils import (
+    generate_unique_scan_id,
+    is_cycodeignore_allowed_by_scan_config,
+    set_issue_detected_by_scan_results,
+)
 from cycode.cyclient.models import ZippedFileScanResult
 from cycode.logger import get_logger
 
@@ -42,7 +46,13 @@ def scan_disk_files(ctx: typer.Context, paths: tuple[str, ...]) -> None:
     progress_bar = ctx.obj['progress_bar']
 
     try:
-        documents = get_relevant_documents(progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, scan_type, paths)
+        documents = get_relevant_documents(
+            progress_bar,
+            ScanProgressBarSection.PREPARE_LOCAL_FILES,
+            scan_type,
+            paths,
+            is_cycodeignore_allowed=is_cycodeignore_allowed_by_scan_config(ctx),
+        )
         add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents)
         scan_documents(ctx, documents, get_scan_parameters(ctx, paths))
     except Exception as e:

cycode/cli/apps/scan/commit_range_scanner.py

@@ -29,6 +29,7 @@
     parse_commit_range_sast,
     parse_commit_range_sca,
 )
+from cycode.cli.files_collector.documents_walk_ignore import filter_documents_with_cycodeignore
 from cycode.cli.files_collector.file_excluder import excluder
 from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip
 from cycode.cli.files_collector.sca.sca_file_collector import (
@@ -40,7 +41,11 @@
 from cycode.cli.utils.git_proxy import git_proxy
 from cycode.cli.utils.path_utils import get_path_by_os
 from cycode.cli.utils.progress_bar import ScanProgressBarSection
-from cycode.cli.utils.scan_utils import generate_unique_scan_id, set_issue_detected_by_scan_results
+from cycode.cli.utils.scan_utils import (
+    generate_unique_scan_id,
+    is_cycodeignore_allowed_by_scan_config,
+    set_issue_detected_by_scan_results,
+)
 from cycode.cyclient.models import ZippedFileScanResult
 from cycode.logger import get_logger
 
@@ -189,6 +194,12 @@ def _scan_sca_commit_range(ctx: typer.Context, repo_path: str, commit_range: str
     from_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, from_commit_documents)
     to_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, to_commit_documents)
 
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    from_commit_documents = filter_documents_with_cycodeignore(
+        from_commit_documents, repo_path, is_cycodeignore_allowed
+    )
+    to_commit_documents = filter_documents_with_cycodeignore(to_commit_documents, repo_path, is_cycodeignore_allowed)
+
     perform_sca_pre_commit_range_scan_actions(
         repo_path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev
     )
@@ -204,6 +215,11 @@ def _scan_secret_commit_range(
         consts.SECRET_SCAN_TYPE, commit_diff_documents_to_scan
     )
 
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    diff_documents_to_scan = filter_documents_with_cycodeignore(
+        diff_documents_to_scan, repo_path, is_cycodeignore_allowed
+    )
+
     scan_documents(
         ctx, diff_documents_to_scan, get_scan_parameters(ctx, (repo_path,)), is_git_diff=True, is_commit_range=True
     )
@@ -221,9 +237,14 @@ def _scan_sast_commit_range(ctx: typer.Context, repo_path: str, commit_range: st
         to_commit_rev,
         reverse_diff=False,
     )
+
     commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, commit_documents)
     diff_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, diff_documents)
 
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    commit_documents = filter_documents_with_cycodeignore(commit_documents, repo_path, is_cycodeignore_allowed)
+    diff_documents = filter_documents_with_cycodeignore(diff_documents, repo_path, is_cycodeignore_allowed)
+
     _scan_commit_range_documents(ctx, commit_documents, diff_documents, scan_parameters=scan_parameters)
 
 
@@ -254,11 +275,18 @@ def _scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None:
         progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES,
         repo_path=repo_path,
     )
+
     git_head_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, git_head_documents)
     pre_committed_documents = excluder.exclude_irrelevant_documents_to_scan(
         consts.SCA_SCAN_TYPE, pre_committed_documents
     )
 
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    git_head_documents = filter_documents_with_cycodeignore(git_head_documents, repo_path, is_cycodeignore_allowed)
+    pre_committed_documents = filter_documents_with_cycodeignore(
+        pre_committed_documents, repo_path, is_cycodeignore_allowed
+    )
+
     perform_sca_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents)
 
     _scan_commit_range_documents(
@@ -288,8 +316,12 @@ def _scan_secret_pre_commit(ctx: typer.Context, repo_path: str) -> None:
                 is_git_diff_format=True,
             )
         )
+
     documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(consts.SECRET_SCAN_TYPE, documents_to_scan)
 
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    documents_to_scan = filter_documents_with_cycodeignore(documents_to_scan, repo_path, is_cycodeignore_allowed)
+
     scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx), is_git_diff=True)
 
 
@@ -301,11 +333,18 @@ def _scan_sast_pre_commit(ctx: typer.Context, repo_path: str, **_) -> None:
         progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES,
         repo_path=repo_path,
     )
+
     pre_committed_documents = excluder.exclude_irrelevant_documents_to_scan(
         consts.SAST_SCAN_TYPE, pre_committed_documents
     )
     diff_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, diff_documents)
 
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    pre_committed_documents = filter_documents_with_cycodeignore(
+        pre_committed_documents, repo_path, is_cycodeignore_allowed
+    )
+    diff_documents = filter_documents_with_cycodeignore(diff_documents, repo_path, is_cycodeignore_allowed)
+
     _scan_commit_range_documents(ctx, pre_committed_documents, diff_documents, scan_parameters=scan_parameters)
 
 

cycode/cli/apps/scan/repository/repository_command.py

@@ -8,13 +8,15 @@
 from cycode.cli.apps.scan.code_scanner import scan_documents
 from cycode.cli.apps.scan.scan_parameters import get_scan_parameters
 from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception
+from cycode.cli.files_collector.documents_walk_ignore import filter_documents_with_cycodeignore
 from cycode.cli.files_collector.file_excluder import excluder
 from cycode.cli.files_collector.repository_documents import get_git_repository_tree_file_entries
 from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed
 from cycode.cli.logger import logger
 from cycode.cli.models import Document
 from cycode.cli.utils.path_utils import get_path_by_os
 from cycode.cli.utils.progress_bar import ScanProgressBarSection
+from cycode.cli.utils.scan_utils import is_cycodeignore_allowed_by_scan_config
 from cycode.cli.utils.sentry import add_breadcrumb
 
 
@@ -60,6 +62,9 @@ def repository_command(
 
         documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan)
 
+        is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+        documents_to_scan = filter_documents_with_cycodeignore(documents_to_scan, str(path), is_cycodeignore_allowed)
+
         add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents_to_scan)
 
         logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch})

cycode/cli/apps/scan/scan_command.py

@@ -1,9 +1,11 @@
+import os
 from pathlib import Path
 from typing import Annotated, Optional
 
 import click
 import typer
 
+from cycode.cli.apps.scan.remote_url_resolver import _try_get_git_remote_url
 from cycode.cli.cli_types import ExportTypeOption, ScanTypeOption, ScaScanTypeOption, SeverityOption
 from cycode.cli.consts import (
     ISSUE_DETECTED_STATUS_CODE,
@@ -161,10 +163,15 @@ def scan_command(
     scan_client = get_scan_cycode_client(ctx)
     ctx.obj['client'] = scan_client
 
-    remote_scan_config = scan_client.get_scan_configuration_safe(scan_type)
+    # Get remote URL from current working directory
+    remote_url = _try_get_git_remote_url(os.getcwd())
+
+    remote_scan_config = scan_client.get_scan_configuration_safe(scan_type, remote_url)
     if remote_scan_config:
         excluder.apply_scan_config(str(scan_type), remote_scan_config)
 
+    ctx.obj['scan_config'] = remote_scan_config
+
     if export_type and export_file:
         console_printer = ctx.obj['console_printer']
         console_printer.enable_recording(export_type, export_file)

cycode/cli/consts.py

@@ -17,6 +17,8 @@
 IAC_SCAN_SUPPORTED_FILE_EXTENSIONS = ('.tf', '.tf.json', '.json', '.yaml', '.yml', '.dockerfile', '.containerfile')
 IAC_SCAN_SUPPORTED_FILE_PREFIXES = ('dockerfile', 'containerfile')
 
+CYCODEIGNORE_FILENAME = '.cycodeignore'
+
 SECRET_SCAN_FILE_EXTENSIONS_TO_IGNORE = (
     '.DS_Store',
     '.bmp',

cycode/cli/files_collector/documents_walk_ignore.py

@@ -0,0 +1,124 @@
+import os
+from typing import TYPE_CHECKING
+
+from cycode.cli import consts
+from cycode.cli.logger import get_logger
+from cycode.cli.utils.ignore_utils import IgnoreFilterManager
+
+if TYPE_CHECKING:
+    from cycode.cli.models import Document
+
+logger = get_logger('Documents Ignores')
+
+
+def _get_cycodeignore_path(repo_path: str) -> str:
+    """Get the path to .cycodeignore file in the repository root."""
+    return os.path.join(repo_path, consts.CYCODEIGNORE_FILENAME)
+
+
+def _create_ignore_filter_manager(repo_path: str, cycodeignore_path: str) -> IgnoreFilterManager:
+    """Create IgnoreFilterManager with .cycodeignore file."""
+    return IgnoreFilterManager.build(
+        path=repo_path,
+        global_ignore_file_paths=[cycodeignore_path],
+        global_patterns=[],
+    )
+
+
+def _log_ignored_files(repo_path: str, dirpath: str, ignored_dirnames: list[str], ignored_filenames: list[str]) -> None:
+    """Log ignored files for debugging (similar to walk_ignore function)."""
+    rel_dirpath = '' if dirpath == repo_path else os.path.relpath(dirpath, repo_path)
+    display_dir = rel_dirpath or '.'
+
+    for is_dir, names in (
+        (True, ignored_dirnames),
+        (False, ignored_filenames),
+    ):
+        for name in names:
+            full_path = os.path.join(repo_path, display_dir, name)
+            if is_dir:
+                full_path = os.path.join(full_path, '*')
+            logger.debug('Ignoring match %s', full_path)
+
+
+def _build_allowed_paths_set(ignore_filter_manager: IgnoreFilterManager, repo_path: str) -> set[str]:
+    """Build set of allowed file paths using walk_with_ignored."""
+    allowed_paths = set()
+
+    for dirpath, _dirnames, filenames, ignored_dirnames, ignored_filenames in ignore_filter_manager.walk_with_ignored():
+        _log_ignored_files(repo_path, dirpath, ignored_dirnames, ignored_filenames)
+
+        for filename in filenames:
+            file_path = os.path.join(dirpath, filename)
+            allowed_paths.add(file_path)
+
+    return allowed_paths
+
+
+def _get_document_check_path(document: 'Document', repo_path: str) -> str:
+    """Get the normalized absolute path for a document to check against allowed paths."""
+    check_path = document.absolute_path
+    if not check_path:
+        check_path = document.path if os.path.isabs(document.path) else os.path.join(repo_path, document.path)
+
+    return os.path.normpath(check_path)
+
+
+def _filter_documents_by_allowed_paths(
+    documents: list['Document'], allowed_paths: set[str], repo_path: str
+) -> list['Document']:
+    """Filter documents by checking if their paths are in the allowed set."""
+    filtered_documents = []
+
+    for document in documents:
+        try:
+            check_path = _get_document_check_path(document, repo_path)
+
+            if check_path in allowed_paths:
+                filtered_documents.append(document)
+            else:
+                relative_path = os.path.relpath(check_path, repo_path)
+                logger.debug('Filtered out document due to .cycodeignore: %s', relative_path)
+        except Exception as e:
+            logger.debug('Error processing document %s: %s', document.path, e)
+            filtered_documents.append(document)
+
+    return filtered_documents
+
+
+def filter_documents_with_cycodeignore(
+    documents: list['Document'], repo_path: str, is_cycodeignore_allowed: bool = True
+) -> list['Document']:
+    """Filter documents based on .cycodeignore patterns.
+
+    This function uses .cycodeignore file in the repository root to filter out
+    documents whose paths match any of those patterns.
+
+    Args:
+        documents: List of Document objects to filter
+        repo_path: Path to the repository root
+        is_cycodeignore_allowed: Whether .cycodeignore filtering is allowed by scan configuration
+
+    Returns:
+        List of Document objects that don't match any .cycodeignore patterns
+    """
+    if not is_cycodeignore_allowed:
+        logger.debug('.cycodeignore filtering is not allowed by scan configuration')
+        return documents
+
+    cycodeignore_path = _get_cycodeignore_path(repo_path)
+
+    if not os.path.exists(cycodeignore_path):
+        logger.debug('.cycodeignore file does not exist in the repository root')
+        return documents
+
+    logger.info('Using %s for filtering documents', cycodeignore_path)
+
+    ignore_filter_manager = _create_ignore_filter_manager(repo_path, cycodeignore_path)
+
+    allowed_paths = _build_allowed_paths_set(ignore_filter_manager, repo_path)
+
+    filtered_documents = _filter_documents_by_allowed_paths(documents, allowed_paths, repo_path)
+
+    logger.debug('Filtered %d documents using .cycodeignore patterns', len(documents) - len(filtered_documents))
+    return filtered_documents