CM-53929 - apply .cycodeignore when it is allowed by API

Author: DmytroLynda

cycode/cli/apps/report/sbom/path/path_command.py

@@ -12,6 +12,7 @@
 from cycode.cli.files_collector.zip_documents import zip_documents
 from cycode.cli.utils.get_api_client import get_report_cycode_client
 from cycode.cli.utils.progress_bar import SbomReportProgressBarSection
+from cycode.cli.utils.scan_utils import is_cycodeignore_allowed_by_scan_config
 from cycode.cli.utils.sentry import add_breadcrumb
 
 
@@ -36,8 +37,9 @@ def path_command(
     report_execution_id = -1
 
     try:
+        is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
         documents = get_relevant_documents(
-            progress_bar, SbomReportProgressBarSection.PREPARE_LOCAL_FILES, consts.SCA_SCAN_TYPE, (str(path),)
+            progress_bar, SbomReportProgressBarSection.PREPARE_LOCAL_FILES, consts.SCA_SCAN_TYPE, (str(path),), is_cycodeignore_allowed=is_cycodeignore_allowed
         )
         # TODO(MarshalX): combine perform_pre_scan_documents_actions with get_relevant_document.
         #  unhardcode usage of context in perform_pre_scan_documents_actions

cycode/cli/apps/scan/code_scanner.py

@@ -23,7 +23,7 @@
 from cycode.cli.models import CliError, Document, LocalScanResult
 from cycode.cli.utils.progress_bar import ScanProgressBarSection
 from cycode.cli.utils.scan_batch import run_parallel_batched_scan
-from cycode.cli.utils.scan_utils import generate_unique_scan_id, set_issue_detected_by_scan_results
+from cycode.cli.utils.scan_utils import generate_unique_scan_id, is_cycodeignore_allowed_by_scan_config, set_issue_detected_by_scan_results
 from cycode.cyclient.models import ZippedFileScanResult
 from cycode.logger import get_logger
 
@@ -42,7 +42,8 @@ def scan_disk_files(ctx: typer.Context, paths: tuple[str, ...]) -> None:
     progress_bar = ctx.obj['progress_bar']
 
     try:
-        documents = get_relevant_documents(progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, scan_type, paths)
+        is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+        documents = get_relevant_documents(progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, scan_type, paths, is_cycodeignore_allowed=is_cycodeignore_allowed)
         add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents)
         scan_documents(ctx, documents, get_scan_parameters(ctx, paths))
     except Exception as e:

cycode/cli/apps/scan/commit_range_scanner.py

@@ -41,7 +41,7 @@
 from cycode.cli.utils.git_proxy import git_proxy
 from cycode.cli.utils.path_utils import get_path_by_os
 from cycode.cli.utils.progress_bar import ScanProgressBarSection
-from cycode.cli.utils.scan_utils import generate_unique_scan_id, set_issue_detected_by_scan_results
+from cycode.cli.utils.scan_utils import generate_unique_scan_id, is_cycodeignore_allowed_by_scan_config, set_issue_detected_by_scan_results
 from cycode.cyclient.models import ZippedFileScanResult
 from cycode.logger import get_logger
 
@@ -190,8 +190,9 @@ def _scan_sca_commit_range(ctx: typer.Context, repo_path: str, commit_range: str
     from_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, from_commit_documents)
     to_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, to_commit_documents)
 
-    from_commit_documents = filter_documents_with_cycodeignore(from_commit_documents, repo_path)
-    to_commit_documents = filter_documents_with_cycodeignore(to_commit_documents, repo_path)
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    from_commit_documents = filter_documents_with_cycodeignore(from_commit_documents, repo_path, is_cycodeignore_allowed)
+    to_commit_documents = filter_documents_with_cycodeignore(to_commit_documents, repo_path, is_cycodeignore_allowed)
 
     perform_sca_pre_commit_range_scan_actions(
         repo_path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev
@@ -208,7 +209,8 @@ def _scan_secret_commit_range(
         consts.SECRET_SCAN_TYPE, commit_diff_documents_to_scan
     )
 
-    diff_documents_to_scan = filter_documents_with_cycodeignore(diff_documents_to_scan, repo_path)
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    diff_documents_to_scan = filter_documents_with_cycodeignore(diff_documents_to_scan, repo_path, is_cycodeignore_allowed)
 
     scan_documents(
         ctx, diff_documents_to_scan, get_scan_parameters(ctx, (repo_path,)), is_git_diff=True, is_commit_range=True
@@ -231,8 +233,9 @@ def _scan_sast_commit_range(ctx: typer.Context, repo_path: str, commit_range: st
     commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, commit_documents)
     diff_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, diff_documents)
 
-    commit_documents = filter_documents_with_cycodeignore(commit_documents, repo_path)
-    diff_documents = filter_documents_with_cycodeignore(diff_documents, repo_path)
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    commit_documents = filter_documents_with_cycodeignore(commit_documents, repo_path, is_cycodeignore_allowed)
+    diff_documents = filter_documents_with_cycodeignore(diff_documents, repo_path, is_cycodeignore_allowed)
 
     _scan_commit_range_documents(ctx, commit_documents, diff_documents, scan_parameters=scan_parameters)
 
@@ -270,8 +273,9 @@ def _scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None:
         consts.SCA_SCAN_TYPE, pre_committed_documents
     )
 
-    git_head_documents = filter_documents_with_cycodeignore(git_head_documents, repo_path)
-    pre_committed_documents = filter_documents_with_cycodeignore(pre_committed_documents, repo_path)
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    git_head_documents = filter_documents_with_cycodeignore(git_head_documents, repo_path, is_cycodeignore_allowed)
+    pre_committed_documents = filter_documents_with_cycodeignore(pre_committed_documents, repo_path, is_cycodeignore_allowed)
 
     perform_sca_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents)
 
@@ -305,7 +309,8 @@ def _scan_secret_pre_commit(ctx: typer.Context, repo_path: str) -> None:
 
     documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(consts.SECRET_SCAN_TYPE, documents_to_scan)
 
-    documents_to_scan = filter_documents_with_cycodeignore(documents_to_scan, repo_path)
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    documents_to_scan = filter_documents_with_cycodeignore(documents_to_scan, repo_path, is_cycodeignore_allowed)
 
     scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx), is_git_diff=True)
 
@@ -324,8 +329,9 @@ def _scan_sast_pre_commit(ctx: typer.Context, repo_path: str, **_) -> None:
     )
     diff_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, diff_documents)
 
-    pre_committed_documents = filter_documents_with_cycodeignore(pre_committed_documents, repo_path)
-    diff_documents = filter_documents_with_cycodeignore(diff_documents, repo_path)
+    is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+    pre_committed_documents = filter_documents_with_cycodeignore(pre_committed_documents, repo_path, is_cycodeignore_allowed)
+    diff_documents = filter_documents_with_cycodeignore(diff_documents, repo_path, is_cycodeignore_allowed)
 
     _scan_commit_range_documents(ctx, pre_committed_documents, diff_documents, scan_parameters=scan_parameters)
 

cycode/cli/apps/scan/repository/repository_command.py

@@ -16,6 +16,7 @@
 from cycode.cli.models import Document
 from cycode.cli.utils.path_utils import get_path_by_os
 from cycode.cli.utils.progress_bar import ScanProgressBarSection
+from cycode.cli.utils.scan_utils import is_cycodeignore_allowed_by_scan_config
 from cycode.cli.utils.sentry import add_breadcrumb
 
 
@@ -61,7 +62,8 @@ def repository_command(
 
         documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan)
 
-        documents_to_scan = filter_documents_with_cycodeignore(documents_to_scan, str(path))
+        is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx)
+        documents_to_scan = filter_documents_with_cycodeignore(documents_to_scan, str(path), is_cycodeignore_allowed)
 
         add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents_to_scan)
 

cycode/cli/apps/scan/scan_command.py

@@ -169,6 +169,8 @@ def scan_command(
     remote_scan_config = scan_client.get_scan_configuration_safe(scan_type, remote_url)
     if remote_scan_config:
         excluder.apply_scan_config(str(scan_type), remote_scan_config)
+    
+    ctx.obj['scan_config'] = remote_scan_config
 
     if export_type and export_file:
         console_printer = ctx.obj['console_printer']

cycode/cli/files_collector/documents_walk_ignore.py

@@ -86,13 +86,16 @@ def _filter_documents_by_allowed_paths(
                 logger.debug('Filtered out document due to .cycodeignore: %s', relative_path)
         except Exception as e:
             logger.debug('Error processing document %s: %s', document.path, e)
-            # Include document if we can't determine if it should be ignored
             filtered_documents.append(document)
 
     return filtered_documents
 
 
-def filter_documents_with_cycodeignore(documents: list['Document'], repo_path: str) -> list['Document']:
+def filter_documents_with_cycodeignore(
+    documents: list['Document'], 
+    repo_path: str, 
+    is_cycodeignore_allowed: bool = True
+) -> list['Document']:
     """Filter documents based on .cycodeignore patterns.
     
     This function uses .cycodeignore file in the repository root to filter out
@@ -101,10 +104,15 @@ def filter_documents_with_cycodeignore(documents: list['Document'], repo_path: s
     Args:
         documents: List of Document objects to filter
         repo_path: Path to the repository root
+        is_cycodeignore_allowed: Whether .cycodeignore filtering is allowed by scan configuration
         
     Returns:
         List of Document objects that don't match any .cycodeignore patterns
     """
+    if not is_cycodeignore_allowed:
+        logger.debug('.cycodeignore filtering is not allowed by scan configuration')
+        return documents
+    
     cycodeignore_path = _get_cycodeignore_path(repo_path)
 
     if not os.path.exists(cycodeignore_path):

cycode/cli/files_collector/path_documents.py

@@ -17,18 +17,27 @@
     from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection
 
 
-def _get_all_existing_files_in_directory(path: str, *, walk_with_ignore_patterns: bool = True) -> list[str]:
+def _get_all_existing_files_in_directory(
+    path: str, 
+    *, 
+    walk_with_ignore_patterns: bool = True,
+    is_cycodeignore_allowed: bool = True
+) -> list[str]:
     files: list[str] = []
 
-    walk_func = walk_ignore if walk_with_ignore_patterns else os.walk
+    if walk_with_ignore_patterns:
+        walk_func = lambda p: walk_ignore(p, is_cycodeignore_allowed=is_cycodeignore_allowed)
+    else:
+        walk_func = os.walk
+        
     for root, _, filenames in walk_func(path):
         for filename in filenames:
             files.append(os.path.join(root, filename))
 
     return files
 
 
-def _get_relevant_files_in_path(path: str) -> list[str]:
+def _get_relevant_files_in_path(path: str, *, is_cycodeignore_allowed: bool = True) -> list[str]:
     absolute_path = get_absolute_path(path)
 
     if not os.path.isfile(absolute_path) and not os.path.isdir(absolute_path):
@@ -37,16 +46,21 @@ def _get_relevant_files_in_path(path: str) -> list[str]:
     if os.path.isfile(absolute_path):
         return [absolute_path]
 
-    file_paths = _get_all_existing_files_in_directory(absolute_path)
+    file_paths = _get_all_existing_files_in_directory(absolute_path, is_cycodeignore_allowed=is_cycodeignore_allowed)
     return [file_path for file_path in file_paths if os.path.isfile(file_path)]
 
 
 def _get_relevant_files(
-    progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, paths: tuple[str, ...]
+    progress_bar: 'BaseProgressBar', 
+    progress_bar_section: 'ProgressBarSection', 
+    scan_type: str, 
+    paths: tuple[str, ...],
+    *,
+    is_cycodeignore_allowed: bool = True
 ) -> list[str]:
     all_files_to_scan = []
     for path in paths:
-        all_files_to_scan.extend(_get_relevant_files_in_path(path))
+        all_files_to_scan.extend(_get_relevant_files_in_path(path, is_cycodeignore_allowed=is_cycodeignore_allowed))
 
     # we are double the progress bar section length because we are going to process the files twice
     # first time to get the file list with respect of excluded patterns (excluding takes seconds to execute)
@@ -94,8 +108,9 @@ def get_relevant_documents(
     paths: tuple[str, ...],
     *,
     is_git_diff: bool = False,
+    is_cycodeignore_allowed: bool = True,
 ) -> list[Document]:
-    relevant_files = _get_relevant_files(progress_bar, progress_bar_section, scan_type, paths)
+    relevant_files = _get_relevant_files(progress_bar, progress_bar_section, scan_type, paths, is_cycodeignore_allowed=is_cycodeignore_allowed)
 
     documents: list[Document] = []
     for file in relevant_files:

cycode/cli/files_collector/walk_ignore.py

@@ -10,7 +10,6 @@
 
 _SUPPORTED_IGNORE_PATTERN_FILES = {
     '.gitignore',
-    consts.CYCODEIGNORE_FILENAME,
 }
 _DEFAULT_GLOBAL_IGNORE_PATTERNS = [
     '.git',
@@ -27,22 +26,29 @@ def _walk_to_top(path: str) -> Iterable[str]:
         yield path  # Include the top-level directory
 
 
-def _collect_top_level_ignore_files(path: str) -> list[str]:
+def _collect_top_level_ignore_files(path: str, *, is_cycodeignore_allowed: bool = True) -> list[str]:
     ignore_files = []
     top_paths = reversed(list(_walk_to_top(path)))  # we must reverse it to make top levels more prioritized
+    
+    supported_files = {_SUPPORTED_IGNORE_PATTERN_FILES}
+    if is_cycodeignore_allowed:
+        supported_files.add(consts.CYCODEIGNORE_FILENAME)
+        logger.debug('.cycodeignore files included due to scan configuration')
+    
     for dir_path in top_paths:
-        for ignore_file in _SUPPORTED_IGNORE_PATTERN_FILES:
+        for ignore_file in supported_files:
             ignore_file_path = os.path.join(dir_path, ignore_file)
             if os.path.exists(ignore_file_path):
                 logger.debug('Reading top level ignore file: %s', ignore_file_path)
                 ignore_files.append(ignore_file_path)
     return ignore_files
 
 
-def walk_ignore(path: str) -> Generator[tuple[str, list[str], list[str]], None, None]:
+def walk_ignore(path: str, *, is_cycodeignore_allowed: bool = True) -> Generator[tuple[str, list[str], list[str]], None, None]:
+    ignore_file_paths = _collect_top_level_ignore_files(path, is_cycodeignore_allowed=is_cycodeignore_allowed)
     ignore_filter_manager = IgnoreFilterManager.build(
         path=path,
-        global_ignore_file_paths=_collect_top_level_ignore_files(path),
+        global_ignore_file_paths=ignore_file_paths,
         global_patterns=_DEFAULT_GLOBAL_IGNORE_PATTERNS,
     )
     for dirpath, dirnames, filenames, ignored_dirnames, ignored_filenames in ignore_filter_manager.walk_with_ignored():

cycode/cli/utils/scan_utils.py

@@ -1,11 +1,12 @@
 import os
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 from uuid import UUID, uuid4
 
 import typer
 
 if TYPE_CHECKING:
     from cycode.cli.models import LocalScanResult
+    from cycode.cyclient.models import ScanConfiguration
 
 
 def set_issue_detected(ctx: typer.Context, issue_detected: bool) -> None:
@@ -22,6 +23,11 @@ def is_scan_failed(ctx: typer.Context) -> bool:
     return did_fail or issue_detected
 
 
+def is_cycodeignore_allowed_by_scan_config(ctx: typer.Context) -> bool:
+    scan_config: Optional['ScanConfiguration'] = ctx.obj.get('scan_config')
+    return scan_config.is_repository_ignore_configuration_allowed if scan_config else True
+
+
 def generate_unique_scan_id() -> UUID:
     if 'PYTEST_TEST_UNIQUE_ID' in os.environ:
         return UUID(os.environ['PYTEST_TEST_UNIQUE_ID'])