Coverage for C:\Repos\ekr-pylint\pylint\lint\pylinter.py: 20%
470 statements
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
1# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
2# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
3# Copyright (c) https://github.com/PyCQA/pylint/blob/main/CONTRIBUTORS.txt
5from __future__ import annotations
7import collections
8import contextlib
9import functools
10import os
11import sys
12import tokenize
13import traceback
14import warnings
15from collections import defaultdict
16from collections.abc import Callable, Iterable, Iterator, Sequence
17from io import TextIOWrapper
18from typing import Any
20import astroid
21from astroid import AstroidError, nodes
23from pylint import checkers, exceptions, interfaces, reporters
24from pylint.checkers.base_checker import BaseChecker
25from pylint.config.arguments_manager import _ArgumentsManager
26from pylint.constants import (
27 MAIN_CHECKER_NAME,
28 MSG_TYPES,
29 MSG_TYPES_STATUS,
30 WarningScope,
31)
32from pylint.lint.base_options import _make_linter_options
33from pylint.lint.caching import load_results, save_results
34from pylint.lint.expand_modules import expand_modules
35from pylint.lint.message_state_handler import _MessageStateHandler
36from pylint.lint.parallel import check_parallel
37from pylint.lint.report_functions import (
38 report_messages_by_module_stats,
39 report_messages_stats,
40 report_total_messages_stats,
41)
42from pylint.lint.utils import (
43 fix_import_path,
44 get_fatal_error_message,
45 prepare_crash_report,
46)
47from pylint.message import Message, MessageDefinition, MessageDefinitionStore
48from pylint.reporters.base_reporter import BaseReporter
49from pylint.reporters.text import TextReporter
50from pylint.reporters.ureports import nodes as report_nodes
51from pylint.typing import (
52 FileItem,
53 ManagedMessage,
54 MessageDefinitionTuple,
55 MessageLocationTuple,
56 ModuleDescriptionDict,
57 Options,
58)
59from pylint.utils import ASTWalker, FileState, LinterStats, utils
61if sys.version_info >= (3, 8):
62 from typing import Protocol
63else:
64 from typing_extensions import Protocol
67MANAGER = astroid.MANAGER
70class GetAstProtocol(Protocol):
71 def __call__(
72 self, filepath: str, modname: str, data: str | None = None
73 ) -> nodes.Module:
74 ...
77def _read_stdin() -> str:
78 # See https://github.com/python/typeshed/pull/5623 for rationale behind assertion
79 assert isinstance(sys.stdin, TextIOWrapper)
80 sys.stdin = TextIOWrapper(sys.stdin.detach(), encoding="utf-8")
81 return sys.stdin.read()
84def _load_reporter_by_class(reporter_class: str) -> type[BaseReporter]:
85 qname = reporter_class
86 module_part = astroid.modutils.get_module_part(qname)
87 module = astroid.modutils.load_module_from_name(module_part)
88 class_name = qname.split(".")[-1]
89 klass = getattr(module, class_name)
90 assert issubclass(klass, BaseReporter), f"{klass} is not a BaseReporter"
91 return klass
94# Python Linter class #########################################################
96# pylint: disable-next=consider-using-namedtuple-or-dataclass
97MSGS: dict[str, MessageDefinitionTuple] = {
98 "F0001": (
99 "%s",
100 "fatal",
101 "Used when an error occurred preventing the analysis of a \
102 module (unable to find it for instance).",
103 {"scope": WarningScope.LINE},
104 ),
105 "F0002": (
106 "%s: %s",
107 "astroid-error",
108 "Used when an unexpected error occurred while building the "
109 "Astroid representation. This is usually accompanied by a "
110 "traceback. Please report such errors !",
111 {"scope": WarningScope.LINE},
112 ),
113 "F0010": (
114 "error while code parsing: %s",
115 "parse-error",
116 "Used when an exception occurred while building the Astroid "
117 "representation which could be handled by astroid.",
118 {"scope": WarningScope.LINE},
119 ),
120 "F0011": (
121 "error while parsing the configuration: %s",
122 "config-parse-error",
123 "Used when an exception occurred while parsing a pylint configuration file.",
124 {"scope": WarningScope.LINE},
125 ),
126 "I0001": (
127 "Unable to run raw checkers on built-in module %s",
128 "raw-checker-failed",
129 "Used to inform that a built-in module has not been checked "
130 "using the raw checkers.",
131 {"scope": WarningScope.LINE},
132 ),
133 "I0010": (
134 "Unable to consider inline option %r",
135 "bad-inline-option",
136 "Used when an inline option is either badly formatted or can't "
137 "be used inside modules.",
138 {"scope": WarningScope.LINE},
139 ),
140 "I0011": (
141 "Locally disabling %s (%s)",
142 "locally-disabled",
143 "Used when an inline option disables a message or a messages category.",
144 {"scope": WarningScope.LINE},
145 ),
146 "I0013": (
147 "Ignoring entire file",
148 "file-ignored",
149 "Used to inform that the file will not be checked",
150 {"scope": WarningScope.LINE},
151 ),
152 "I0020": (
153 "Suppressed %s (from line %d)",
154 "suppressed-message",
155 "A message was triggered on a line, but suppressed explicitly "
156 "by a disable= comment in the file. This message is not "
157 "generated for messages that are ignored due to configuration "
158 "settings.",
159 {"scope": WarningScope.LINE},
160 ),
161 "I0021": (
162 "Useless suppression of %s",
163 "useless-suppression",
164 "Reported when a message is explicitly disabled for a line or "
165 "a block of code, but never triggered.",
166 {"scope": WarningScope.LINE},
167 ),
168 "I0022": (
169 'Pragma "%s" is deprecated, use "%s" instead',
170 "deprecated-pragma",
171 "Some inline pylint options have been renamed or reworked, "
172 "only the most recent form should be used. "
173 "NOTE:skip-all is only available with pylint >= 0.26",
174 {
175 "old_names": [("I0014", "deprecated-disable-all")],
176 "scope": WarningScope.LINE,
177 },
178 ),
179 "E0001": (
180 "%s",
181 "syntax-error",
182 "Used when a syntax error is raised for a module.",
183 {"scope": WarningScope.LINE},
184 ),
185 "E0011": (
186 "Unrecognized file option %r",
187 "unrecognized-inline-option",
188 "Used when an unknown inline option is encountered.",
189 {"scope": WarningScope.LINE},
190 ),
191 "E0012": (
192 "Bad option value for %s",
193 "bad-option-value",
194 "Used when a bad value for an inline option is encountered.",
195 {"scope": WarningScope.LINE},
196 ),
197 "E0013": (
198 "Plugin '%s' is impossible to load, is it installed ? ('%s')",
199 "bad-plugin-value",
200 "Used when a bad value is used in 'load-plugins'.",
201 {"scope": WarningScope.LINE},
202 ),
203 "E0014": (
204 "Out-of-place setting encountered in top level configuration-section '%s' : '%s'",
205 "bad-configuration-section",
206 "Used when we detect a setting in the top level of a toml configuration that shouldn't be there.",
207 {"scope": WarningScope.LINE},
208 ),
209 "E0015": (
210 "Unrecognized option found: %s",
211 "unrecognized-option",
212 "Used when we detect an option that we do not recognize.",
213 {"scope": WarningScope.LINE},
214 ),
215}
218# pylint: disable=too-many-instance-attributes,too-many-public-methods
219class PyLinter(
220 _ArgumentsManager,
221 _MessageStateHandler,
222 reporters.ReportsHandlerMixIn,
223 checkers.BaseChecker,
224):
225 """Lint Python modules using external checkers.
227 This is the main checker controlling the other ones and the reports
228 generation. It is itself both a raw checker and an astroid checker in order
229 to:
230 * handle message activation / deactivation at the module level
231 * handle some basic but necessary stats' data (number of classes, methods...)
233 IDE plugin developers: you may have to call
234 `astroid.builder.MANAGER.astroid_cache.clear()` across runs if you want
235 to ensure the latest code version is actually checked.
237 This class needs to support pickling for parallel linting to work. The exception
238 is reporter member; see check_parallel function for more details.
239 """
241 name = MAIN_CHECKER_NAME
242 msgs = MSGS
243 # Will be used like this : datetime.now().strftime(crash_file_path)
244 crash_file_path: str = "pylint-crash-%Y-%m-%d-%H.txt"
246 option_groups_descs = {
247 "Messages control": "Options controlling analysis messages",
248 "Reports": "Options related to output formatting and reporting",
249 }
251 def __init__(
252 self,
253 options: Options = (),
254 reporter: reporters.BaseReporter | reporters.MultiReporter | None = None,
255 option_groups: tuple[tuple[str, str], ...] = (),
256 # TODO: Deprecate passing the pylintrc parameter
257 pylintrc: str | None = None, # pylint: disable=unused-argument
258 ) -> None:
259 _ArgumentsManager.__init__(self, prog="pylint")
260 _MessageStateHandler.__init__(self, self)
262 # Some stuff has to be done before initialization of other ancestors...
263 # messages store / checkers / reporter / astroid manager
265 # Attributes for reporters
266 self.reporter: reporters.BaseReporter | reporters.MultiReporter
267 if reporter:
268 self.set_reporter(reporter)
269 else:
270 self.set_reporter(TextReporter())
271 self._reporters: dict[str, type[reporters.BaseReporter]] = {}
272 """Dictionary of possible but non-initialized reporters."""
274 # Attributes for checkers and plugins
275 self._checkers: defaultdict[
276 str, list[checkers.BaseChecker]
277 ] = collections.defaultdict(list)
278 """Dictionary of registered and initialized checkers."""
279 self._dynamic_plugins: set[str] = set()
280 """Set of loaded plugin names."""
282 # Attributes related to registering messages and their handling
283 self.msgs_store = MessageDefinitionStore()
284 self.msg_status = 0
285 self._by_id_managed_msgs: list[ManagedMessage] = []
287 # Attributes related to visiting files
288 self.file_state = FileState("", self.msgs_store, is_base_filestate=True)
289 self.current_name: str | None = None
290 self.current_file: str | None = None
291 self._ignore_file = False
293 # Attributes related to stats
294 self.stats = LinterStats()
296 # Attributes related to (command-line) options and their parsing
297 self.options: Options = options + _make_linter_options(self)
298 for opt_group in option_groups:
299 self.option_groups_descs[opt_group[0]] = opt_group[1]
300 self._option_groups: tuple[tuple[str, str], ...] = option_groups + (
301 ("Messages control", "Options controlling analysis messages"),
302 ("Reports", "Options related to output formatting and reporting"),
303 )
304 self.fail_on_symbols: list[str] = []
305 """List of message symbols on which pylint should fail, set by --fail-on."""
306 self._error_mode = False
308 reporters.ReportsHandlerMixIn.__init__(self)
309 checkers.BaseChecker.__init__(self, self)
310 # provided reports
311 self.reports = (
312 ("RP0001", "Messages by category", report_total_messages_stats),
313 (
314 "RP0002",
315 "% errors / warnings by module",
316 report_messages_by_module_stats,
317 ),
318 ("RP0003", "Messages", report_messages_stats),
319 )
320 self.register_checker(self)
322 @property
323 def option_groups(self) -> tuple[tuple[str, str], ...]:
324 # TODO: 3.0: Remove deprecated attribute
325 warnings.warn(
326 "The option_groups attribute has been deprecated and will be removed in pylint 3.0",
327 DeprecationWarning,
328 )
329 return self._option_groups
331 @option_groups.setter
332 def option_groups(self, value: tuple[tuple[str, str], ...]) -> None:
333 warnings.warn(
334 "The option_groups attribute has been deprecated and will be removed in pylint 3.0",
335 DeprecationWarning,
336 )
337 self._option_groups = value
339 def load_default_plugins(self) -> None:
340 checkers.initialize(self)
341 reporters.initialize(self)
343 def load_plugin_modules(self, modnames: list[str]) -> None:
344 """Check a list pylint plugins modules, load and register them."""
345 for modname in modnames:
346 if modname in self._dynamic_plugins:
347 continue
348 self._dynamic_plugins.add(modname)
349 try:
350 module = astroid.modutils.load_module_from_name(modname)
351 module.register(self)
352 except ModuleNotFoundError:
353 pass
355 def load_plugin_configuration(self) -> None:
356 """Call the configuration hook for plugins.
358 This walks through the list of plugins, grabs the "load_configuration"
359 hook, if exposed, and calls it to allow plugins to configure specific
360 settings.
361 """
362 for modname in self._dynamic_plugins:
363 try:
364 module = astroid.modutils.load_module_from_name(modname)
365 if hasattr(module, "load_configuration"):
366 module.load_configuration(self)
367 except ModuleNotFoundError as e:
368 self.add_message("bad-plugin-value", args=(modname, e), line=0)
370 def _load_reporters(self, reporter_names: str) -> None:
371 """Load the reporters if they are available on _reporters."""
372 if not self._reporters:
373 return
374 sub_reporters = []
375 output_files = []
376 with contextlib.ExitStack() as stack:
377 for reporter_name in reporter_names.split(","):
378 reporter_name, *reporter_output = reporter_name.split(":", 1)
380 reporter = self._load_reporter_by_name(reporter_name)
381 sub_reporters.append(reporter)
382 if reporter_output:
383 output_file = stack.enter_context(
384 open(reporter_output[0], "w", encoding="utf-8")
385 )
386 reporter.out = output_file
387 output_files.append(output_file)
389 # Extend the lifetime of all opened output files
390 close_output_files = stack.pop_all().close
392 if len(sub_reporters) > 1 or output_files:
393 self.set_reporter(
394 reporters.MultiReporter(
395 sub_reporters,
396 close_output_files,
397 )
398 )
399 else:
400 self.set_reporter(sub_reporters[0])
402 def _load_reporter_by_name(self, reporter_name: str) -> reporters.BaseReporter:
403 name = reporter_name.lower()
404 if name in self._reporters:
405 return self._reporters[name]()
407 try:
408 reporter_class = _load_reporter_by_class(reporter_name)
409 except (ImportError, AttributeError, AssertionError) as e:
410 raise exceptions.InvalidReporterError(name) from e
411 else:
412 return reporter_class()
414 def set_reporter(
415 self, reporter: reporters.BaseReporter | reporters.MultiReporter
416 ) -> None:
417 """Set the reporter used to display messages and reports."""
418 self.reporter = reporter
419 reporter.linter = self
421 def register_reporter(self, reporter_class: type[reporters.BaseReporter]) -> None:
422 """Registers a reporter class on the _reporters attribute."""
423 self._reporters[reporter_class.name] = reporter_class
425 def report_order(self) -> list[BaseChecker]:
426 reports = sorted(self._reports, key=lambda x: getattr(x, "name", ""))
427 try:
428 # Remove the current reporter and add it
429 # at the end of the list.
430 reports.pop(reports.index(self))
431 except ValueError:
432 pass
433 else:
434 reports.append(self)
435 return reports
437 # checkers manipulation methods ############################################
439 def register_checker(self, checker: checkers.BaseChecker) -> None:
440 """This method auto registers the checker."""
441 self._checkers[checker.name].append(checker)
442 for r_id, r_title, r_cb in checker.reports:
443 self.register_report(r_id, r_title, r_cb, checker)
444 if hasattr(checker, "msgs"):
445 self.msgs_store.register_messages_from_checker(checker)
446 # Register the checker, but disable all of its messages.
447 if not getattr(checker, "enabled", True):
448 self.disable(checker.name)
450 def enable_fail_on_messages(self) -> None:
451 """Enable 'fail on' msgs.
453 Convert values in config.fail_on (which might be msg category, msg id,
454 or symbol) to specific msgs, then enable and flag them for later.
455 """
456 fail_on_vals = self.config.fail_on
457 if not fail_on_vals:
458 return
460 fail_on_cats = set()
461 fail_on_msgs = set()
462 for val in fail_on_vals:
463 # If value is a category, add category, else add message
464 if val in MSG_TYPES:
465 fail_on_cats.add(val)
466 else:
467 fail_on_msgs.add(val)
469 # For every message in every checker, if cat or msg flagged, enable check
470 for all_checkers in self._checkers.values():
471 for checker in all_checkers:
472 for msg in checker.messages:
473 if msg.msgid in fail_on_msgs or msg.symbol in fail_on_msgs:
474 # message id/symbol matched, enable and flag it
475 self.enable(msg.msgid)
476 self.fail_on_symbols.append(msg.symbol)
477 elif msg.msgid[0] in fail_on_cats:
478 # message starts with a category value, flag (but do not enable) it
479 self.fail_on_symbols.append(msg.symbol)
481 def any_fail_on_issues(self) -> bool:
482 return any(x in self.fail_on_symbols for x in self.stats.by_msg.keys())
484 def disable_reporters(self) -> None:
485 """Disable all reporters."""
486 for _reporters in self._reports.values():
487 for report_id, _, _ in _reporters:
488 self.disable_report(report_id)
490 def _parse_error_mode(self) -> None:
491 """Parse the current state of the error mode.
493 Error mode: enable only errors; no reports, no persistent.
494 """
495 if not self._error_mode:
496 return
498 self.disable_noerror_messages()
499 self.disable("miscellaneous")
500 self.set_option("reports", False)
501 self.set_option("persistent", False)
502 self.set_option("score", False)
504 # code checking methods ###################################################
506 def get_checkers(self) -> list[BaseChecker]:
507 """Return all available checkers as an ordered list."""
508 return sorted(c for _checkers in self._checkers.values() for c in _checkers)
510 def get_checker_names(self) -> list[str]:
511 """Get all the checker names that this linter knows about."""
512 return sorted(
513 {
514 checker.name
515 for checker in self.get_checkers()
516 if checker.name != MAIN_CHECKER_NAME
517 }
518 )
520 def prepare_checkers(self) -> list[BaseChecker]:
521 """Return checkers needed for activated messages and reports."""
522 if not self.config.reports:
523 self.disable_reporters()
524 # get needed checkers
525 needed_checkers: list[BaseChecker] = [self]
526 for checker in self.get_checkers()[1:]:
527 messages = {msg for msg in checker.msgs if self.is_message_enabled(msg)}
528 if messages or any(self.report_is_enabled(r[0]) for r in checker.reports):
529 needed_checkers.append(checker)
530 return needed_checkers
532 # pylint: disable=unused-argument
533 @staticmethod
534 def should_analyze_file(modname: str, path: str, is_argument: bool = False) -> bool:
535 """Returns whether a module should be checked.
537 This implementation returns True for all python source file, indicating
538 that all files should be linted.
540 Subclasses may override this method to indicate that modules satisfying
541 certain conditions should not be linted.
543 :param str modname: The name of the module to be checked.
544 :param str path: The full path to the source code of the module.
545 :param bool is_argument: Whether the file is an argument to pylint or not.
546 Files which respect this property are always
547 checked, since the user requested it explicitly.
548 :returns: True if the module should be checked.
549 """
550 if is_argument:
551 return True
552 return path.endswith(".py")
554 # pylint: enable=unused-argument
556 def initialize(self) -> None:
557 """Initialize linter for linting.
559 This method is called before any linting is done.
560 """
561 # initialize msgs_state now that all messages have been registered into
562 # the store
563 for msg in self.msgs_store.messages:
564 if not msg.may_be_emitted():
565 self._msgs_state[msg.msgid] = False
567 @staticmethod
568 def _discover_files(files_or_modules: Sequence[str]) -> Iterator[str]:
569 """Discover python modules and packages in sub-directory.
571 Returns iterator of paths to discovered modules and packages.
572 """
573 for something in files_or_modules:
574 if os.path.isdir(something) and not os.path.isfile(
575 os.path.join(something, "__init__.py")
576 ):
577 skip_subtrees: list[str] = []
578 for root, _, files in os.walk(something):
579 if any(root.startswith(s) for s in skip_subtrees):
580 # Skip subtree of already discovered package.
581 continue
582 if "__init__.py" in files:
583 skip_subtrees.append(root)
584 yield root
585 else:
586 yield from (
587 os.path.join(root, file)
588 for file in files
589 if file.endswith(".py")
590 )
591 else:
592 yield something
594 def check(self, files_or_modules: Sequence[str] | str) -> None:
595 """Main checking entry: check a list of files or modules from their name.
597 files_or_modules is either a string or list of strings presenting modules to check.
598 """
599 self.initialize()
600 if not isinstance(files_or_modules, (list, tuple)):
601 # TODO: 3.0: Remove deprecated typing and update docstring
602 warnings.warn(
603 "In pylint 3.0, the checkers check function will only accept sequence of string",
604 DeprecationWarning,
605 )
606 files_or_modules = (files_or_modules,) # type: ignore[assignment]
607 if self.config.recursive:
608 files_or_modules = tuple(self._discover_files(files_or_modules))
609 if self.config.from_stdin:
610 if len(files_or_modules) != 1:
611 raise exceptions.InvalidArgsError(
612 "Missing filename required for --from-stdin"
613 )
615 filepath = files_or_modules[0]
616 with fix_import_path(files_or_modules):
617 self._check_files(
618 functools.partial(self.get_ast, data=_read_stdin()),
619 [self._get_file_descr_from_stdin(filepath)],
620 )
621 elif self.config.jobs == 1:
622 with fix_import_path(files_or_modules):
623 self._check_files(
624 self.get_ast, self._iterate_file_descrs(files_or_modules)
625 )
626 else:
627 check_parallel(
628 self,
629 self.config.jobs,
630 self._iterate_file_descrs(files_or_modules),
631 files_or_modules,
632 )
634 def check_single_file(self, name: str, filepath: str, modname: str) -> None:
635 warnings.warn(
636 "In pylint 3.0, the checkers check_single_file function will be removed. "
637 "Use check_single_file_item instead.",
638 DeprecationWarning,
639 )
640 self.check_single_file_item(FileItem(name, filepath, modname))
642 def check_single_file_item(self, file: FileItem) -> None:
643 """Check single file item.
645 The arguments are the same that are documented in _check_files
647 initialize() should be called before calling this method
648 """
649 with self._astroid_module_checker() as check_astroid_module:
650 self._check_file(self.get_ast, check_astroid_module, file)
652 def _check_files(
653 self,
654 get_ast: GetAstProtocol,
655 file_descrs: Iterable[FileItem],
656 ) -> None:
657 """Check all files from file_descrs."""
658 with self._astroid_module_checker() as check_astroid_module:
659 for file in file_descrs:
660 try:
661 self._check_file(get_ast, check_astroid_module, file)
662 except Exception as ex: # pylint: disable=broad-except
663 template_path = prepare_crash_report(
664 ex, file.filepath, self.crash_file_path
665 )
666 msg = get_fatal_error_message(file.filepath, template_path)
667 if isinstance(ex, AstroidError):
668 symbol = "astroid-error"
669 self.add_message(symbol, args=(file.filepath, msg))
670 else:
671 symbol = "fatal"
672 self.add_message(symbol, args=msg)
674 def _check_file(
675 self,
676 get_ast: GetAstProtocol,
677 check_astroid_module: Callable[[nodes.Module], bool | None],
678 file: FileItem,
679 ) -> None:
680 """Check a file using the passed utility functions (get_ast and check_astroid_module).
682 :param callable get_ast: callable returning AST from defined file taking the following arguments
683 - filepath: path to the file to check
684 - name: Python module name
685 :param callable check_astroid_module: callable checking an AST taking the following arguments
686 - ast: AST of the module
687 :param FileItem file: data about the file
688 """
689 self.set_current_module(file.name, file.filepath)
690 # get the module representation
691 ast_node = get_ast(file.filepath, file.name)
692 if ast_node is None:
693 return
695 self._ignore_file = False
697 self.file_state = FileState(file.modpath, self.msgs_store, ast_node)
698 # fix the current file (if the source file was not available or
699 # if it's actually a c extension)
700 self.current_file = ast_node.file
701 check_astroid_module(ast_node)
702 # warn about spurious inline messages handling
703 spurious_messages = self.file_state.iter_spurious_suppression_messages(
704 self.msgs_store
705 )
706 for msgid, line, args in spurious_messages:
707 self.add_message(msgid, line, None, args)
709 @staticmethod
710 def _get_file_descr_from_stdin(filepath: str) -> FileItem:
711 """Return file description (tuple of module name, file path, base name) from given file path.
713 This method is used for creating suitable file description for _check_files when the
714 source is standard input.
715 """
716 try:
717 # Note that this function does not really perform an
718 # __import__ but may raise an ImportError exception, which
719 # we want to catch here.
720 modname = ".".join(astroid.modutils.modpath_from_file(filepath))
721 except ImportError:
722 modname = os.path.splitext(os.path.basename(filepath))[0]
724 return FileItem(modname, filepath, filepath)
726 def _iterate_file_descrs(
727 self, files_or_modules: Sequence[str]
728 ) -> Iterator[FileItem]:
729 """Return generator yielding file descriptions (tuples of module name, file path, base name).
731 The returned generator yield one item for each Python module that should be linted.
732 """
733 for descr in self._expand_files(files_or_modules):
734 name, filepath, is_arg = descr["name"], descr["path"], descr["isarg"]
735 if self.should_analyze_file(name, filepath, is_argument=is_arg):
736 yield FileItem(name, filepath, descr["basename"])
738 def _expand_files(self, modules: Sequence[str]) -> list[ModuleDescriptionDict]:
739 """Get modules and errors from a list of modules and handle errors."""
740 result, errors = expand_modules(
741 modules,
742 self.config.ignore,
743 self.config.ignore_patterns,
744 self._ignore_paths,
745 )
746 for error in errors:
747 message = modname = error["mod"]
748 key = error["key"]
749 self.set_current_module(modname)
750 if key == "fatal":
751 message = str(error["ex"]).replace(os.getcwd() + os.sep, "")
752 self.add_message(key, args=message)
753 return result
755 def set_current_module(
756 self, modname: str | None, filepath: str | None = None
757 ) -> None:
758 """Set the name of the currently analyzed module and
759 init statistics for it.
760 """
761 if not modname and filepath is None:
762 return
763 self.reporter.on_set_current_module(modname or "", filepath)
764 if modname is None:
765 # TODO: 3.0: Remove all modname or ""'s in this method
766 warnings.warn(
767 (
768 "In pylint 3.0 modname should be a string so that it can be used to "
769 "correctly set the current_name attribute of the linter instance. "
770 "If unknown it should be initialized as an empty string."
771 ),
772 DeprecationWarning,
773 )
774 self.current_name = modname
775 self.current_file = filepath or modname
776 self.stats.init_single_module(modname or "")
778 @contextlib.contextmanager
779 def _astroid_module_checker(
780 self,
781 ) -> Iterator[Callable[[nodes.Module], bool | None]]:
782 """Context manager for checking ASTs.
784 The value in the context is callable accepting AST as its only argument.
785 """
786 walker = ASTWalker(self)
787 _checkers = self.prepare_checkers()
788 tokencheckers = [
789 c
790 for c in _checkers
791 if isinstance(c, checkers.BaseTokenChecker) and c is not self
792 ]
793 # TODO: 3.0: Remove deprecated for-loop
794 for c in _checkers:
795 with warnings.catch_warnings():
796 warnings.filterwarnings("ignore", category=DeprecationWarning)
797 if (
798 interfaces.implements(c, interfaces.ITokenChecker)
799 and c not in tokencheckers
800 and c is not self
801 ):
802 tokencheckers.append(c) # type: ignore[arg-type] # pragma: no cover
803 warnings.warn( # pragma: no cover
804 "Checkers should subclass BaseTokenChecker "
805 "instead of using the __implements__ mechanism. Use of __implements__ "
806 "will no longer be supported in pylint 3.0",
807 DeprecationWarning,
808 )
809 rawcheckers = [
810 c for c in _checkers if isinstance(c, checkers.BaseRawFileChecker)
811 ]
812 # TODO: 3.0: Remove deprecated if-statement
813 for c in _checkers:
814 with warnings.catch_warnings():
815 warnings.filterwarnings("ignore", category=DeprecationWarning)
816 if (
817 interfaces.implements(c, interfaces.IRawChecker)
818 and c not in rawcheckers
819 ):
820 rawcheckers.append(c) # type: ignore[arg-type] # pragma: no cover
821 warnings.warn( # pragma: no cover
822 "Checkers should subclass BaseRawFileChecker "
823 "instead of using the __implements__ mechanism. Use of __implements__ "
824 "will no longer be supported in pylint 3.0",
825 DeprecationWarning,
826 )
827 # notify global begin
828 for checker in _checkers:
829 checker.open()
830 walker.add_checker(checker)
832 yield functools.partial(
833 self.check_astroid_module,
834 walker=walker,
835 tokencheckers=tokencheckers,
836 rawcheckers=rawcheckers,
837 )
839 # notify global end
840 self.stats.statement = walker.nbstatements
841 for checker in reversed(_checkers):
842 checker.close()
844 def get_ast(
845 self, filepath: str, modname: str, data: str | None = None
846 ) -> nodes.Module:
847 """Return an ast(roid) representation of a module or a string.
849 :param str filepath: path to checked file.
850 :param str modname: The name of the module to be checked.
851 :param str data: optional contents of the checked file.
852 :returns: the AST
853 :rtype: astroid.nodes.Module
854 :raises AstroidBuildingError: Whenever we encounter an unexpected exception
855 """
856 try:
857 if data is None:
858 return MANAGER.ast_from_file(filepath, modname, source=True)
859 return astroid.builder.AstroidBuilder(MANAGER).string_build(
860 data, modname, filepath
861 )
862 except astroid.AstroidSyntaxError as ex:
863 # pylint: disable=no-member
864 self.add_message(
865 "syntax-error",
866 line=getattr(ex.error, "lineno", 0),
867 col_offset=getattr(ex.error, "offset", None),
868 args=str(ex.error),
869 )
870 except astroid.AstroidBuildingError as ex:
871 self.add_message("parse-error", args=ex)
872 except Exception as ex:
873 traceback.print_exc()
874 # We raise BuildingError here as this is essentially an astroid issue
875 # Creating an issue template and adding the 'astroid-error' message is handled
876 # by caller: _check_files
877 raise astroid.AstroidBuildingError(
878 "Building error when trying to create ast representation of module '{modname}'",
879 modname=modname,
880 ) from ex
881 return None
883 def check_astroid_module(
884 self,
885 ast_node: nodes.Module,
886 walker: ASTWalker,
887 rawcheckers: list[checkers.BaseRawFileChecker],
888 tokencheckers: list[checkers.BaseTokenChecker],
889 ) -> bool | None:
890 """Check a module from its astroid representation.
892 For return value see _check_astroid_module
893 """
894 before_check_statements = walker.nbstatements
896 retval = self._check_astroid_module(
897 ast_node, walker, rawcheckers, tokencheckers
898 )
900 # TODO: 3.0: Remove unnecessary assertion
901 assert self.current_name
903 self.stats.by_module[self.current_name]["statement"] = (
904 walker.nbstatements - before_check_statements
905 )
907 return retval
909 def _check_astroid_module(
910 self,
911 node: nodes.Module,
912 walker: ASTWalker,
913 rawcheckers: list[checkers.BaseRawFileChecker],
914 tokencheckers: list[checkers.BaseTokenChecker],
915 ) -> bool | None:
916 """Check given AST node with given walker and checkers.
918 :param astroid.nodes.Module node: AST node of the module to check
919 :param pylint.utils.ast_walker.ASTWalker walker: AST walker
920 :param list rawcheckers: List of token checkers to use
921 :param list tokencheckers: List of raw checkers to use
923 :returns: True if the module was checked, False if ignored,
924 None if the module contents could not be parsed
925 """
926 try:
927 tokens = utils.tokenize_module(node)
928 except tokenize.TokenError as ex:
929 self.add_message("syntax-error", line=ex.args[1][0], args=ex.args[0])
930 return None
932 if not node.pure_python:
933 self.add_message("raw-checker-failed", args=node.name)
934 else:
935 # assert astroid.file.endswith('.py')
936 # Parse module/block level option pragma's
937 self.process_tokens(tokens)
938 if self._ignore_file:
939 return False
940 # run raw and tokens checkers
941 for raw_checker in rawcheckers:
942 raw_checker.process_module(node)
943 for token_checker in tokencheckers:
944 token_checker.process_tokens(tokens)
945 # generate events to astroid checkers
946 walker.walk(node)
947 return True
949 def open(self) -> None:
950 """Initialize counters."""
951 self.stats = LinterStats()
952 MANAGER.always_load_extensions = self.config.unsafe_load_any_extension
953 MANAGER.max_inferable_values = self.config.limit_inference_results
954 MANAGER.extension_package_whitelist.update(self.config.extension_pkg_allow_list)
955 if self.config.extension_pkg_whitelist:
956 MANAGER.extension_package_whitelist.update(
957 self.config.extension_pkg_whitelist
958 )
959 self.stats.reset_message_count()
960 self._ignore_paths = self.linter.config.ignore_paths
962 def generate_reports(self) -> int | None:
963 """Close the whole package /module, it's time to make reports !
965 if persistent run, pickle results for later comparison
966 """
967 # Display whatever messages are left on the reporter.
968 self.reporter.display_messages(report_nodes.Section())
970 # TODO: 3.0: Remove second half of if-statement
971 if (
972 not self.file_state._is_base_filestate
973 and self.file_state.base_name is not None
974 ):
975 # load previous results if any
976 previous_stats = load_results(self.file_state.base_name)
977 self.reporter.on_close(self.stats, previous_stats)
978 if self.config.reports:
979 sect = self.make_reports(self.stats, previous_stats)
980 else:
981 sect = report_nodes.Section()
983 if self.config.reports:
984 self.reporter.display_reports(sect)
985 score_value = self._report_evaluation()
986 # save results if persistent run
987 if self.config.persistent:
988 save_results(self.stats, self.file_state.base_name)
989 else:
990 self.reporter.on_close(self.stats, LinterStats())
991 score_value = None
992 return score_value
994 def _report_evaluation(self) -> int | None:
995 """Make the global evaluation report."""
996 # check with at least check 1 statements (usually 0 when there is a
997 # syntax error preventing pylint from further processing)
998 note = None
999 # TODO: 3.0: Remove assertion
1000 assert self.file_state.base_name is not None
1001 previous_stats = load_results(self.file_state.base_name)
1002 if self.stats.statement == 0:
1003 return note
1005 # get a global note for the code
1006 evaluation = self.config.evaluation
1007 try:
1008 stats_dict = {
1009 "fatal": self.stats.fatal,
1010 "error": self.stats.error,
1011 "warning": self.stats.warning,
1012 "refactor": self.stats.refactor,
1013 "convention": self.stats.convention,
1014 "statement": self.stats.statement,
1015 "info": self.stats.info,
1016 }
1017 note = eval(evaluation, {}, stats_dict) # pylint: disable=eval-used
1018 except Exception as ex: # pylint: disable=broad-except
1019 msg = f"An exception occurred while rating: {ex}"
1020 else:
1021 self.stats.global_note = note
1022 msg = f"Your code has been rated at {note:.2f}/10"
1023 if previous_stats:
1024 pnote = previous_stats.global_note
1025 if pnote is not None:
1026 msg += f" (previous run: {pnote:.2f}/10, {note - pnote:+.2f})"
1028 if self.config.score:
1029 sect = report_nodes.EvaluationSection(msg)
1030 self.reporter.display_reports(sect)
1031 return note
1033 def _add_one_message(
1034 self,
1035 message_definition: MessageDefinition,
1036 line: int | None,
1037 node: nodes.NodeNG | None,
1038 args: Any | None,
1039 confidence: interfaces.Confidence | None,
1040 col_offset: int | None,
1041 end_lineno: int | None,
1042 end_col_offset: int | None,
1043 ) -> None:
1044 """After various checks have passed a single Message is
1045 passed to the reporter and added to stats.
1046 """
1047 message_definition.check_message_definition(line, node)
1049 # Look up "location" data of node if not yet supplied
1050 if node:
1051 if node.position:
1052 if not line:
1053 line = node.position.lineno
1054 if not col_offset:
1055 col_offset = node.position.col_offset
1056 if not end_lineno:
1057 end_lineno = node.position.end_lineno
1058 if not end_col_offset:
1059 end_col_offset = node.position.end_col_offset
1060 else:
1061 if not line:
1062 line = node.fromlineno
1063 if not col_offset:
1064 col_offset = node.col_offset
1065 if not end_lineno:
1066 end_lineno = node.end_lineno
1067 if not end_col_offset:
1068 end_col_offset = node.end_col_offset
1070 # should this message be displayed
1071 if not self.is_message_enabled(message_definition.msgid, line, confidence):
1072 self.file_state.handle_ignored_message(
1073 self._get_message_state_scope(
1074 message_definition.msgid, line, confidence
1075 ),
1076 message_definition.msgid,
1077 line,
1078 )
1079 return
1081 # update stats
1082 msg_cat = MSG_TYPES[message_definition.msgid[0]]
1083 self.msg_status |= MSG_TYPES_STATUS[message_definition.msgid[0]]
1084 self.stats.increase_single_message_count(msg_cat, 1)
1085 self.stats.increase_single_module_message_count(
1086 self.current_name, # type: ignore[arg-type] # Should be removable after https://github.com/PyCQA/pylint/pull/5580
1087 msg_cat,
1088 1,
1089 )
1090 try:
1091 self.stats.by_msg[message_definition.symbol] += 1
1092 except KeyError:
1093 self.stats.by_msg[message_definition.symbol] = 1
1094 # Interpolate arguments into message string
1095 msg = message_definition.msg
1096 if args is not None:
1097 msg %= args
1098 # get module and object
1099 if node is None:
1100 module, obj = self.current_name, ""
1101 abspath = self.current_file
1102 else:
1103 module, obj = utils.get_module_and_frameid(node)
1104 abspath = node.root().file
1105 if abspath is not None:
1106 path = abspath.replace(self.reporter.path_strip_prefix, "", 1)
1107 else:
1108 path = "configuration"
1109 # add the message
1110 self.reporter.handle_message(
1111 Message(
1112 message_definition.msgid,
1113 message_definition.symbol,
1114 MessageLocationTuple(
1115 abspath or "",
1116 path,
1117 module or "",
1118 obj,
1119 line or 1,
1120 col_offset or 0,
1121 end_lineno,
1122 end_col_offset,
1123 ),
1124 msg,
1125 confidence,
1126 )
1127 )
1129 def add_message(
1130 self,
1131 msgid: str,
1132 line: int | None = None,
1133 node: nodes.NodeNG | None = None,
1134 args: Any | None = None,
1135 confidence: interfaces.Confidence | None = None,
1136 col_offset: int | None = None,
1137 end_lineno: int | None = None,
1138 end_col_offset: int | None = None,
1139 ) -> None:
1140 """Adds a message given by ID or name.
1142 If provided, the message string is expanded using args.
1144 AST checkers must provide the node argument (but may optionally
1145 provide line if the line number is different), raw and token checkers
1146 must provide the line argument.
1147 """
1148 if confidence is None:
1149 confidence = interfaces.UNDEFINED
1150 message_definitions = self.msgs_store.get_message_definitions(msgid)
1151 for message_definition in message_definitions:
1152 self._add_one_message(
1153 message_definition,
1154 line,
1155 node,
1156 args,
1157 confidence,
1158 col_offset,
1159 end_lineno,
1160 end_col_offset,
1161 )
1163 def add_ignored_message(
1164 self,
1165 msgid: str,
1166 line: int,
1167 node: nodes.NodeNG | None = None,
1168 confidence: interfaces.Confidence | None = interfaces.UNDEFINED,
1169 ) -> None:
1170 """Prepares a message to be added to the ignored message storage.
1172 Some checks return early in special cases and never reach add_message(),
1173 even though they would normally issue a message.
1174 This creates false positives for useless-suppression.
1175 This function avoids this by adding those message to the ignored msgs attribute
1176 """
1177 message_definitions = self.msgs_store.get_message_definitions(msgid)
1178 for message_definition in message_definitions:
1179 message_definition.check_message_definition(line, node)
1180 self.file_state.handle_ignored_message(
1181 self._get_message_state_scope(
1182 message_definition.msgid, line, confidence
1183 ),
1184 message_definition.msgid,
1185 line,
1186 )