[WIP] test parallel checking

mypy/__main__.py

@@ -6,6 +6,7 @@
 import sys
 import traceback
 
+from mypy.fscache import FileSystemCache
 from mypy.main import main, process_options
 from mypy.util import FancyFormatter
 
@@ -22,7 +23,8 @@ def console_entry() -> None:
         os.dup2(devnull, sys.stdout.fileno())
         sys.exit(2)
     except KeyboardInterrupt:
-        _, options = process_options(args=sys.argv[1:])
+        # Setting fscache prevents bogus errors when --package-root is set.
+        _, options = process_options(args=sys.argv[1:], fscache=FileSystemCache())
         if options.show_traceback:
             sys.stdout.write(traceback.format_exc())
         formatter = FancyFormatter(sys.stdout, sys.stderr, False)
@@ -36,7 +38,7 @@ def console_entry() -> None:
         try:
             import mypy.errors
 
-            _, options = process_options(args=sys.argv[1:])
+            _, options = process_options(args=sys.argv[1:], fscache=FileSystemCache())
             mypy.errors.report_internal_error(e, None, 0, None, options)
         except Exception:
             pass

mypy/build.py

@@ -1025,7 +1025,7 @@ def parse_all(self, states: list[State], post_parse: bool = True) -> None:
             if state.tree is not None:
                 # The file was already parsed.
                 continue
-            if not self.fscache.exists(state.xpath):
+            if not self.fscache.exists(state.xpath, real_only=True):
                 # New parser only supports parsing on-disk files.
                 sequential_states.append(state)
                 continue
@@ -1083,7 +1083,6 @@ def parse_parallel(self, sequential_states: list[State], parallel_states: list[S
                 elif state.source_hash is None:
                     # At least namespace packages may not have source.
                     state.get_source()
-                state.size_hint = os.path.getsize(state.xpath)
                 state.early_errors = list(self.errors.error_info_map.get(state.xpath, []))
                 state.semantic_analysis_pass1()
                 self.ast_cache[state.id] = (state.tree, state.early_errors, state.source_hash)
@@ -1271,7 +1270,7 @@ def parse_file(
 
         Raise CompileError if there is a parse error.
         """
-        file_exists = self.fscache.exists(path)
+        file_exists = self.fscache.exists(path, real_only=True)
         t0 = time.time()
         if raw_data:
             # If possible, deserialize from known binary data instead of parsing from scratch.
@@ -3358,9 +3357,15 @@ def compute_dependencies(self) -> None:
         self.priorities = {}  # id -> priority
         self.dep_line_map = {}  # id -> line
         self.dep_hashes = {}
+        copied_imports = False
+        if not self.tree.defs and self.tree.raw_data is not None:
+            self.tree.defs = list(self.tree.imports)
+            copied_imports = True
         dep_entries = manager.all_imported_modules_in_file(
             self.tree
         ) + self.manager.plugin.get_additional_deps(self.tree)
+        if copied_imports:
+            self.tree.defs = []
         for pri, id, line in dep_entries:
             self.priorities[id] = min(pri, self.priorities.get(id, PRI_ALL))
             if id == self.id:

mypy/build_worker/worker.py

@@ -48,6 +48,7 @@
 )
 from mypy.cache import Tag, read_int_list, read_json
 from mypy.defaults import RECURSION_LIMIT, WORKER_CONNECTION_TIMEOUT, WORKER_IDLE_TIMEOUT
+from mypy.error_formatter import OUTPUT_CHOICES
 from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error
 from mypy.fscache import FileSystemCache
 from mypy.ipc import IPCException, IPCServer, ready_to_read, receive, send
@@ -102,6 +103,7 @@ def main(argv: list[str]) -> None:
         raise
 
     fscache = FileSystemCache()
+    fscache.set_package_root(options.package_root)
     cached_read = fscache.read
     errors = Errors(options, read_source=lambda path: read_py_file(path, cached_read))
 
@@ -319,24 +321,27 @@ def flush_errors(filename: str | None, new_messages: list[str], is_serious: bool
         # We never flush errors in the worker, we send them back to coordinator.
         pass
 
-    return BuildManager(
-        data_dir,
-        search_paths,
-        ignore_prefix=os.getcwd(),
-        source_set=source_set,
-        reports=None,
-        options=options,
-        version_id=__version__,
-        plugin=plugin,
-        plugins_snapshot=snapshot,
-        errors=ctx.errors,
-        error_formatter=None,
-        flush_errors=flush_errors,
-        fscache=ctx.fscache,
-        stdout=sys.stdout,
-        stderr=sys.stderr,
-        parallel_worker=True,
-    )
+    try:
+        return BuildManager(
+            data_dir,
+            search_paths,
+            ignore_prefix=os.getcwd(),
+            source_set=source_set,
+            reports=None,
+            options=options,
+            version_id=__version__,
+            plugin=plugin,
+            plugins_snapshot=snapshot,
+            errors=ctx.errors,
+            error_formatter=None if options.output is None else OUTPUT_CHOICES.get(options.output),
+            flush_errors=flush_errors,
+            fscache=ctx.fscache,
+            stdout=sys.stdout,
+            stderr=sys.stderr,
+            parallel_worker=True,
+        )
+    except CompileError:
+        return None
 
 
 def console_entry() -> None:

mypy/checker.py

@@ -493,6 +493,7 @@ def __init__(
         self.binder = ConditionalTypeBinder(options)
         self.globals_binder = self.binder
         self.globals = tree.names
+        self.globals_unreachable: set[DeferredNodeType] = set()
         self.return_types = []
         self.dynamic_funcs = []
         self.partial_types = []
@@ -577,6 +578,7 @@ def reset(self) -> None:
         self.partial_types = []
         self.inferred_attribute_types = None
         self.scope = CheckerScope(self.tree)
+        self.globals_unreachable.clear()
 
     def check_first_pass(self, recurse_into_functions: bool = True) -> None:
         """Type check the entire file, but defer functions with unresolved references.
@@ -595,8 +597,16 @@ def check_first_pass(self, recurse_into_functions: bool = True) -> None:
             )
             with self.tscope.module_scope(self.tree.fullname):
                 with self.enter_partial_types(), self.binder.top_frame_context():
-                    for d in self.tree.defs:
+                    marked_unreachable = False
+                    for i, d in enumerate(self.tree.defs):
                         if self.binder.is_unreachable():
+                            if not marked_unreachable:
+                                self.globals_unreachable.update(
+                                    stmt
+                                    for stmt in self.tree.defs[i:]
+                                    if isinstance(stmt, (FuncDef, Decorator, OverloadedFuncDef))
+                                )
+                                marked_unreachable = True
                             if not self.should_report_unreachable_issues():
                                 break
                             if not self.is_noop_for_reachability(d):
@@ -676,6 +686,8 @@ def check_partial(
                 if not impl_only:
                     self.accept(node)
                     return
+                if node in self.globals_unreachable:
+                    return
                 if isinstance(node, (FuncDef, Decorator)):
                     self.check_partial_impl(node)
                 else:
@@ -701,6 +713,8 @@ def check_partial_impl(self, impl: FuncDef | Decorator) -> None:
         if isinstance(impl, FuncDef):
             self.visit_func_def_impl(impl)
         else:
+            if any(refers_to_fullname(d, "typing.no_type_check") for d in impl.decorators):
+                return
             with self.tscope.function_scope(impl.func):
                 self.check_func_item(impl.func, name=impl.func.name)
 
@@ -3244,8 +3258,16 @@ def visit_block(self, b: Block) -> None:
             # as unreachable -- so we don't display an error.
             self.binder.unreachable()
             return
-        for s in b.body:
+        marked_unreachable = False
+        for i, s in enumerate(b.body):
             if self.binder.is_unreachable():
+                if not marked_unreachable and self.scope.top_level_function() is None:
+                    self.globals_unreachable.update(
+                        stmt
+                        for stmt in b.body[i:]
+                        if isinstance(stmt, (FuncDef, Decorator, OverloadedFuncDef))
+                    )
+                    marked_unreachable = True
                 if not self.should_report_unreachable_issues():
                     break
                 if not self.is_noop_for_reachability(s):
@@ -5689,12 +5711,10 @@ def visit_del_stmt(self, s: DelStmt) -> None:
                     )
 
     def visit_decorator(self, e: Decorator) -> None:
-        for d in e.decorators:
-            if isinstance(d, RefExpr):
-                if d.fullname == "typing.no_type_check":
-                    e.var.type = AnyType(TypeOfAny.special_form)
-                    e.var.is_ready = True
-                    return
+        if any(refers_to_fullname(d, "typing.no_type_check") for d in e.decorators):
+            e.var.type = AnyType(TypeOfAny.special_form)
+            e.var.is_ready = True
+            return
         self.visit_decorator_inner(e)
 
     def visit_decorator_inner(

mypy/fscache.py

@@ -253,9 +253,14 @@ def isdir(self, path: str) -> bool:
             return False
         return stat.S_ISDIR(st.st_mode)
 
-    def exists(self, path: str) -> bool:
+    def exists(self, path: str, real_only: bool = False) -> bool:
         st = self.stat_or_none(path)
-        return st is not None
+        if st is None:
+            return False
+        if real_only:
+            dirname, _ = os.path.split(path)
+            return dirname not in self.fake_package_cache
+        return True
 
     def read(self, path: str) -> bytes:
         if path in self.read_cache:

mypy/main.py

@@ -97,6 +97,10 @@ def main(
         stdout, stderr, options.hide_error_codes, hide_success=bool(options.output)
     )
 
+    options.num_workers = 1
+    if options.cache_dir == os.devnull:
+        options.cache_dir = defaults.CACHE_DIR
+
     if options.num_workers:
         # Supporting both parsers would be really tricky, so just support the new one.
         options.native_parser = True

mypy/options.py

@@ -479,8 +479,13 @@ def process_error_codes(self, *, error_callback: Callable[[str], Any]) -> None:
         if invalid_code_names_here:
             error_callback(f"Invalid error code(s): {', '.join(sorted(invalid_code_names_here))}")
 
-        self.disabled_error_codes |= {error_codes[code] for code in disabled_code_names}
-        self.enabled_error_codes |= {error_codes[code] for code in enabled_code_names}
+        # Ignore invalid error codes.
+        self.disabled_error_codes |= {
+            error_codes[code] for code in disabled_code_names if code in error_codes
+        }
+        self.enabled_error_codes |= {
+            error_codes[code] for code in enabled_code_names if code in error_codes
+        }
 
         # Enabling an error code always overrides disabling
         self.disabled_error_codes -= self.enabled_error_codes

test-data/unit/check-functions.test

@@ -1022,6 +1022,14 @@ def foo(x: {1:2}) -> [1]:
     x = y
 [typing fixtures/typing-medium.pyi]
 
+[case testNoTypeCheckDecoratorNoUntypedDefs]
+# flags: --disallow-untyped-defs
+import typing
+
+@typing.no_type_check
+def foo(x: "foo") -> "bar":
+    x: "whatever"
+[typing fixtures/typing-medium.pyi]
 
 -- Forward references to decorated functions
 -- -----------------------------------------

test-data/unit/check-unreachable-code.test

@@ -1701,3 +1701,30 @@ def main(contents: Any, commit: str | None) -> None:
 
 main({"commit": None}, None)
 [builtins fixtures/tuple.pyi]
+
+[case testUnreachableFunctionAfterTopLevelAssert]
+def foo() -> int:
+    ...
+
+raise Exception
+
+x = foo()
+
+def test() -> None:
+    # It is questionable to allow this, this test exists to check 1:1
+    # behavior match of parallel checking with sequential checking.
+    x + "hm..."
+[builtins fixtures/exception.pyi]
+
+[case testUnreachableFunctionAfterClassLevelAssert]
+def foo() -> int:
+    ...
+
+class C:
+    x = foo()
+    raise Exception
+    def test(self) -> None:
+        # It is questionable to allow this, this test exists to check 1:1
+        # behavior match of parallel checking with sequential checking.
+        C.x + "hm..."
+[builtins fixtures/exception.pyi]