[stubtest] Check runtime availability of private types not marked @type_check_only

mypy/stubtest.py

@@ -14,6 +14,7 @@
 import importlib
 import importlib.machinery
 import inspect
+import keyword
 import os
 import pkgutil
 import re
@@ -356,11 +357,7 @@ def verify_mypyfile(
         runtime_all_as_set = None
 
     # Check things in the stub
-    to_check = {
-        m
-        for m, o in stub.names.items()
-        if not o.module_hidden and (not is_probably_private(m) or hasattr(runtime, m))
-    }
+    to_check = {m for m, o in stub.names.items() if not o.module_hidden}
 
     def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool:
         """Heuristics to determine whether a name originates from another module."""
@@ -418,6 +415,15 @@ def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool:
             # Don't recursively check exported modules, since that leads to infinite recursion
             continue
         assert stub_entry is not None
+        if (
+            is_probably_private(entry)
+            and not hasattr(runtime, entry)
+            and not isinstance(stub_entry, Missing)
+            and not _is_decoratable(stub_entry)
+        ):
+            # Skip private names that don't exist at runtime and which cannot
+            # be marked with @type_check_only.
+            continue
         try:
             runtime_entry = getattr(runtime, entry, MISSING)
         except Exception:
@@ -427,6 +433,23 @@ def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool:
         yield from verify(stub_entry, runtime_entry, object_path + [entry])
 
 
+def _is_decoratable(stub: nodes.SymbolNode) -> bool:
+    if not isinstance(stub, nodes.TypeInfo):
+        return False
+    if stub.is_newtype:
+        return False
+    if stub.typeddict_type is not None:
+        return all(
+            name.isidentifier() and not keyword.iskeyword(name)
+            for name in stub.typeddict_type.items.keys()
+        )
+    if stub.is_named_tuple:
+        return all(
+            name.isidentifier() and not keyword.iskeyword(name) for name in stub.names.keys()
+        )
+    return True
+
+
 def _verify_final(
     stub: nodes.TypeInfo, runtime: type[Any], object_path: list[str]
 ) -> Iterator[Error]:
@@ -526,7 +549,10 @@ def verify_typeinfo(
         return
 
     if isinstance(runtime, Missing):
-        yield Error(object_path, "is not present at runtime", stub, runtime, stub_desc=repr(stub))
+        msg = "is not present at runtime"
+        if is_probably_private(stub.name):
+            msg += '. Maybe mark it as "@type_check_only"?'
+        yield Error(object_path, msg, stub, runtime, stub_desc=repr(stub))
         return
     if not isinstance(runtime, type):
         # Yes, some runtime objects can be not types, no way to tell mypy about that.

mypy/test/teststubtest.py

@@ -52,6 +52,7 @@ def __getitem__(self, typeargs: Any) -> object: ...
 
 Final = 0
 Literal = 0
+NewType = 0
 TypedDict = 0
 
 class TypeVar:
@@ -1122,7 +1123,7 @@ def test_type_alias(self) -> Iterator[Case]:
             import collections.abc
             import re
             import typing
-            from typing import Callable, Dict, Generic, Iterable, List, Match, Tuple, TypeVar, Union
+            from typing import Callable, Dict, Generic, Iterable, List, Match, Tuple, TypeVar, Union, type_check_only
             """,
             runtime="""
             import collections.abc
@@ -1193,6 +1194,7 @@ class Y: ...
         yield Case(
             stub="""
             _T = TypeVar("_T")
+            @type_check_only
             class _Spam(Generic[_T]):
                 def foo(self) -> None: ...
             IntFood = _Spam[int]
@@ -1477,6 +1479,7 @@ def test_missing(self) -> Iterator[Case]:
         yield Case(stub="x = 5", runtime="", error="x")
         yield Case(stub="def f(): ...", runtime="", error="f")
         yield Case(stub="class X: ...", runtime="", error="X")
+        yield Case(stub="class _X: ...", runtime="", error="_X")
         yield Case(
             stub="""
             from typing import overload
@@ -1533,6 +1536,8 @@ def __delattr__(self, name: str, /) -> None: ...
             runtime="class FakeDelattrClass: ...",
             error="FakeDelattrClass.__delattr__",
         )
+        yield Case(stub="from typing import NewType", runtime="", error=None)
+        yield Case(stub="_Int = NewType('_Int', int)", runtime="", error=None)
 
     @collect_cases
     def test_missing_no_runtime_all(self) -> Iterator[Case]:
@@ -2048,8 +2053,9 @@ def test_special_subtype(self) -> Iterator[Case]:
         )
         yield Case(
             stub="""
-            from typing import TypedDict
+            from typing import TypedDict, type_check_only
 
+            @type_check_only
             class _Options(TypedDict):
                 a: str
                 b: int
@@ -2472,6 +2478,23 @@ def func2() -> None: ...
             runtime="def func2() -> None: ...",
             error="func2",
         )
+        # The same is true for private types
+        yield Case(
+            stub="""
+            @type_check_only
+            class _P1: ...
+            """,
+            runtime="",
+            error=None,
+        )
+        yield Case(
+            stub="""
+            @type_check_only
+            class _P2: ...
+            """,
+            runtime="class _P2: ...",
+            error="_P2",
+        )
         # A type that exists at runtime is allowed to alias a type marked
         # as '@type_check_only' in the stubs.
         yield Case(