This commit is contained in:
ton
2024-10-07 10:13:40 +07:00
parent aa1631742f
commit 3a7d696db6
9729 changed files with 1832837 additions and 161742 deletions

View File

@@ -9,7 +9,7 @@ from pip._internal.utils.misc import strtobool
from .base import BaseDistribution, BaseEnvironment, FilesystemWheel, MemoryWheel, Wheel
if TYPE_CHECKING:
from typing import Protocol
from typing import Literal, Protocol
else:
Protocol = object
@@ -50,6 +50,7 @@ def _should_use_importlib_metadata() -> bool:
class Backend(Protocol):
NAME: 'Literal["importlib", "pkg_resources"]'
Distribution: Type[BaseDistribution]
Environment: Type[BaseEnvironment]

View File

@@ -2,7 +2,7 @@
from email.header import Header, decode_header, make_header
from email.message import Message
from typing import Any, Dict, List, Union
from typing import Any, Dict, List, Union, cast
METADATA_FIELDS = [
# Name, Multiple-Use
@@ -64,10 +64,10 @@ def msg_to_json(msg: Message) -> Dict[str, Any]:
key = json_name(field)
if multi:
value: Union[str, List[str]] = [
sanitise_header(v) for v in msg.get_all(field)
sanitise_header(v) for v in msg.get_all(field) # type: ignore
]
else:
value = sanitise_header(msg.get(field))
value = sanitise_header(msg.get(field)) # type: ignore
if key == "keywords":
# Accept both comma-separated and space-separated
# forms, for better compatibility with old data.
@@ -77,7 +77,7 @@ def msg_to_json(msg: Message) -> Dict[str, Any]:
value = value.split()
result[key] = value
payload = msg.get_payload()
payload = cast(str, msg.get_payload())
if payload:
result["description"] = payload

View File

@@ -8,7 +8,6 @@ import re
import zipfile
from typing import (
IO,
TYPE_CHECKING,
Any,
Collection,
Container,
@@ -18,14 +17,15 @@ from typing import (
List,
NamedTuple,
Optional,
Protocol,
Tuple,
Union,
)
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
from pip._vendor.packaging.utils import NormalizedName
from pip._vendor.packaging.version import LegacyVersion, Version
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._internal.exceptions import NoneMetadataError
from pip._internal.locations import site_packages, user_site
@@ -37,18 +37,10 @@ from pip._internal.models.direct_url import (
from pip._internal.utils.compat import stdlib_pkgs # TODO: Move definition here.
from pip._internal.utils.egg_link import egg_link_path_from_sys_path
from pip._internal.utils.misc import is_local, normalize_path
from pip._internal.utils.packaging import safe_extra
from pip._internal.utils.urls import url_to_path
from ._json import msg_to_json
if TYPE_CHECKING:
from typing import Protocol
else:
Protocol = object
DistributionVersion = Union[LegacyVersion, Version]
InfoPath = Union[str, pathlib.PurePath]
logger = logging.getLogger(__name__)
@@ -146,10 +138,10 @@ class BaseDistribution(Protocol):
raise NotImplementedError()
def __repr__(self) -> str:
return f"{self.raw_name} {self.version} ({self.location})"
return f"{self.raw_name} {self.raw_version} ({self.location})"
def __str__(self) -> str:
return f"{self.raw_name} {self.version}"
return f"{self.raw_name} {self.raw_version}"
@property
def location(self) -> Optional[str]:
@@ -280,7 +272,11 @@ class BaseDistribution(Protocol):
raise NotImplementedError()
@property
def version(self) -> DistributionVersion:
def version(self) -> Version:
raise NotImplementedError()
@property
def raw_version(self) -> str:
raise NotImplementedError()
@property
@@ -386,15 +382,7 @@ class BaseDistribution(Protocol):
def _metadata_impl(self) -> email.message.Message:
raise NotImplementedError()
@functools.lru_cache(maxsize=1)
def _metadata_cached(self) -> email.message.Message:
# When we drop python 3.7 support, move this to the metadata property and use
# functools.cached_property instead of lru_cache.
metadata = self._metadata_impl()
self._add_egg_info_requires(metadata)
return metadata
@property
@functools.cached_property
def metadata(self) -> email.message.Message:
"""Metadata of distribution parsed from e.g. METADATA or PKG-INFO.
@@ -403,7 +391,9 @@ class BaseDistribution(Protocol):
:raises NoneMetadataError: If the metadata file is available, but does
not contain valid metadata.
"""
return self._metadata_cached()
metadata = self._metadata_impl()
self._add_egg_info_requires(metadata)
return metadata
@property
def metadata_dict(self) -> Dict[str, Any]:
@@ -455,11 +445,19 @@ class BaseDistribution(Protocol):
"""
raise NotImplementedError()
def iter_provided_extras(self) -> Iterable[str]:
def iter_raw_dependencies(self) -> Iterable[str]:
"""Raw Requires-Dist metadata."""
return self.metadata.get_all("Requires-Dist", [])
def iter_provided_extras(self) -> Iterable[NormalizedName]:
"""Extras provided by this distribution.
For modern .dist-info distributions, this is the collection of
"Provides-Extra:" entries in distribution metadata.
The return value of this function is expected to be normalised names,
per PEP 685, with the returned value being handled appropriately by
`iter_dependencies`.
"""
raise NotImplementedError()
@@ -537,10 +535,11 @@ class BaseDistribution(Protocol):
"""Get extras from the egg-info directory."""
known_extras = {""}
for entry in self._iter_requires_txt_entries():
if entry.extra in known_extras:
extra = canonicalize_name(entry.extra)
if extra in known_extras:
continue
known_extras.add(entry.extra)
yield entry.extra
known_extras.add(extra)
yield extra
def _iter_egg_info_dependencies(self) -> Iterable[str]:
"""Get distribution dependencies from the egg-info directory.
@@ -556,10 +555,11 @@ class BaseDistribution(Protocol):
all currently available PEP 517 backends, although not standardized.
"""
for entry in self._iter_requires_txt_entries():
if entry.extra and entry.marker:
marker = f'({entry.marker}) and extra == "{safe_extra(entry.extra)}"'
elif entry.extra:
marker = f'extra == "{safe_extra(entry.extra)}"'
extra = canonicalize_name(entry.extra)
if extra and entry.marker:
marker = f'({entry.marker}) and extra == "{extra}"'
elif extra:
marker = f'extra == "{extra}"'
elif entry.marker:
marker = entry.marker
else:

View File

@@ -1,4 +1,6 @@
from ._dists import Distribution
from ._envs import Environment
__all__ = ["Distribution", "Environment"]
__all__ = ["NAME", "Distribution", "Environment"]
NAME = "importlib"

View File

@@ -1,5 +1,8 @@
import importlib.metadata
from typing import Any, Optional, Protocol, cast
import os
from typing import Any, Optional, Protocol, Tuple, cast
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
class BadMetadata(ValueError):
@@ -43,13 +46,40 @@ def get_info_location(d: importlib.metadata.Distribution) -> Optional[BasePath]:
return getattr(d, "_path", None)
def get_dist_name(dist: importlib.metadata.Distribution) -> str:
"""Get the distribution's project name.
def parse_name_and_version_from_info_directory(
dist: importlib.metadata.Distribution,
) -> Tuple[Optional[str], Optional[str]]:
"""Get a name and version from the metadata directory name.
This is much faster than reading distribution metadata.
"""
info_location = get_info_location(dist)
if info_location is None:
return None, None
stem, suffix = os.path.splitext(info_location.name)
if suffix == ".dist-info":
name, sep, version = stem.partition("-")
if sep:
return name, version
if suffix == ".egg-info":
name = stem.split("-", 1)[0]
return name, None
return None, None
def get_dist_canonical_name(dist: importlib.metadata.Distribution) -> NormalizedName:
"""Get the distribution's normalized name.
The ``name`` attribute is only available in Python 3.10 or later. We are
targeting exactly that, but Mypy does not know this.
"""
if name := parse_name_and_version_from_info_directory(dist)[0]:
return canonicalize_name(name)
name = cast(Any, dist).name
if not isinstance(name, str):
raise BadMetadata(dist, reason="invalid metadata entry 'name'")
return name
return canonicalize_name(name)

View File

@@ -1,6 +1,5 @@
import email.message
import importlib.metadata
import os
import pathlib
import zipfile
from typing import (
@@ -16,22 +15,26 @@ from typing import (
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
from pip._internal.exceptions import InvalidWheel, UnsupportedWheel
from pip._internal.metadata.base import (
BaseDistribution,
BaseEntryPoint,
DistributionVersion,
InfoPath,
Wheel,
)
from pip._internal.utils.misc import normalize_path
from pip._internal.utils.packaging import safe_extra
from pip._internal.utils.packaging import get_requirement
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.wheel import parse_wheel, read_wheel_metadata_file
from ._compat import BasePath, get_dist_name
from ._compat import (
BasePath,
get_dist_canonical_name,
parse_name_and_version_from_info_directory,
)
class WheelDistribution(importlib.metadata.Distribution):
@@ -134,8 +137,6 @@ class Distribution(BaseDistribution):
dist = WheelDistribution.from_zipfile(zf, name, wheel.location)
except zipfile.BadZipFile as e:
raise InvalidWheel(wheel.location, name) from e
except UnsupportedWheel as e:
raise UnsupportedWheel(f"{name} has an invalid wheel, {e}")
return cls(dist, dist.info_location, pathlib.PurePosixPath(wheel.location))
@property
@@ -156,27 +157,20 @@ class Distribution(BaseDistribution):
return None
return normalize_path(str(self._installed_location))
def _get_dist_name_from_location(self) -> Optional[str]:
"""Try to get the name from the metadata directory name.
This is much faster than reading metadata.
"""
if self._info_location is None:
return None
stem, suffix = os.path.splitext(self._info_location.name)
if suffix not in (".dist-info", ".egg-info"):
return None
return stem.split("-", 1)[0]
@property
def canonical_name(self) -> NormalizedName:
name = self._get_dist_name_from_location() or get_dist_name(self._dist)
return canonicalize_name(name)
return get_dist_canonical_name(self._dist)
@property
def version(self) -> DistributionVersion:
def version(self) -> Version:
if version := parse_name_and_version_from_info_directory(self._dist)[1]:
return parse_version(version)
return parse_version(self._dist.version)
@property
def raw_version(self) -> str:
return self._dist.version
def is_file(self, path: InfoPath) -> bool:
return self._dist.read_text(str(path)) is not None
@@ -207,15 +201,18 @@ class Distribution(BaseDistribution):
# until upstream can improve the protocol. (python/cpython#94952)
return cast(email.message.Message, self._dist.metadata)
def iter_provided_extras(self) -> Iterable[str]:
return (
safe_extra(extra) for extra in self.metadata.get_all("Provides-Extra", [])
)
def iter_provided_extras(self) -> Iterable[NormalizedName]:
return [
canonicalize_name(extra)
for extra in self.metadata.get_all("Provides-Extra", [])
]
def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
contexts: Sequence[Dict[str, str]] = [{"extra": safe_extra(e)} for e in extras]
contexts: Sequence[Dict[str, str]] = [{"extra": e} for e in extras]
for req_string in self.metadata.get_all("Requires-Dist", []):
req = Requirement(req_string)
# strip() because email.message.Message.get_all() may return a leading \n
# in case a long header was wrapped.
req = get_requirement(req_string.strip())
if not req.marker:
yield req
elif not extras and req.marker.evaluate({"extra": ""}):

View File

@@ -15,7 +15,7 @@ from pip._internal.models.wheel import Wheel
from pip._internal.utils.deprecation import deprecated
from pip._internal.utils.filetypes import WHEEL_EXTENSION
from ._compat import BadMetadata, BasePath, get_dist_name, get_info_location
from ._compat import BadMetadata, BasePath, get_dist_canonical_name, get_info_location
from ._dists import Distribution
logger = logging.getLogger(__name__)
@@ -61,14 +61,13 @@ class _DistributionFinder:
for dist in importlib.metadata.distributions(path=[location]):
info_location = get_info_location(dist)
try:
raw_name = get_dist_name(dist)
name = get_dist_canonical_name(dist)
except BadMetadata as e:
logger.warning("Skipping %s due to %s", info_location, e.reason)
continue
normalized_name = canonicalize_name(raw_name)
if normalized_name in self._found_names:
if name in self._found_names:
continue
self._found_names.add(normalized_name)
self._found_names.add(name)
yield dist, info_location
def find(self, location: str) -> Iterator[BaseDistribution]:
@@ -150,8 +149,9 @@ class _DistributionFinder:
def _emit_egg_deprecation(location: Optional[str]) -> None:
deprecated(
reason=f"Loading egg at {location} is deprecated.",
replacement="to use pip for package installation.",
gone_in="23.3",
replacement="to use pip for package installation",
gone_in="24.3",
issue=12330,
)
@@ -180,9 +180,10 @@ class Environment(BaseEnvironment):
yield from finder.find_linked(location)
def get_distribution(self, name: str) -> Optional[BaseDistribution]:
canonical_name = canonicalize_name(name)
matches = (
distribution
for distribution in self.iter_all_distributions()
if distribution.canonical_name == canonicalize_name(name)
if distribution.canonical_name == canonical_name
)
return next(matches, None)

View File

@@ -3,11 +3,20 @@ import email.parser
import logging
import os
import zipfile
from typing import Collection, Iterable, Iterator, List, Mapping, NamedTuple, Optional
from typing import (
Collection,
Iterable,
Iterator,
List,
Mapping,
NamedTuple,
Optional,
)
from pip._vendor import pkg_resources
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
from pip._internal.exceptions import InvalidWheel, NoneMetadataError, UnsupportedWheel
@@ -19,13 +28,16 @@ from .base import (
BaseDistribution,
BaseEntryPoint,
BaseEnvironment,
DistributionVersion,
InfoPath,
Wheel,
)
__all__ = ["NAME", "Distribution", "Environment"]
logger = logging.getLogger(__name__)
NAME = "pkg_resources"
class EntryPoint(NamedTuple):
name: str
@@ -71,6 +83,18 @@ class InMemoryMetadata:
class Distribution(BaseDistribution):
def __init__(self, dist: pkg_resources.Distribution) -> None:
self._dist = dist
# This is populated lazily, to avoid loading metadata for all possible
# distributions eagerly.
self.__extra_mapping: Optional[Mapping[NormalizedName, str]] = None
@property
def _extra_mapping(self) -> Mapping[NormalizedName, str]:
if self.__extra_mapping is None:
self.__extra_mapping = {
canonicalize_name(extra): extra for extra in self._dist.extras
}
return self.__extra_mapping
@classmethod
def from_directory(cls, directory: str) -> BaseDistribution:
@@ -164,9 +188,13 @@ class Distribution(BaseDistribution):
return canonicalize_name(self._dist.project_name)
@property
def version(self) -> DistributionVersion:
def version(self) -> Version:
return parse_version(self._dist.version)
@property
def raw_version(self) -> str:
return self._dist.version
def is_file(self, path: InfoPath) -> bool:
return self._dist.has_metadata(str(path))
@@ -211,12 +239,15 @@ class Distribution(BaseDistribution):
return feed_parser.close()
def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
if extras: # pkg_resources raises on invalid extras, so we sanitize.
extras = frozenset(extras).intersection(self._dist.extras)
if extras:
relevant_extras = set(self._extra_mapping) & set(
map(canonicalize_name, extras)
)
extras = [self._extra_mapping[extra] for extra in relevant_extras]
return self._dist.requires(extras)
def iter_provided_extras(self) -> Iterable[str]:
return self._dist.extras
def iter_provided_extras(self) -> Iterable[NormalizedName]:
return self._extra_mapping.keys()
class Environment(BaseEnvironment):