This commit is contained in:
ton
2024-10-07 10:13:40 +07:00
parent aa1631742f
commit 3a7d696db6
9729 changed files with 1832837 additions and 161742 deletions

View File

@@ -3,9 +3,8 @@ import hashlib
import logging
import os
from types import TracebackType
from typing import Dict, Generator, Optional, Set, Type, Union
from typing import Dict, Generator, Optional, Type, Union
from pip._internal.models.link import Link
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.temp_dir import TempDirectory
@@ -51,10 +50,22 @@ def get_build_tracker() -> Generator["BuildTracker", None, None]:
yield tracker
class TrackerId(str):
"""Uniquely identifying string provided to the build tracker."""
class BuildTracker:
"""Ensure that an sdist cannot request itself as a setup requirement.
When an sdist is prepared, it identifies its setup requirements in the
context of ``BuildTracker.track()``. If a requirement shows up recursively, this
raises an exception.
This stops fork bombs embedded in malicious packages."""
def __init__(self, root: str) -> None:
self._root = root
self._entries: Set[InstallRequirement] = set()
self._entries: Dict[TrackerId, InstallRequirement] = {}
logger.debug("Created build tracker: %s", self._root)
def __enter__(self) -> "BuildTracker":
@@ -69,16 +80,15 @@ class BuildTracker:
) -> None:
self.cleanup()
def _entry_path(self, link: Link) -> str:
hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest()
def _entry_path(self, key: TrackerId) -> str:
hashed = hashlib.sha224(key.encode()).hexdigest()
return os.path.join(self._root, hashed)
def add(self, req: InstallRequirement) -> None:
def add(self, req: InstallRequirement, key: TrackerId) -> None:
"""Add an InstallRequirement to build tracking."""
assert req.link
# Get the file to write information about this requirement.
entry_path = self._entry_path(req.link)
entry_path = self._entry_path(key)
# Try reading from the file. If it exists and can be read from, a build
# is already in progress, so a LookupError is raised.
@@ -88,37 +98,41 @@ class BuildTracker:
except FileNotFoundError:
pass
else:
message = "{} is already being built: {}".format(req.link, contents)
message = f"{req.link} is already being built: {contents}"
raise LookupError(message)
# If we're here, req should really not be building already.
assert req not in self._entries
assert key not in self._entries
# Start tracking this requirement.
with open(entry_path, "w", encoding="utf-8") as fp:
fp.write(str(req))
self._entries.add(req)
self._entries[key] = req
logger.debug("Added %s to build tracker %r", req, self._root)
def remove(self, req: InstallRequirement) -> None:
def remove(self, req: InstallRequirement, key: TrackerId) -> None:
"""Remove an InstallRequirement from build tracking."""
assert req.link
# Delete the created file and the corresponding entries.
os.unlink(self._entry_path(req.link))
self._entries.remove(req)
# Delete the created file and the corresponding entry.
os.unlink(self._entry_path(key))
del self._entries[key]
logger.debug("Removed %s from build tracker %r", req, self._root)
def cleanup(self) -> None:
for req in set(self._entries):
self.remove(req)
for key, req in list(self._entries.items()):
self.remove(req, key)
logger.debug("Removed build tracker: %r", self._root)
@contextlib.contextmanager
def track(self, req: InstallRequirement) -> Generator[None, None, None]:
self.add(req)
def track(self, req: InstallRequirement, key: str) -> Generator[None, None, None]:
"""Ensure that `key` cannot install itself as a setup requirement.
:raises LookupError: If `key` was already provided in a parent invocation of
the context introduced by this method."""
tracker_id = TrackerId(key)
self.add(req, tracker_id)
yield
self.remove(req)
self.remove(req, tracker_id)

View File

@@ -27,7 +27,7 @@ def _find_egg_info(directory: str) -> str:
if len(filenames) > 1:
raise InstallationError(
"More than one .egg-info directory found in {}".format(directory)
f"More than one .egg-info directory found in {directory}"
)
return os.path.join(directory, filenames[0])

View File

@@ -40,16 +40,16 @@ def get_legacy_build_wheel_path(
# Sort for determinism.
names = sorted(names)
if not names:
msg = ("Legacy build of wheel for {!r} created no files.\n").format(name)
msg = f"Legacy build of wheel for {name!r} created no files.\n"
msg += format_command_result(command_args, command_output)
logger.warning(msg)
return None
if len(names) > 1:
msg = (
"Legacy build of wheel for {!r} created more than one file.\n"
"Filenames (choosing first): {}\n"
).format(name, names)
f"Legacy build of wheel for {name!r} created more than one file.\n"
f"Filenames (choosing first): {names}\n"
)
msg += format_command_result(command_args, command_output)
logger.warning(msg)

View File

@@ -2,31 +2,44 @@
"""
import logging
from typing import Callable, Dict, List, NamedTuple, Optional, Set, Tuple
from contextlib import suppress
from email.parser import Parser
from functools import reduce
from typing import (
Callable,
Dict,
FrozenSet,
Generator,
Iterable,
List,
NamedTuple,
Optional,
Set,
Tuple,
)
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.packaging.specifiers import LegacySpecifier
from pip._vendor.packaging.tags import Tag, parse_tag
from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
from pip._vendor.packaging.version import LegacyVersion
from pip._vendor.packaging.version import Version
from pip._internal.distributions import make_distribution_for_install_requirement
from pip._internal.metadata import get_default_environment
from pip._internal.metadata.base import DistributionVersion
from pip._internal.metadata.base import BaseDistribution
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.deprecation import deprecated
logger = logging.getLogger(__name__)
class PackageDetails(NamedTuple):
version: DistributionVersion
version: Version
dependencies: List[Requirement]
# Shorthands
PackageSet = Dict[NormalizedName, PackageDetails]
Missing = Tuple[NormalizedName, Requirement]
Conflicting = Tuple[NormalizedName, DistributionVersion, Requirement]
Conflicting = Tuple[NormalizedName, Version, Requirement]
MissingDict = Dict[NormalizedName, List[Missing]]
ConflictingDict = Dict[NormalizedName, List[Conflicting]]
@@ -46,7 +59,7 @@ def create_package_set_from_installed() -> Tuple[PackageSet, bool]:
package_set[name] = PackageDetails(dist.version, dependencies)
except (OSError, ValueError) as e:
# Don't crash on unreadable or broken metadata.
logger.warning("Error parsing requirements for %s: %s", name, e)
logger.warning("Error parsing dependencies of %s: %s", name, e)
problems = True
return package_set, problems
@@ -60,8 +73,6 @@ def check_package_set(
package name and returns a boolean.
"""
warn_legacy_versions_and_specifiers(package_set)
missing = {}
conflicting = {}
@@ -118,6 +129,22 @@ def check_install_conflicts(to_install: List[InstallRequirement]) -> ConflictDet
)
def check_unsupported(
packages: Iterable[BaseDistribution],
supported_tags: Iterable[Tag],
) -> Generator[BaseDistribution, None, None]:
for p in packages:
with suppress(FileNotFoundError):
wheel_file = p.read_text("WHEEL")
wheel_tags: FrozenSet[Tag] = reduce(
frozenset.union,
map(parse_tag, Parser().parsestr(wheel_file).get_all("Tag", [])),
frozenset(),
)
if wheel_tags.isdisjoint(supported_tags):
yield p
def _simulate_installation_of(
to_install: List[InstallRequirement], package_set: PackageSet
) -> Set[NormalizedName]:
@@ -152,36 +179,3 @@ def _create_whitelist(
break
return packages_affected
def warn_legacy_versions_and_specifiers(package_set: PackageSet) -> None:
for project_name, package_details in package_set.items():
if isinstance(package_details.version, LegacyVersion):
deprecated(
reason=(
f"{project_name} {package_details.version} "
f"has a non-standard version number."
),
replacement=(
f"to upgrade to a newer version of {project_name} "
f"or contact the author to suggest that they "
f"release a version with a conforming version number"
),
issue=12063,
gone_in="23.3",
)
for dep in package_details.dependencies:
if any(isinstance(spec, LegacySpecifier) for spec in dep.specifier):
deprecated(
reason=(
f"{project_name} {package_details.version} "
f"has a non-standard dependency specifier {dep}."
),
replacement=(
f"to upgrade to a newer version of {project_name} "
f"or contact the author to suggest that they "
f"release a version with a conforming dependency specifiers"
),
issue=12063,
gone_in="23.3",
)

View File

@@ -4,7 +4,7 @@ import os
from typing import Container, Dict, Generator, Iterable, List, NamedTuple, Optional, Set
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import InvalidVersion
from pip._internal.exceptions import BadCommand, InstallationError
from pip._internal.metadata import BaseDistribution, get_environment
@@ -145,10 +145,13 @@ def freeze(
def _format_as_name_version(dist: BaseDistribution) -> str:
dist_version = dist.version
if isinstance(dist_version, Version):
try:
dist_version = dist.version
except InvalidVersion:
# legacy version
return f"{dist.raw_name}==={dist.raw_version}"
else:
return f"{dist.raw_name}=={dist_version}"
return f"{dist.raw_name}==={dist_version}"
def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:

View File

@@ -1,5 +1,6 @@
"""Legacy editable installation process, i.e. `setup.py develop`.
"""
import logging
from typing import Optional, Sequence

View File

@@ -28,6 +28,7 @@ from typing import (
List,
NewType,
Optional,
Protocol,
Sequence,
Set,
Tuple,
@@ -50,7 +51,7 @@ from pip._internal.metadata import (
from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl
from pip._internal.models.scheme import SCHEME_KEYS, Scheme
from pip._internal.utils.filesystem import adjacent_tmp_file, replace
from pip._internal.utils.misc import captured_stdout, ensure_dir, hash_file, partition
from pip._internal.utils.misc import StreamWrapper, ensure_dir, hash_file, partition
from pip._internal.utils.unpacking import (
current_umask,
is_within_directory,
@@ -60,7 +61,6 @@ from pip._internal.utils.unpacking import (
from pip._internal.utils.wheel import parse_wheel
if TYPE_CHECKING:
from typing import Protocol
class File(Protocol):
src_record_path: "RecordPath"
@@ -164,16 +164,14 @@ def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> Optional[str]:
for parent_dir, dir_scripts in warn_for.items():
sorted_scripts: List[str] = sorted(dir_scripts)
if len(sorted_scripts) == 1:
start_text = "script {} is".format(sorted_scripts[0])
start_text = f"script {sorted_scripts[0]} is"
else:
start_text = "scripts {} are".format(
", ".join(sorted_scripts[:-1]) + " and " + sorted_scripts[-1]
)
msg_lines.append(
"The {} installed in '{}' which is not on PATH.".format(
start_text, parent_dir
)
f"The {start_text} installed in '{parent_dir}' which is not on PATH."
)
last_line_fmt = (
@@ -267,9 +265,9 @@ def get_csv_rows_for_installed(
path = _fs_to_record_path(f, lib_dir)
digest, length = rehash(f)
installed_rows.append((path, digest, length))
for installed_record_path in installed.values():
installed_rows.append((installed_record_path, "", ""))
return installed_rows
return installed_rows + [
(installed_record_path, "", "") for installed_record_path in installed.values()
]
def get_console_script_specs(console: Dict[str, str]) -> List[str]:
@@ -290,17 +288,15 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]:
# the wheel metadata at build time, and so if the wheel is installed with
# a *different* version of Python the entry points will be wrong. The
# correct fix for this is to enhance the metadata to be able to describe
# such versioned entry points, but that won't happen till Metadata 2.0 is
# available.
# In the meantime, projects using versioned entry points will either have
# such versioned entry points.
# Currently, projects using versioned entry points will either have
# incorrect versioned entry points, or they will not be able to distribute
# "universal" wheels (i.e., they will need a wheel per Python version).
#
# Because setuptools and pip are bundled with _ensurepip and virtualenv,
# we need to use universal wheels. So, as a stopgap until Metadata 2.0, we
# we need to use universal wheels. As a workaround, we
# override the versioned entry points in the wheel and generate the
# correct ones. This code is purely a short-term measure until Metadata 2.0
# is available.
# correct ones.
#
# To add the level of hack in this section of code, in order to support
# ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment
@@ -321,9 +317,7 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]:
scripts_to_generate.append("pip = " + pip_script)
if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall":
scripts_to_generate.append(
"pip{} = {}".format(sys.version_info[0], pip_script)
)
scripts_to_generate.append(f"pip{sys.version_info[0]} = {pip_script}")
scripts_to_generate.append(f"pip{get_major_minor_version()} = {pip_script}")
# Delete any other versioned pip entry points
@@ -336,9 +330,7 @@ def get_console_script_specs(console: Dict[str, str]) -> List[str]:
scripts_to_generate.append("easy_install = " + easy_install_script)
scripts_to_generate.append(
"easy_install-{} = {}".format(
get_major_minor_version(), easy_install_script
)
f"easy_install-{get_major_minor_version()} = {easy_install_script}"
)
# Delete any other versioned easy_install entry points
easy_install_ep = [
@@ -366,12 +358,6 @@ class ZipBackedFile:
return self._zip_file.getinfo(self.src_record_path)
def save(self) -> None:
# directory creation is lazy and after file filtering
# to ensure we don't install empty dirs; empty dirs can't be
# uninstalled.
parent_dir = os.path.dirname(self.dest_path)
ensure_dir(parent_dir)
# When we open the output file below, any existing file is truncated
# before we start writing the new contents. This is fine in most
# cases, but can cause a segfault if pip has loaded a shared
@@ -385,9 +371,13 @@ class ZipBackedFile:
zipinfo = self._getinfo()
with self._zip_file.open(zipinfo) as f:
with open(self.dest_path, "wb") as dest:
shutil.copyfileobj(f, dest)
# optimization: the file is created by open(),
# skip the decompression when there is 0 bytes to decompress.
with open(self.dest_path, "wb") as dest:
if zipinfo.file_size > 0:
with self._zip_file.open(zipinfo) as f:
blocksize = min(zipinfo.file_size, 1024 * 1024)
shutil.copyfileobj(f, dest, blocksize)
if zip_item_is_executable(zipinfo):
set_extracted_file_to_default_mode_plus_executable(self.dest_path)
@@ -408,10 +398,10 @@ class ScriptFile:
class MissingCallableSuffix(InstallationError):
def __init__(self, entry_point: str) -> None:
super().__init__(
"Invalid script entry point: {} - A callable "
f"Invalid script entry point: {entry_point} - A callable "
"suffix is required. Cf https://packaging.python.org/"
"specifications/entry-points/#use-for-scripts for more "
"information.".format(entry_point)
"information."
)
@@ -429,7 +419,7 @@ class PipScriptMaker(ScriptMaker):
return super().make(specification, options)
def _install_wheel(
def _install_wheel( # noqa: C901, PLR0915 function is too long
name: str,
wheel_zip: ZipFile,
wheel_path: str,
@@ -513,9 +503,9 @@ def _install_wheel(
_, scheme_key, dest_subpath = normed_path.split(os.path.sep, 2)
except ValueError:
message = (
"Unexpected file in {}: {!r}. .data directory contents"
" should be named like: '<scheme key>/<path>'."
).format(wheel_path, record_path)
f"Unexpected file in {wheel_path}: {record_path!r}. .data directory"
" contents should be named like: '<scheme key>/<path>'."
)
raise InstallationError(message)
try:
@@ -523,10 +513,11 @@ def _install_wheel(
except KeyError:
valid_scheme_keys = ", ".join(sorted(scheme_paths))
message = (
"Unknown scheme key used in {}: {} (for file {!r}). .data"
" directory contents should be in subdirectories named"
" with a valid scheme key ({})"
).format(wheel_path, scheme_key, record_path, valid_scheme_keys)
f"Unknown scheme key used in {wheel_path}: {scheme_key} "
f"(for file {record_path!r}). .data directory contents "
f"should be in subdirectories named with a valid scheme "
f"key ({valid_scheme_keys})"
)
raise InstallationError(message)
dest_path = os.path.join(scheme_path, dest_subpath)
@@ -587,7 +578,15 @@ def _install_wheel(
script_scheme_files = map(ScriptFile, script_scheme_files)
files = chain(files, script_scheme_files)
existing_parents = set()
for file in files:
# directory creation is lazy and after file filtering
# to ensure we don't install empty dirs; empty dirs can't be
# uninstalled.
parent_dir = os.path.dirname(file.dest_path)
if parent_dir not in existing_parents:
ensure_dir(parent_dir)
existing_parents.add(parent_dir)
file.save()
record_installed(file.src_record_path, file.dest_path, file.changed)
@@ -610,7 +609,9 @@ def _install_wheel(
# Compile all of the pyc files for the installed files
if pycompile:
with captured_stdout() as stdout:
with contextlib.redirect_stdout(
StreamWrapper.from_stream(sys.stdout)
) as stdout:
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
for path in pyc_source_file_paths():
@@ -712,7 +713,7 @@ def req_error_context(req_description: str) -> Generator[None, None, None]:
try:
yield
except InstallationError as e:
message = "For req: {}. {}".format(req_description, e.args[0])
message = f"For req: {req_description}. {e.args[0]}"
raise InstallationError(message) from e

View File

@@ -4,10 +4,11 @@
# The following comment should be removed at some point in the future.
# mypy: strict-optional=False
import logging
import mimetypes
import os
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from pip._vendor.packaging.utils import canonicalize_name
@@ -21,7 +22,6 @@ from pip._internal.exceptions import (
InstallationError,
MetadataInconsistent,
NetworkConnectionError,
PreviousBuildDirError,
VcsHashUnsupported,
)
from pip._internal.index.package_finder import PackageFinder
@@ -37,6 +37,7 @@ from pip._internal.network.lazy_wheel import (
from pip._internal.network.session import PipSession
from pip._internal.operations.build.build_tracker import BuildTracker
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils._log import getLogger
from pip._internal.utils.direct_url_helpers import (
direct_url_for_editable,
direct_url_from_link,
@@ -47,13 +48,13 @@ from pip._internal.utils.misc import (
display_path,
hash_file,
hide_url,
is_installable_dir,
redact_auth_from_requirement,
)
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.unpacking import unpack_file
from pip._internal.vcs import vcs
logger = logging.getLogger(__name__)
logger = getLogger(__name__)
def _get_prepared_distribution(
@@ -65,10 +66,12 @@ def _get_prepared_distribution(
) -> BaseDistribution:
"""Prepare a distribution for installation."""
abstract_dist = make_distribution_for_install_requirement(req)
with build_tracker.track(req):
abstract_dist.prepare_distribution_metadata(
finder, build_isolation, check_build_deps
)
tracker_id = abstract_dist.build_tracker_id
if tracker_id is not None:
with build_tracker.track(req, tracker_id):
abstract_dist.prepare_distribution_metadata(
finder, build_isolation, check_build_deps
)
return abstract_dist.get_metadata_distribution()
@@ -78,13 +81,14 @@ def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
vcs_backend.unpack(location, url=hide_url(link.url), verbosity=verbosity)
@dataclass
class File:
def __init__(self, path: str, content_type: Optional[str]) -> None:
self.path = path
if content_type is None:
self.content_type = mimetypes.guess_type(path)[0]
else:
self.content_type = content_type
path: str
content_type: Optional[str] = None
def __post_init__(self) -> None:
if self.content_type is None:
self.content_type = mimetypes.guess_type(self.path)[0]
def get_http_url(
@@ -276,7 +280,7 @@ class RequirementPreparer:
information = str(display_path(req.link.file_path))
else:
message = "Collecting %s"
information = str(req.req or req)
information = redact_auth_from_requirement(req.req) if req.req else str(req)
# If we used req.req, inject requirement source if available (this
# would already be included if we used req directly)
@@ -317,21 +321,7 @@ class RequirementPreparer:
autodelete=True,
parallel_builds=parallel_builds,
)
# If a checkout exists, it's unwise to keep going. version
# inconsistencies are logged later, but do not fail the
# installation.
# FIXME: this won't upgrade when there's an existing
# package unpacked in `req.source_dir`
# TODO: this check is now probably dead code
if is_installable_dir(req.source_dir):
raise PreviousBuildDirError(
"pip can't proceed with requirements '{}' due to a"
"pre-existing build directory ({}). This is likely "
"due to a previous installation that failed . pip is "
"being responsible and not assuming it can delete this. "
"Please delete it and try again.".format(req, req.source_dir)
)
req.ensure_pristine_source_checkout()
def _get_linked_req_hashes(self, req: InstallRequirement) -> Hashes:
# By the time this is called, the requirement's link should have
@@ -394,7 +384,7 @@ class RequirementPreparer:
if metadata_link is None:
return None
assert req.req is not None
logger.info(
logger.verbose(
"Obtaining dependency information for %s from %s",
req.req,
metadata_link,
@@ -479,20 +469,19 @@ class RequirementPreparer:
for link, (filepath, _) in batch_download:
logger.debug("Downloading link %s to %s", link, filepath)
req = links_to_fully_download[link]
# Record the downloaded file path so wheel reqs can extract a Distribution
# in .get_dist().
req.local_file_path = filepath
# TODO: This needs fixing for sdists
# This is an emergency fix for #11847, which reports that
# distributions get downloaded twice when metadata is loaded
# from a PEP 658 standalone metadata file. Setting _downloaded
# fixes this for wheels, but breaks the sdist case (tests
# test_download_metadata). As PyPI is currently only serving
# metadata for wheels, this is not an immediate issue.
# Fixing the problem properly looks like it will require a
# complete refactoring of the `prepare_linked_requirements_more`
# logic, and I haven't a clue where to start on that, so for now
# I have fixed the issue *just* for wheels.
if req.is_wheel:
self._downloaded[req.link.url] = filepath
# Record that the file is downloaded so we don't do it again in
# _prepare_linked_requirement().
self._downloaded[req.link.url] = filepath
# If this is an sdist, we need to unpack it after downloading, but the
# .source_dir won't be set up until we are in _prepare_linked_requirement().
# Add the downloaded archive to the install requirement to unpack after
# preparing the source dir.
if not req.is_wheel:
req.needs_unpacked_archive(Path(filepath))
# This step is necessary to ensure all lazy wheels are processed
# successfully by the 'download', 'wheel', and 'install' commands.
@@ -616,8 +605,8 @@ class RequirementPreparer:
)
except NetworkConnectionError as exc:
raise InstallationError(
"Could not install requirement {} because of HTTP "
"error {} for URL {}".format(req, exc, link)
f"Could not install requirement {req} because of HTTP "
f"error {exc} for URL {link}"
)
else:
file_path = self._downloaded[link.url]
@@ -697,9 +686,9 @@ class RequirementPreparer:
with indent_log():
if self.require_hashes:
raise InstallationError(
"The editable requirement {} cannot be installed when "
f"The editable requirement {req} cannot be installed when "
"requiring hashes, because there is no single file to "
"hash.".format(req)
"hash."
)
req.ensure_has_source_dir(self.src_dir)
req.update_editable()
@@ -727,7 +716,7 @@ class RequirementPreparer:
assert req.satisfied_by, "req should have been satisfied but isn't"
assert skip_reason is not None, (
"did not get skip reason skipped but req.satisfied_by "
"is set to {}".format(req.satisfied_by)
f"is set to {req.satisfied_by}"
)
logger.info(
"Requirement %s: %s (%s)", skip_reason, req, req.satisfied_by.version