This commit is contained in:
ton
2024-10-07 10:13:40 +07:00
parent aa1631742f
commit 3a7d696db6
9729 changed files with 1832837 additions and 161742 deletions

View File

@@ -88,7 +88,7 @@ def join_continuation(lines):
['foobarbaz']
Not sure why, but...
The character preceeding the backslash is also elided.
The character preceding the backslash is also elided.
>>> list(join_continuation(['goo\\', 'dly']))
['godly']

View File

@@ -1,9 +1,11 @@
"""Stuff that differs in different Python versions and platform
distributions."""
import importlib.resources
import logging
import os
import sys
from typing import IO
__all__ = ["get_path_uid", "stdlib_pkgs", "WINDOWS"]
@@ -51,6 +53,20 @@ def get_path_uid(path: str) -> int:
return file_uid
# The importlib.resources.open_text function was deprecated in 3.11 with suggested
# replacement we use below.
if sys.version_info < (3, 11):
open_text_resource = importlib.resources.open_text
else:
def open_text_resource(
package: str, resource: str, encoding: str = "utf-8", errors: str = "strict"
) -> IO[str]:
return (importlib.resources.files(package) / resource).open(
"r", encoding=encoding, errors=errors
)
# packages in the stdlib that may have installation metadata, but should not be
# considered 'installed'. this theoretically could be determined based on
# dist.location (py27:`sysconfig.get_paths()['stdlib']`,

View File

@@ -87,9 +87,11 @@ def deprecated(
(reason, f"{DEPRECATION_MSG_PREFIX}{{}}"),
(
gone_in,
"pip {} will enforce this behaviour change."
if not is_gone
else "Since pip {}, this is no longer supported.",
(
"pip {} will enforce this behaviour change."
if not is_gone
else "Since pip {}, this is no longer supported."
),
),
(
replacement,
@@ -97,9 +99,11 @@ def deprecated(
),
(
feature_flag,
"You can use the flag --use-feature={} to test the upcoming behaviour."
if not is_gone
else None,
(
"You can use the flag --use-feature={} to test the upcoming behaviour."
if not is_gone
else None
),
),
(
issue,

View File

@@ -12,8 +12,8 @@ def direct_url_as_pep440_direct_reference(direct_url: DirectUrl, name: str) -> s
requirement = name + " @ "
fragments = []
if isinstance(direct_url.info, VcsInfo):
requirement += "{}+{}@{}".format(
direct_url.info.vcs, direct_url.url, direct_url.info.commit_id
requirement += (
f"{direct_url.info.vcs}+{direct_url.url}@{direct_url.info.commit_id}"
)
elif isinstance(direct_url.info, ArchiveInfo):
requirement += direct_url.url

View File

@@ -15,24 +15,31 @@ __all__ = [
]
def _egg_link_name(raw_name: str) -> str:
def _egg_link_names(raw_name: str) -> List[str]:
"""
Convert a Name metadata value to a .egg-link name, by applying
the same substitution as pkg_resources's safe_name function.
Note: we cannot use canonicalize_name because it has a different logic.
We also look for the raw name (without normalization) as setuptools 69 changed
the way it names .egg-link files (https://github.com/pypa/setuptools/issues/4167).
"""
return re.sub("[^A-Za-z0-9.]+", "-", raw_name) + ".egg-link"
return [
re.sub("[^A-Za-z0-9.]+", "-", raw_name) + ".egg-link",
f"{raw_name}.egg-link",
]
def egg_link_path_from_sys_path(raw_name: str) -> Optional[str]:
"""
Look for a .egg-link file for project name, by walking sys.path.
"""
egg_link_name = _egg_link_name(raw_name)
egg_link_names = _egg_link_names(raw_name)
for path_item in sys.path:
egg_link = os.path.join(path_item, egg_link_name)
if os.path.isfile(egg_link):
return egg_link
for egg_link_name in egg_link_names:
egg_link = os.path.join(path_item, egg_link_name)
if os.path.isfile(egg_link):
return egg_link
return None
@@ -64,9 +71,10 @@ def egg_link_path_from_location(raw_name: str) -> Optional[str]:
sites.append(user_site)
sites.append(site_packages)
egg_link_name = _egg_link_name(raw_name)
egg_link_names = _egg_link_names(raw_name)
for site in sites:
egglink = os.path.join(site, egg_link_name)
if os.path.isfile(egglink):
return egglink
for egg_link_name in egg_link_names:
egglink = os.path.join(site, egg_link_name)
if os.path.isfile(egglink):
return egglink
return None

View File

@@ -7,10 +7,9 @@ from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from typing import Any, BinaryIO, Generator, List, Union, cast
from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
from pip._internal.utils.compat import get_path_uid
from pip._internal.utils.misc import format_size
from pip._internal.utils.retry import retry
def check_path_owner(path: str) -> bool:
@@ -65,10 +64,7 @@ def adjacent_tmp_file(path: str, **kwargs: Any) -> Generator[BinaryIO, None, Non
os.fsync(result.fileno())
# Tenacity raises RetryError by default, explicitly raise the original exception
_replace_retry = retry(reraise=True, stop=stop_after_delay(1), wait=wait_fixed(0.25))
replace = _replace_retry(os.replace)
replace = retry(stop_after_delay=1, wait=0.25)(os.replace)
# test_writable_dir and _test_writable_dir_win are copied from Flit,

View File

@@ -40,7 +40,20 @@ def glibc_version_string_ctypes() -> Optional[str]:
# manpage says, "If filename is NULL, then the returned handle is for the
# main program". This way we can let the linker do the work to figure out
# which libc our process is actually using.
process_namespace = ctypes.CDLL(None)
#
# We must also handle the special case where the executable is not a
# dynamically linked executable. This can occur when using musl libc,
# for example. In this situation, dlopen() will error, leading to an
# OSError. Interestingly, at least in the case of musl, there is no
# errno set on the OSError. The single string argument used to construct
# OSError comes from libc itself and is therefore not portable to
# hard code here. In any case, failure to call dlopen() means we
# can't proceed, so we bail on our attempt.
try:
process_namespace = ctypes.CDLL(None)
except OSError:
return None
try:
gnu_get_libc_version = process_namespace.gnu_get_libc_version
except AttributeError:
@@ -50,7 +63,7 @@ def glibc_version_string_ctypes() -> Optional[str]:
# Call gnu_get_libc_version, which returns a string like "2.5"
gnu_get_libc_version.restype = ctypes.c_char_p
version_str = gnu_get_libc_version()
version_str: str = gnu_get_libc_version()
# py2 / py3 compatibility:
if not isinstance(version_str, str):
version_str = version_str.decode("ascii")

View File

@@ -1,5 +1,5 @@
import hashlib
from typing import TYPE_CHECKING, BinaryIO, Dict, Iterable, List, Optional
from typing import TYPE_CHECKING, BinaryIO, Dict, Iterable, List, NoReturn, Optional
from pip._internal.exceptions import HashMismatch, HashMissing, InstallationError
from pip._internal.utils.misc import read_chunks
@@ -7,10 +7,6 @@ from pip._internal.utils.misc import read_chunks
if TYPE_CHECKING:
from hashlib import _Hash
# NoReturn introduced in 3.6.2; imported only for type checking to maintain
# pip compatibility with older patch versions of Python 3.6
from typing import NoReturn
# The recommended hash algo of the moment. Change this whenever the state of
# the art changes; it won't hurt backward compatibility.
@@ -37,7 +33,7 @@ class Hashes:
if hashes is not None:
for alg, keys in hashes.items():
# Make sure values are always sorted (to ease equality checks)
allowed[alg] = sorted(keys)
allowed[alg] = [k.lower() for k in sorted(keys)]
self._allowed = allowed
def __and__(self, other: "Hashes") -> "Hashes":

View File

@@ -1,35 +0,0 @@
"""A helper module that injects SecureTransport, on import.
The import should be done as early as possible, to ensure all requests and
sessions (or whatever) are created after injecting SecureTransport.
Note that we only do the injection on macOS, when the linked OpenSSL is too
old to handle TLSv1.2.
"""
import sys
def inject_securetransport() -> None:
# Only relevant on macOS
if sys.platform != "darwin":
return
try:
import ssl
except ImportError:
return
# Checks for OpenSSL 1.0.1
if ssl.OPENSSL_VERSION_NUMBER >= 0x1000100F:
return
try:
from pip._vendor.urllib3.contrib import securetransport
except (ImportError, OSError):
return
securetransport.inject_into_urllib3()
inject_securetransport()

View File

@@ -154,9 +154,9 @@ class RichPipStreamHandler(RichHandler):
style: Optional[Style] = None
# If we are given a diagnostic error to present, present it with indentation.
assert isinstance(record.args, tuple)
if record.msg == "[present-rich] %s" and len(record.args) == 1:
rich_renderable = record.args[0]
if getattr(record, "rich", False):
assert isinstance(record.args, tuple)
(rich_renderable,) = record.args
assert isinstance(
rich_renderable, (ConsoleRenderable, RichCast, str)
), f"{rich_renderable} is not rich-console-renderable"
@@ -212,7 +212,6 @@ class MaxLevelFilter(Filter):
class ExcludeLoggerFilter(Filter):
"""
A logging Filter that excludes records from a logger (or its children).
"""

View File

@@ -1,8 +1,6 @@
import contextlib
import errno
import getpass
import hashlib
import io
import logging
import os
import posixpath
@@ -11,14 +9,16 @@ import stat
import sys
import sysconfig
import urllib.parse
from dataclasses import dataclass
from functools import partial
from io import StringIO
from itertools import filterfalse, tee, zip_longest
from types import TracebackType
from pathlib import Path
from types import FunctionType, TracebackType
from typing import (
Any,
BinaryIO,
Callable,
ContextManager,
Dict,
Generator,
Iterable,
@@ -33,13 +33,14 @@ from typing import (
cast,
)
from pip._vendor.packaging.requirements import Requirement
from pip._vendor.pyproject_hooks import BuildBackendHookCaller
from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
from pip import __version__
from pip._internal.exceptions import CommandError, ExternallyManagedEnvironment
from pip._internal.locations import get_major_minor_version
from pip._internal.utils.compat import WINDOWS
from pip._internal.utils.retry import retry
from pip._internal.utils.virtualenv import running_under_virtualenv
__all__ = [
@@ -53,7 +54,6 @@ __all__ = [
"normalize_path",
"renames",
"get_prog",
"captured_stdout",
"ensure_dir",
"remove_auth_from_url",
"check_externally_managed",
@@ -66,17 +66,17 @@ T = TypeVar("T")
ExcInfo = Tuple[Type[BaseException], BaseException, TracebackType]
VersionInfo = Tuple[int, int, int]
NetlocTuple = Tuple[str, Tuple[Optional[str], Optional[str]]]
OnExc = Callable[[FunctionType, Path, BaseException], Any]
OnErr = Callable[[FunctionType, Path, ExcInfo], Any]
FILE_CHUNK_SIZE = 1024 * 1024
def get_pip_version() -> str:
pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..")
pip_pkg_dir = os.path.abspath(pip_pkg_dir)
return "pip {} from {} (python {})".format(
__version__,
pip_pkg_dir,
get_major_minor_version(),
)
return f"pip {__version__} from {pip_pkg_dir} (python {get_major_minor_version()})"
def normalize_version_info(py_version_info: Tuple[int, ...]) -> Tuple[int, int, int]:
@@ -121,35 +121,74 @@ def get_prog() -> str:
# Retry every half second for up to 3 seconds
# Tenacity raises RetryError by default, explicitly raise the original exception
@retry(reraise=True, stop=stop_after_delay(3), wait=wait_fixed(0.5))
def rmtree(dir: str, ignore_errors: bool = False) -> None:
@retry(stop_after_delay=3, wait=0.5)
def rmtree(
dir: str, ignore_errors: bool = False, onexc: Optional[OnExc] = None
) -> None:
if ignore_errors:
onexc = _onerror_ignore
if onexc is None:
onexc = _onerror_reraise
handler: OnErr = partial(
# `[func, path, Union[ExcInfo, BaseException]] -> Any` is equivalent to
# `Union[([func, path, ExcInfo] -> Any), ([func, path, BaseException] -> Any)]`.
cast(Union[OnExc, OnErr], rmtree_errorhandler),
onexc=onexc,
)
if sys.version_info >= (3, 12):
shutil.rmtree(dir, ignore_errors=ignore_errors, onexc=rmtree_errorhandler)
# See https://docs.python.org/3.12/whatsnew/3.12.html#shutil.
shutil.rmtree(dir, onexc=handler) # type: ignore
else:
shutil.rmtree(dir, ignore_errors=ignore_errors, onerror=rmtree_errorhandler)
shutil.rmtree(dir, onerror=handler) # type: ignore
def _onerror_ignore(*_args: Any) -> None:
pass
def _onerror_reraise(*_args: Any) -> None:
raise # noqa: PLE0704 - Bare exception used to reraise existing exception
def rmtree_errorhandler(
func: Callable[..., Any], path: str, exc_info: Union[ExcInfo, BaseException]
func: FunctionType,
path: Path,
exc_info: Union[ExcInfo, BaseException],
*,
onexc: OnExc = _onerror_reraise,
) -> None:
"""On Windows, the files in .svn are read-only, so when rmtree() tries to
remove them, an exception is thrown. We catch that here, remove the
read-only attribute, and hopefully continue without problems."""
"""
`rmtree` error handler to 'force' a file remove (i.e. like `rm -f`).
* If a file is readonly then it's write flag is set and operation is
retried.
* `onerror` is the original callback from `rmtree(... onerror=onerror)`
that is chained at the end if the "rm -f" still fails.
"""
try:
has_attr_readonly = not (os.stat(path).st_mode & stat.S_IWRITE)
st_mode = os.stat(path).st_mode
except OSError:
# it's equivalent to os.path.exists
return
if has_attr_readonly:
if not st_mode & stat.S_IWRITE:
# convert to read/write
os.chmod(path, stat.S_IWRITE)
# use the original function to repeat the operation
func(path)
return
else:
raise
try:
os.chmod(path, st_mode | stat.S_IWRITE)
except OSError:
pass
else:
# use the original function to repeat the operation
try:
func(path)
return
except OSError:
pass
if not isinstance(exc_info, BaseException):
_, exc_info, _ = exc_info
onexc(func, path, exc_info)
def display_path(path: str) -> str:
@@ -232,13 +271,13 @@ def strtobool(val: str) -> int:
def format_size(bytes: float) -> str:
if bytes > 1000 * 1000:
return "{:.1f} MB".format(bytes / 1000.0 / 1000)
return f"{bytes / 1000.0 / 1000:.1f} MB"
elif bytes > 10 * 1000:
return "{} kB".format(int(bytes / 1000))
return f"{int(bytes / 1000)} kB"
elif bytes > 1000:
return "{:.1f} kB".format(bytes / 1000.0)
return f"{bytes / 1000.0:.1f} kB"
else:
return "{} bytes".format(int(bytes))
return f"{int(bytes)} bytes"
def tabulate(rows: Iterable[Iterable[Any]]) -> Tuple[List[str], List[int]]:
@@ -273,7 +312,7 @@ def is_installable_dir(path: str) -> bool:
def read_chunks(
file: BinaryIO, size: int = io.DEFAULT_BUFFER_SIZE
file: BinaryIO, size: int = FILE_CHUNK_SIZE
) -> Generator[bytes, None, None]:
"""Yield pieces of data from a file-like object until EOF."""
while True:
@@ -356,40 +395,6 @@ class StreamWrapper(StringIO):
return self.orig_stream.encoding
@contextlib.contextmanager
def captured_output(stream_name: str) -> Generator[StreamWrapper, None, None]:
"""Return a context manager used by captured_stdout/stdin/stderr
that temporarily replaces the sys stream *stream_name* with a StringIO.
Taken from Lib/support/__init__.py in the CPython repo.
"""
orig_stdout = getattr(sys, stream_name)
setattr(sys, stream_name, StreamWrapper.from_stream(orig_stdout))
try:
yield getattr(sys, stream_name)
finally:
setattr(sys, stream_name, orig_stdout)
def captured_stdout() -> ContextManager[StreamWrapper]:
"""Capture the output of sys.stdout:
with captured_stdout() as stdout:
print('hello')
self.assertEqual(stdout.getvalue(), 'hello\n')
Taken from Lib/support/__init__.py in the CPython repo.
"""
return captured_output("stdout")
def captured_stderr() -> ContextManager[StreamWrapper]:
"""
See captured_stdout().
"""
return captured_output("stderr")
# Simulates an enum
def enum(*sequential: Any, **named: Any) -> Type[Any]:
enums = dict(zip(sequential, range(len(sequential))), **named)
@@ -475,9 +480,7 @@ def redact_netloc(netloc: str) -> str:
else:
user = urllib.parse.quote(user)
password = ":****"
return "{user}{password}@{netloc}".format(
user=user, password=password, netloc=netloc
)
return f"{user}{password}@{netloc}"
def _transform_url(
@@ -532,13 +535,20 @@ def redact_auth_from_url(url: str) -> str:
return _transform_url(url, _redact_netloc)[0]
def redact_auth_from_requirement(req: Requirement) -> str:
"""Replace the password in a given requirement url with ****."""
if not req.url:
return str(req)
return str(req).replace(req.url, redact_auth_from_url(req.url))
@dataclass(frozen=True)
class HiddenText:
def __init__(self, secret: str, redacted: str) -> None:
self.secret = secret
self.redacted = redacted
secret: str
redacted: str
def __repr__(self) -> str:
return "<HiddenText {!r}>".format(str(self))
return f"<HiddenText {str(self)!r}>"
def __str__(self) -> str:
return self.redacted
@@ -632,8 +642,7 @@ def pairwise(iterable: Iterable[Any]) -> Iterator[Tuple[Any, Any]]:
def partition(
pred: Callable[[T], bool],
iterable: Iterable[T],
pred: Callable[[T], bool], iterable: Iterable[T]
) -> Tuple[Iterable[T], Iterable[T]]:
"""
Use a predicate to partition entries into false entries and true entries,
@@ -733,3 +742,36 @@ class ConfiguredBuildBackendHookCaller(BuildBackendHookCaller):
config_settings=cs,
_allow_fallback=_allow_fallback,
)
def warn_if_run_as_root() -> None:
"""Output a warning for sudo users on Unix.
In a virtual environment, sudo pip still writes to virtualenv.
On Windows, users may run pip as Administrator without issues.
This warning only applies to Unix root users outside of virtualenv.
"""
if running_under_virtualenv():
return
if not hasattr(os, "getuid"):
return
# On Windows, there are no "system managed" Python packages. Installing as
# Administrator via pip is the correct way of updating system environments.
#
# We choose sys.platform over utils.compat.WINDOWS here to enable Mypy platform
# checks: https://mypy.readthedocs.io/en/stable/common_issues.html
if sys.platform == "win32" or sys.platform == "cygwin":
return
if os.getuid() != 0:
return
logger.warning(
"Running pip as the 'root' user can result in broken permissions and "
"conflicting behaviour with the system package manager, possibly "
"rendering your system unusable."
"It is recommended to use a virtual environment instead: "
"https://pip.pypa.io/warnings/venv. "
"Use the --root-user-action option if you know what you are doing and "
"want to suppress this warning."
)

View File

@@ -1,39 +0,0 @@
"""Utilities for defining models
"""
import operator
from typing import Any, Callable, Type
class KeyBasedCompareMixin:
"""Provides comparison capabilities that is based on a key"""
__slots__ = ["_compare_key", "_defining_class"]
def __init__(self, key: Any, defining_class: Type["KeyBasedCompareMixin"]) -> None:
self._compare_key = key
self._defining_class = defining_class
def __hash__(self) -> int:
return hash(self._compare_key)
def __lt__(self, other: Any) -> bool:
return self._compare(other, operator.__lt__)
def __le__(self, other: Any) -> bool:
return self._compare(other, operator.__le__)
def __gt__(self, other: Any) -> bool:
return self._compare(other, operator.__gt__)
def __ge__(self, other: Any) -> bool:
return self._compare(other, operator.__ge__)
def __eq__(self, other: Any) -> bool:
return self._compare(other, operator.__eq__)
def _compare(self, other: Any, method: Callable[[Any, Any], bool]) -> bool:
if not isinstance(other, self._defining_class):
return NotImplemented
return method(self._compare_key, other._compare_key)

View File

@@ -34,7 +34,7 @@ def check_requires_python(
return python_version in requires_python_specifier
@functools.lru_cache(maxsize=512)
@functools.lru_cache(maxsize=2048)
def get_requirement(req_string: str) -> Requirement:
"""Construct a packaging.Requirement object with caching"""
# Parsing requirement strings is expensive, and is also expected to happen

View File

@@ -0,0 +1,42 @@
import functools
from time import perf_counter, sleep
from typing import Callable, TypeVar
from pip._vendor.typing_extensions import ParamSpec
T = TypeVar("T")
P = ParamSpec("P")
def retry(
wait: float, stop_after_delay: float
) -> Callable[[Callable[P, T]], Callable[P, T]]:
"""Decorator to automatically retry a function on error.
If the function raises, the function is recalled with the same arguments
until it returns or the time limit is reached. When the time limit is
surpassed, the last exception raised is reraised.
:param wait: The time to wait after an error before retrying, in seconds.
:param stop_after_delay: The time limit after which retries will cease,
in seconds.
"""
def wrapper(func: Callable[P, T]) -> Callable[P, T]:
@functools.wraps(func)
def retry_wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
# The performance counter is monotonic on all platforms we care
# about and has much better resolution than time.monotonic().
start_time = perf_counter()
while True:
try:
return func(*args, **kwargs)
except Exception:
if perf_counter() - start_time > stop_after_delay:
raise
sleep(wait)
return retry_wrapped
return wrapper

View File

@@ -2,16 +2,7 @@ import logging
import os
import shlex
import subprocess
from typing import (
TYPE_CHECKING,
Any,
Callable,
Iterable,
List,
Mapping,
Optional,
Union,
)
from typing import Any, Callable, Iterable, List, Literal, Mapping, Optional, Union
from pip._vendor.rich.markup import escape
@@ -20,12 +11,6 @@ from pip._internal.exceptions import InstallationSubprocessError
from pip._internal.utils.logging import VERBOSE, subprocess_logger
from pip._internal.utils.misc import HiddenText
if TYPE_CHECKING:
# Literal was introduced in Python 3.8.
#
# TODO: Remove `if TYPE_CHECKING` when dropping support for Python 3.7.
from typing import Literal
CommandArgs = List[Union[str, HiddenText]]
@@ -209,7 +194,7 @@ def call_subprocess(
output_lines=all_output if not showing_subprocess else None,
)
if log_failed_cmd:
subprocess_logger.error("[present-rich] %s", error)
subprocess_logger.error("%s", error, extra={"rich": True})
subprocess_logger.verbose(
"[bold magenta]full command[/]: [blue]%s[/]",
escape(format_command_args(cmd)),

View File

@@ -3,8 +3,19 @@ import itertools
import logging
import os.path
import tempfile
import traceback
from contextlib import ExitStack, contextmanager
from typing import Any, Dict, Generator, Optional, TypeVar, Union
from pathlib import Path
from typing import (
Any,
Callable,
Dict,
Generator,
List,
Optional,
TypeVar,
Union,
)
from pip._internal.utils.misc import enum, rmtree
@@ -106,6 +117,7 @@ class TempDirectory:
delete: Union[bool, None, _Default] = _default,
kind: str = "temp",
globally_managed: bool = False,
ignore_cleanup_errors: bool = True,
):
super().__init__()
@@ -128,6 +140,7 @@ class TempDirectory:
self._deleted = False
self.delete = delete
self.kind = kind
self.ignore_cleanup_errors = ignore_cleanup_errors
if globally_managed:
assert _tempdir_manager is not None
@@ -170,7 +183,44 @@ class TempDirectory:
self._deleted = True
if not os.path.exists(self._path):
return
rmtree(self._path)
errors: List[BaseException] = []
def onerror(
func: Callable[..., Any],
path: Path,
exc_val: BaseException,
) -> None:
"""Log a warning for a `rmtree` error and continue"""
formatted_exc = "\n".join(
traceback.format_exception_only(type(exc_val), exc_val)
)
formatted_exc = formatted_exc.rstrip() # remove trailing new line
if func in (os.unlink, os.remove, os.rmdir):
logger.debug(
"Failed to remove a temporary file '%s' due to %s.\n",
path,
formatted_exc,
)
else:
logger.debug("%s failed with %s.", func.__qualname__, formatted_exc)
errors.append(exc_val)
if self.ignore_cleanup_errors:
try:
# first try with @retry; retrying to handle ephemeral errors
rmtree(self._path, ignore_errors=False)
except OSError:
# last pass ignore/log all errors
rmtree(self._path, onexc=onerror)
if errors:
logger.warning(
"Failed to remove contents in a temporary directory '%s'.\n"
"You can safely remove it manually.",
self._path,
)
else:
rmtree(self._path)
class AdjacentTempDirectory(TempDirectory):

View File

@@ -5,6 +5,7 @@ import logging
import os
import shutil
import stat
import sys
import tarfile
import zipfile
from typing import Iterable, List, Optional
@@ -85,12 +86,16 @@ def is_within_directory(directory: str, target: str) -> bool:
return prefix == abs_directory
def _get_default_mode_plus_executable() -> int:
return 0o777 & ~current_umask() | 0o111
def set_extracted_file_to_default_mode_plus_executable(path: str) -> None:
"""
Make file present at path have execute for user/group/world
(chmod +x) is no-op on windows per python docs
"""
os.chmod(path, (0o777 & ~current_umask() | 0o111))
os.chmod(path, _get_default_mode_plus_executable())
def zip_item_is_executable(info: ZipInfo) -> bool:
@@ -151,8 +156,8 @@ def untar_file(filename: str, location: str) -> None:
Untar the file (with path `filename`) to the destination `location`.
All files are written based on system defaults and umask (i.e. permissions
are not preserved), except that regular file members with any execute
permissions (user, group, or world) have "chmod +x" applied after being
written. Note that for windows, any execute changes using os.chmod are
permissions (user, group, or world) have "chmod +x" applied on top of the
default. Note that for windows, any execute changes using os.chmod are
no-ops per the python docs.
"""
ensure_dir(location)
@@ -170,62 +175,137 @@ def untar_file(filename: str, location: str) -> None:
filename,
)
mode = "r:*"
tar = tarfile.open(filename, mode, encoding="utf-8")
try:
leading = has_leading_dir([member.name for member in tar.getmembers()])
for member in tar.getmembers():
fn = member.name
# PEP 706 added `tarfile.data_filter`, and made some other changes to
# Python's tarfile module (see below). The features were backported to
# security releases.
try:
data_filter = tarfile.data_filter
except AttributeError:
_untar_without_filter(filename, location, tar, leading)
else:
default_mode_plus_executable = _get_default_mode_plus_executable()
if leading:
fn = split_leading_dir(fn)[1]
path = os.path.join(location, fn)
if not is_within_directory(location, path):
message = (
"The tar file ({}) has a file ({}) trying to install "
"outside target directory ({})"
)
raise InstallationError(message.format(filename, path, location))
if member.isdir():
ensure_dir(path)
elif member.issym():
# Strip the leading directory from all files in the archive,
# including hardlink targets (which are relative to the
# unpack location).
for member in tar.getmembers():
name_lead, name_rest = split_leading_dir(member.name)
member.name = name_rest
if member.islnk():
lnk_lead, lnk_rest = split_leading_dir(member.linkname)
if lnk_lead == name_lead:
member.linkname = lnk_rest
def pip_filter(member: tarfile.TarInfo, path: str) -> tarfile.TarInfo:
orig_mode = member.mode
try:
tar._extract_member(member, path)
except Exception as exc:
# Some corrupt tar files seem to produce this
# (specifically bad symlinks)
logger.warning(
"In the tar file %s the member %s is invalid: %s",
filename,
member.name,
exc,
try:
member = data_filter(member, location)
except tarfile.LinkOutsideDestinationError:
if sys.version_info[:3] in {
(3, 8, 17),
(3, 9, 17),
(3, 10, 12),
(3, 11, 4),
}:
# The tarfile filter in specific Python versions
# raises LinkOutsideDestinationError on valid input
# (https://github.com/python/cpython/issues/107845)
# Ignore the error there, but do use the
# more lax `tar_filter`
member = tarfile.tar_filter(member, location)
else:
raise
except tarfile.TarError as exc:
message = "Invalid member in the tar file {}: {}"
# Filter error messages mention the member name.
# No need to add it here.
raise InstallationError(
message.format(
filename,
exc,
)
)
continue
else:
try:
fp = tar.extractfile(member)
except (KeyError, AttributeError) as exc:
# Some corrupt tar files seem to produce this
# (specifically bad symlinks)
logger.warning(
"In the tar file %s the member %s is invalid: %s",
filename,
member.name,
exc,
)
continue
ensure_dir(os.path.dirname(path))
assert fp is not None
with open(path, "wb") as destfp:
shutil.copyfileobj(fp, destfp)
fp.close()
# Update the timestamp (useful for cython compiled files)
tar.utime(member, path)
# member have any execute permissions for user/group/world?
if member.mode & 0o111:
set_extracted_file_to_default_mode_plus_executable(path)
if member.isfile() and orig_mode & 0o111:
member.mode = default_mode_plus_executable
else:
# See PEP 706 note above.
# The PEP changed this from `int` to `Optional[int]`,
# where None means "use the default". Mypy doesn't
# know this yet.
member.mode = None # type: ignore [assignment]
return member
tar.extractall(location, filter=pip_filter)
finally:
tar.close()
def _untar_without_filter(
filename: str,
location: str,
tar: tarfile.TarFile,
leading: bool,
) -> None:
"""Fallback for Python without tarfile.data_filter"""
for member in tar.getmembers():
fn = member.name
if leading:
fn = split_leading_dir(fn)[1]
path = os.path.join(location, fn)
if not is_within_directory(location, path):
message = (
"The tar file ({}) has a file ({}) trying to install "
"outside target directory ({})"
)
raise InstallationError(message.format(filename, path, location))
if member.isdir():
ensure_dir(path)
elif member.issym():
try:
tar._extract_member(member, path)
except Exception as exc:
# Some corrupt tar files seem to produce this
# (specifically bad symlinks)
logger.warning(
"In the tar file %s the member %s is invalid: %s",
filename,
member.name,
exc,
)
continue
else:
try:
fp = tar.extractfile(member)
except (KeyError, AttributeError) as exc:
# Some corrupt tar files seem to produce this
# (specifically bad symlinks)
logger.warning(
"In the tar file %s the member %s is invalid: %s",
filename,
member.name,
exc,
)
continue
ensure_dir(os.path.dirname(path))
assert fp is not None
with open(path, "wb") as destfp:
shutil.copyfileobj(fp, destfp)
fp.close()
# Update the timestamp (useful for cython compiled files)
tar.utime(member, path)
# member have any execute permissions for user/group/world?
if member.mode & 0o111:
set_extracted_file_to_default_mode_plus_executable(path)
def unpack_file(
filename: str,
location: str,

View File

@@ -2,17 +2,10 @@ import os
import string
import urllib.parse
import urllib.request
from typing import Optional
from .compat import WINDOWS
def get_url_scheme(url: str) -> Optional[str]:
if ":" not in url:
return None
return url.split(":", 1)[0].lower()
def path_to_url(path: str) -> str:
"""
Convert a path to a file: URL. The path will be made absolute and have

View File

@@ -28,7 +28,7 @@ def parse_wheel(wheel_zip: ZipFile, name: str) -> Tuple[str, Message]:
metadata = wheel_metadata(wheel_zip, info_dir)
version = wheel_version(metadata)
except UnsupportedWheel as e:
raise UnsupportedWheel("{} has an invalid wheel, {}".format(name, str(e)))
raise UnsupportedWheel(f"{name} has an invalid wheel, {e}")
check_compatibility(version, name)
@@ -60,9 +60,7 @@ def wheel_dist_info_dir(source: ZipFile, name: str) -> str:
canonical_name = canonicalize_name(name)
if not info_dir_name.startswith(canonical_name):
raise UnsupportedWheel(
".dist-info directory {!r} does not start with {!r}".format(
info_dir, canonical_name
)
f".dist-info directory {info_dir!r} does not start with {canonical_name!r}"
)
return info_dir