update
This commit is contained in:
@@ -2,89 +2,86 @@
|
||||
|
||||
Utility functions for manipulating directories and directory trees."""
|
||||
|
||||
import functools
|
||||
import itertools
|
||||
import os
|
||||
import errno
|
||||
from .errors import DistutilsInternalError, DistutilsFileError
|
||||
import pathlib
|
||||
|
||||
from . import file_util
|
||||
from ._log import log
|
||||
|
||||
# cache for by mkpath() -- in addition to cheapening redundant calls,
|
||||
# eliminates redundant "creating /foo/bar/baz" messages in dry-run mode
|
||||
_path_created = {}
|
||||
from .errors import DistutilsFileError, DistutilsInternalError
|
||||
|
||||
|
||||
def mkpath(name, mode=0o777, verbose=1, dry_run=0): # noqa: C901
|
||||
class SkipRepeatAbsolutePaths(set):
|
||||
"""
|
||||
Cache for mkpath.
|
||||
|
||||
In addition to cheapening redundant calls, eliminates redundant
|
||||
"creating /foo/bar/baz" messages in dry-run mode.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
SkipRepeatAbsolutePaths.instance = self
|
||||
|
||||
@classmethod
|
||||
def clear(cls):
|
||||
super(cls, cls.instance).clear()
|
||||
|
||||
def wrap(self, func):
|
||||
@functools.wraps(func)
|
||||
def wrapper(path, *args, **kwargs):
|
||||
if path.absolute() in self:
|
||||
return
|
||||
self.add(path.absolute())
|
||||
return func(path, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
# Python 3.8 compatibility
|
||||
wrapper = SkipRepeatAbsolutePaths().wrap
|
||||
|
||||
|
||||
@functools.singledispatch
|
||||
@wrapper
|
||||
def mkpath(name: pathlib.Path, mode=0o777, verbose=True, dry_run=False):
|
||||
"""Create a directory and any missing ancestor directories.
|
||||
|
||||
If the directory already exists (or if 'name' is the empty string, which
|
||||
means the current directory, which of course exists), then do nothing.
|
||||
Raise DistutilsFileError if unable to create some directory along the way
|
||||
(eg. some sub-path exists, but is a file rather than a directory).
|
||||
If 'verbose' is true, print a one-line summary of each mkdir to stdout.
|
||||
If 'verbose' is true, log the directory created.
|
||||
Return the list of directories actually created.
|
||||
|
||||
os.makedirs is not used because:
|
||||
|
||||
a) It's new to Python 1.5.2, and
|
||||
b) it blows up if the directory already exists (in which case it should
|
||||
silently succeed).
|
||||
"""
|
||||
if verbose and not name.is_dir():
|
||||
log.info("creating %s", name)
|
||||
|
||||
global _path_created
|
||||
ancestry = itertools.chain((name,), name.parents)
|
||||
missing = (path for path in ancestry if not path.is_dir())
|
||||
|
||||
# Detect a common bug -- name is None
|
||||
if not isinstance(name, str):
|
||||
raise DistutilsInternalError(
|
||||
"mkpath: 'name' must be a string (got {!r})".format(name)
|
||||
)
|
||||
try:
|
||||
dry_run or name.mkdir(mode=mode, parents=True, exist_ok=True)
|
||||
except OSError as exc:
|
||||
raise DistutilsFileError(f"could not create '{name}': {exc.args[-1]}")
|
||||
|
||||
# XXX what's the better way to handle verbosity? print as we create
|
||||
# each directory in the path (the current behaviour), or only announce
|
||||
# the creation of the whole path? (quite easy to do the latter since
|
||||
# we're not using a recursive algorithm)
|
||||
|
||||
name = os.path.normpath(name)
|
||||
created_dirs = []
|
||||
if os.path.isdir(name) or name == '':
|
||||
return created_dirs
|
||||
if _path_created.get(os.path.abspath(name)):
|
||||
return created_dirs
|
||||
|
||||
(head, tail) = os.path.split(name)
|
||||
tails = [tail] # stack of lone dirs to create
|
||||
|
||||
while head and tail and not os.path.isdir(head):
|
||||
(head, tail) = os.path.split(head)
|
||||
tails.insert(0, tail) # push next higher dir onto stack
|
||||
|
||||
# now 'head' contains the deepest directory that already exists
|
||||
# (that is, the child of 'head' in 'name' is the highest directory
|
||||
# that does *not* exist)
|
||||
for d in tails:
|
||||
# print "head = %s, d = %s: " % (head, d),
|
||||
head = os.path.join(head, d)
|
||||
abs_head = os.path.abspath(head)
|
||||
|
||||
if _path_created.get(abs_head):
|
||||
continue
|
||||
|
||||
if verbose >= 1:
|
||||
log.info("creating %s", head)
|
||||
|
||||
if not dry_run:
|
||||
try:
|
||||
os.mkdir(head, mode)
|
||||
except OSError as exc:
|
||||
if not (exc.errno == errno.EEXIST and os.path.isdir(head)):
|
||||
raise DistutilsFileError(
|
||||
"could not create '{}': {}".format(head, exc.args[-1])
|
||||
)
|
||||
created_dirs.append(head)
|
||||
|
||||
_path_created[abs_head] = 1
|
||||
return created_dirs
|
||||
return list(map(str, missing))
|
||||
|
||||
|
||||
def create_tree(base_dir, files, mode=0o777, verbose=1, dry_run=0):
|
||||
@mkpath.register
|
||||
def _(name: str, *args, **kwargs):
|
||||
return mkpath(pathlib.Path(name), *args, **kwargs)
|
||||
|
||||
|
||||
@mkpath.register
|
||||
def _(name: None, *args, **kwargs):
|
||||
"""
|
||||
Detect a common bug -- name is None.
|
||||
"""
|
||||
raise DistutilsInternalError(f"mkpath: 'name' must be a string (got {name!r})")
|
||||
|
||||
|
||||
def create_tree(base_dir, files, mode=0o777, verbose=True, dry_run=False):
|
||||
"""Create all the empty directories under 'base_dir' needed to put 'files'
|
||||
there.
|
||||
|
||||
@@ -95,24 +92,22 @@ def create_tree(base_dir, files, mode=0o777, verbose=1, dry_run=0):
|
||||
'dry_run' flags are as for 'mkpath()'.
|
||||
"""
|
||||
# First get the list of directories to create
|
||||
need_dir = set()
|
||||
for file in files:
|
||||
need_dir.add(os.path.join(base_dir, os.path.dirname(file)))
|
||||
need_dir = set(os.path.join(base_dir, os.path.dirname(file)) for file in files)
|
||||
|
||||
# Now create them
|
||||
for dir in sorted(need_dir):
|
||||
mkpath(dir, mode, verbose=verbose, dry_run=dry_run)
|
||||
|
||||
|
||||
def copy_tree( # noqa: C901
|
||||
def copy_tree(
|
||||
src,
|
||||
dst,
|
||||
preserve_mode=1,
|
||||
preserve_times=1,
|
||||
preserve_symlinks=0,
|
||||
update=0,
|
||||
verbose=1,
|
||||
dry_run=0,
|
||||
preserve_mode=True,
|
||||
preserve_times=True,
|
||||
preserve_symlinks=False,
|
||||
update=False,
|
||||
verbose=True,
|
||||
dry_run=False,
|
||||
):
|
||||
"""Copy an entire directory tree 'src' to a new location 'dst'.
|
||||
|
||||
@@ -133,67 +128,82 @@ def copy_tree( # noqa: C901
|
||||
(the default), the destination of the symlink will be copied.
|
||||
'update' and 'verbose' are the same as for 'copy_file'.
|
||||
"""
|
||||
from distutils.file_util import copy_file
|
||||
|
||||
if not dry_run and not os.path.isdir(src):
|
||||
raise DistutilsFileError("cannot copy tree '%s': not a directory" % src)
|
||||
raise DistutilsFileError(f"cannot copy tree '{src}': not a directory")
|
||||
try:
|
||||
names = os.listdir(src)
|
||||
except OSError as e:
|
||||
if dry_run:
|
||||
names = []
|
||||
else:
|
||||
raise DistutilsFileError(
|
||||
"error listing files in '{}': {}".format(src, e.strerror)
|
||||
)
|
||||
raise DistutilsFileError(f"error listing files in '{src}': {e.strerror}")
|
||||
|
||||
if not dry_run:
|
||||
mkpath(dst, verbose=verbose)
|
||||
|
||||
outputs = []
|
||||
copy_one = functools.partial(
|
||||
_copy_one,
|
||||
src=src,
|
||||
dst=dst,
|
||||
preserve_symlinks=preserve_symlinks,
|
||||
verbose=verbose,
|
||||
dry_run=dry_run,
|
||||
preserve_mode=preserve_mode,
|
||||
preserve_times=preserve_times,
|
||||
update=update,
|
||||
)
|
||||
return list(itertools.chain.from_iterable(map(copy_one, names)))
|
||||
|
||||
for n in names:
|
||||
src_name = os.path.join(src, n)
|
||||
dst_name = os.path.join(dst, n)
|
||||
|
||||
if n.startswith('.nfs'):
|
||||
# skip NFS rename files
|
||||
continue
|
||||
def _copy_one(
|
||||
name,
|
||||
*,
|
||||
src,
|
||||
dst,
|
||||
preserve_symlinks,
|
||||
verbose,
|
||||
dry_run,
|
||||
preserve_mode,
|
||||
preserve_times,
|
||||
update,
|
||||
):
|
||||
src_name = os.path.join(src, name)
|
||||
dst_name = os.path.join(dst, name)
|
||||
|
||||
if preserve_symlinks and os.path.islink(src_name):
|
||||
link_dest = os.readlink(src_name)
|
||||
if verbose >= 1:
|
||||
log.info("linking %s -> %s", dst_name, link_dest)
|
||||
if not dry_run:
|
||||
os.symlink(link_dest, dst_name)
|
||||
outputs.append(dst_name)
|
||||
if name.startswith('.nfs'):
|
||||
# skip NFS rename files
|
||||
return
|
||||
|
||||
elif os.path.isdir(src_name):
|
||||
outputs.extend(
|
||||
copy_tree(
|
||||
src_name,
|
||||
dst_name,
|
||||
preserve_mode,
|
||||
preserve_times,
|
||||
preserve_symlinks,
|
||||
update,
|
||||
verbose=verbose,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
)
|
||||
else:
|
||||
copy_file(
|
||||
src_name,
|
||||
dst_name,
|
||||
preserve_mode,
|
||||
preserve_times,
|
||||
update,
|
||||
verbose=verbose,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
outputs.append(dst_name)
|
||||
if preserve_symlinks and os.path.islink(src_name):
|
||||
link_dest = os.readlink(src_name)
|
||||
if verbose >= 1:
|
||||
log.info("linking %s -> %s", dst_name, link_dest)
|
||||
if not dry_run:
|
||||
os.symlink(link_dest, dst_name)
|
||||
yield dst_name
|
||||
|
||||
return outputs
|
||||
elif os.path.isdir(src_name):
|
||||
yield from copy_tree(
|
||||
src_name,
|
||||
dst_name,
|
||||
preserve_mode,
|
||||
preserve_times,
|
||||
preserve_symlinks,
|
||||
update,
|
||||
verbose=verbose,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
else:
|
||||
file_util.copy_file(
|
||||
src_name,
|
||||
dst_name,
|
||||
preserve_mode,
|
||||
preserve_times,
|
||||
update,
|
||||
verbose=verbose,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
yield dst_name
|
||||
|
||||
|
||||
def _build_cmdtuple(path, cmdtuples):
|
||||
@@ -207,14 +217,12 @@ def _build_cmdtuple(path, cmdtuples):
|
||||
cmdtuples.append((os.rmdir, path))
|
||||
|
||||
|
||||
def remove_tree(directory, verbose=1, dry_run=0):
|
||||
def remove_tree(directory, verbose=True, dry_run=False):
|
||||
"""Recursively remove an entire directory tree.
|
||||
|
||||
Any errors are ignored (apart from being reported to stdout if 'verbose'
|
||||
is true).
|
||||
"""
|
||||
global _path_created
|
||||
|
||||
if verbose >= 1:
|
||||
log.info("removing '%s' (and everything under it)", directory)
|
||||
if dry_run:
|
||||
@@ -224,10 +232,8 @@ def remove_tree(directory, verbose=1, dry_run=0):
|
||||
for cmd in cmdtuples:
|
||||
try:
|
||||
cmd[0](cmd[1])
|
||||
# remove dir from cache if it's already there
|
||||
abspath = os.path.abspath(cmd[1])
|
||||
if abspath in _path_created:
|
||||
_path_created.pop(abspath)
|
||||
# Clear the cache
|
||||
SkipRepeatAbsolutePaths.clear()
|
||||
except OSError as exc:
|
||||
log.warning("error removing %s: %s", directory, exc)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user