using for loop to install conda package

This commit is contained in:
ton
2023-04-16 11:03:27 +07:00
parent 49da9f29c1
commit 0c2b34d6f8
12168 changed files with 2656238 additions and 1 deletions

View File

@@ -0,0 +1,119 @@
from typing import Any
from ._version import __version__
try:
__PYBASE64_SETUP__ # type: ignore
except NameError:
__PYBASE64_SETUP__ = False
# Only go on if not in setup.py
if not __PYBASE64_SETUP__:
from ._license import _license
try:
from ._pybase64 import ( # noqa: F401
_get_simd_path,
b64decode,
b64decode_as_bytearray,
b64encode,
b64encode_as_string,
encodebytes,
)
_has_extension = True
except ImportError:
from ._fallback import ( # noqa: F401
_get_simd_path,
b64decode,
b64decode_as_bytearray,
b64encode,
b64encode_as_string,
encodebytes,
)
_has_extension = False
def get_license_text() -> str:
"""Returns pybase64 license information as a :class:`str` object.
The result includes libbase64 license information as well.
"""
return _license
def get_version() -> str:
"""Returns pybase64 version as a :class:`str` object.
The result reports if the C extension is used or not.
e.g. `1.0.0 (C extension active - AVX2)`
"""
if _has_extension:
simd_flag = _get_simd_path()
if simd_flag == 0:
simd_name = "No SIMD"
elif simd_flag == 4:
simd_name = "SSSE3"
elif simd_flag == 8:
simd_name = "SSE41"
elif simd_flag == 16:
simd_name = "SSE42"
elif simd_flag == 32:
simd_name = "AVX"
elif simd_flag == 64:
simd_name = "AVX2"
else: # pragma: no branch
simd_name = "Unknown" # pragma: no cover
return __version__ + " (C extension active - " + simd_name + ")"
return __version__ + " (C extension inactive)"
def standard_b64encode(s: Any) -> bytes:
"""Encode bytes using the standard Base64 alphabet.
Argument ``s`` is a :term:`bytes-like object` to encode.
The result is returned as a :class:`bytes` object.
"""
return b64encode(s)
def standard_b64decode(s: Any) -> bytes:
"""Decode bytes encoded with the standard Base64 alphabet.
Argument ``s`` is a :term:`bytes-like object` or ASCII string to
decode.
The result is returned as a :class:`bytes` object.
A :exc:`binascii.Error` is raised if the input is incorrectly padded.
Characters that are not in the standard alphabet are discarded prior
to the padding check.
"""
return b64decode(s)
def urlsafe_b64encode(s: Any) -> bytes:
"""Encode bytes using the URL- and filesystem-safe Base64 alphabet.
Argument ``s`` is a :term:`bytes-like object` to encode.
The result is returned as a :class:`bytes` object.
The alphabet uses '-' instead of '+' and '_' instead of '/'.
"""
return b64encode(s, b"-_")
def urlsafe_b64decode(s: Any) -> bytes:
"""Decode bytes using the URL- and filesystem-safe Base64 alphabet.
Argument ``s`` is a :term:`bytes-like object` or ASCII string to
decode.
The result is returned as a :class:`bytes` object.
A :exc:`binascii.Error` is raised if the input is incorrectly padded.
Characters that are not in the URL-safe base-64 alphabet, and are not
a plus '+' or slash '/', are discarded prior to the padding check.
The alphabet uses '-' instead of '+' and '_' instead of '/'.
"""
return b64decode(s, b"-_")

View File

@@ -0,0 +1,285 @@
import argparse
import base64
import sys
from base64 import b64decode as b64decodeValidate
from base64 import encodebytes as b64encodebytes
from timeit import default_timer as timer
import pybase64
def bench_one(duration, data, enc, dec, encbytes, altchars=None, validate=False):
duration = duration / 2.0
if not validate and altchars is None:
number = 0
time = timer()
while True:
encodedcontent = encbytes(data)
number += 1
if timer() - time > duration:
break
iter = number
time = timer()
while iter > 0:
encodedcontent = encbytes(data)
iter -= 1
time = timer() - time
print(
"{:<32s} {:9.3f} MB/s ({:,d} bytes -> {:,d} bytes)".format(
encbytes.__module__ + "." + encbytes.__name__ + ":",
((number * len(data)) / (1024.0 * 1024.0)) / time,
len(data),
len(encodedcontent),
)
)
number = 0
time = timer()
while True:
encodedcontent = enc(data, altchars=altchars)
number += 1
if timer() - time > duration:
break
iter = number
time = timer()
while iter > 0:
encodedcontent = enc(data, altchars=altchars)
iter -= 1
time = timer() - time
print(
"{:<32s} {:9.3f} MB/s ({:,d} bytes -> {:,d} bytes)".format(
enc.__module__ + "." + enc.__name__ + ":",
((number * len(data)) / (1024.0 * 1024.0)) / time,
len(data),
len(encodedcontent),
)
)
number = 0
time = timer()
while True:
decodedcontent = dec(encodedcontent, altchars=altchars, validate=validate)
number += 1
if timer() - time > duration:
break
iter = number
time = timer()
while iter > 0:
decodedcontent = dec(encodedcontent, altchars=altchars, validate=validate)
iter -= 1
time = timer() - time
print(
"{0:<32s} {1:9.3f} MB/s ({3:,d} bytes -> {2:,d} bytes)".format(
dec.__module__ + "." + dec.__name__ + ":",
((number * len(data)) / (1024.0 * 1024.0)) / time,
len(data),
len(encodedcontent),
)
)
assert decodedcontent == data
def readall(file):
if file == sys.stdin:
if hasattr(file, "buffer"):
# Python 3 does not honor the binary flag,
# read from the underlying buffer
return file.buffer.read()
else:
return file.read()
# do not close the file
else:
try:
return file.read()
finally:
file.close()
def writeall(file, data):
if file == sys.stdout:
if hasattr(file, "buffer"):
# Python 3 does not honor the binary flag,
# write to the underlying buffer
file.buffer.write(data)
else:
file.write(data)
# do not close the file
else:
try:
file.write(data)
finally:
file.close()
def benchmark(args):
print(__package__ + " " + pybase64.get_version())
data = readall(args.input)
for altchars in [None, b"-_"]:
for validate in [False, True]:
print(
"bench: altchars={:s}, validate={:s}".format(
repr(altchars), repr(validate)
)
)
bench_one(
args.duration,
data,
pybase64.b64encode,
pybase64.b64decode,
pybase64.encodebytes,
altchars,
validate,
)
bench_one(
args.duration,
data,
base64.b64encode,
b64decodeValidate,
b64encodebytes,
altchars,
validate,
)
def encode(args):
data = readall(args.input)
data = pybase64.b64encode(data, args.altchars)
writeall(args.output, data)
def decode(args):
data = readall(args.input)
data = pybase64.b64decode(data, args.altchars, args.validate)
writeall(args.output, data)
class LicenseAction(argparse.Action):
def __init__(
self,
option_strings,
license=None,
dest=argparse.SUPPRESS,
default=argparse.SUPPRESS,
help="show license information and exit",
):
super().__init__(
option_strings=option_strings,
dest=dest,
default=default,
nargs=0,
help=help,
)
self.license = license
def __call__(self, parser, namespace, values, option_string=None):
print(self.license)
parser.exit()
def main(args=None):
# main parser
parser = argparse.ArgumentParser(
prog=__package__, description=__package__ + " command-line tool."
)
parser.add_argument(
"-V",
"--version",
action="version",
version=__package__ + " " + pybase64.get_version(),
)
parser.add_argument(
"--license", action=LicenseAction, license=pybase64.get_license_text()
)
# create sub-parsers
subparsers = parser.add_subparsers(help="tool help")
# benchmark parser
benchmark_parser = subparsers.add_parser("benchmark", help="-h for usage")
benchmark_parser.add_argument(
"-d",
"--duration",
metavar="D",
dest="duration",
type=float,
default=1.0,
help="expected duration for a single encode or decode test",
)
benchmark_parser.add_argument(
"input", type=argparse.FileType("rb"), help="input file used for the benchmark"
)
benchmark_parser.set_defaults(func=benchmark)
# encode parser
encode_parser = subparsers.add_parser("encode", help="-h for usage")
encode_parser.add_argument(
"input", type=argparse.FileType("rb"), help="input file to be encoded"
)
group = encode_parser.add_mutually_exclusive_group()
group.add_argument(
"-u",
"--url",
action="store_const",
const=b"-_",
dest="altchars",
help="use URL encoding",
)
group.add_argument(
"-a",
"--altchars",
dest="altchars",
help="use alternative characters for encoding",
)
encode_parser.add_argument(
"-o",
"--output",
dest="output",
type=argparse.FileType("wb"),
default=sys.stdout,
help="encoded output file (default to stdout)",
)
encode_parser.set_defaults(func=encode)
# decode parser
decode_parser = subparsers.add_parser("decode", help="-h for usage")
decode_parser.add_argument(
"input", type=argparse.FileType("rb"), help="input file to be decoded"
)
group = decode_parser.add_mutually_exclusive_group()
group.add_argument(
"-u",
"--url",
action="store_const",
const=b"-_",
dest="altchars",
help="use URL decoding",
)
group.add_argument(
"-a",
"--altchars",
dest="altchars",
help="use alternative characters for decoding",
)
decode_parser.add_argument(
"-o",
"--output",
dest="output",
type=argparse.FileType("wb"),
default=sys.stdout,
help="decoded output file (default to stdout)",
)
decode_parser.add_argument(
"--no-validation",
dest="validate",
action="store_false",
help="disable validation of the input data",
)
decode_parser.set_defaults(func=decode)
# ready, parse
if args is None:
args = sys.argv[1:]
if len(args) == 0:
args = ["-h"]
args = parser.parse_args(args=args)
args.func(args)
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,151 @@
from base64 import b64decode as builtin_decode
from base64 import b64encode as builtin_encode
from base64 import encodebytes as builtin_encodebytes
from binascii import Error as BinAsciiError
from typing import Any, Optional, Union
__all__ = [
"_get_simd_path",
"b64decode",
"b64encode",
"b64encode_as_string",
"encodebytes",
]
_bytes_types = (bytes, bytearray) # Types acceptable as binary data
def _get_simd_path() -> Optional[int]:
return None
def _get_bytes(s: Any) -> Union[bytes, bytearray]:
if isinstance(s, str):
try:
return s.encode("ascii")
except UnicodeEncodeError:
raise ValueError("string argument should contain only ASCII " "characters")
if isinstance(s, _bytes_types):
return s
try:
return memoryview(s).tobytes()
except TypeError:
raise TypeError(
"argument should be a bytes-like object or ASCII "
"string, not %r" % s.__class__.__name__
)
def b64decode(s: Any, altchars: Any = None, validate: bool = False) -> bytes:
"""Decode bytes encoded with the standard Base64 alphabet.
Argument ``s`` is a :term:`bytes-like object` or ASCII string to
decode.
Optional ``altchars`` must be a :term:`bytes-like object` or ASCII
string of length 2 which specifies the alternative alphabet used instead
of the '+' and '/' characters.
If ``validate`` is ``False`` (the default), characters that are neither in
the normal base-64 alphabet nor the alternative alphabet are discarded
prior to the padding check.
If ``validate`` is ``True``, these non-alphabet characters in the input
result in a :exc:`binascii.Error`.
The result is returned as a :class:`bytes` object.
A :exc:`binascii.Error` is raised if ``s`` is incorrectly padded.
"""
if validate:
if len(s) % 4 != 0:
raise BinAsciiError("Incorrect padding")
s = _get_bytes(s)
if altchars is not None:
altchars = _get_bytes(altchars)
assert len(altchars) == 2, repr(altchars)
map = bytes.maketrans(altchars, b"+/")
s = s.translate(map)
try:
result = builtin_decode(s, altchars, validate=False)
except TypeError as e:
raise BinAsciiError(str(e))
# check length of result vs length of input
padding = 0
if len(s) > 1 and s[-2] in (b"=", 61):
padding = padding + 1
if len(s) > 0 and s[-1] in (b"=", 61):
padding = padding + 1
if 3 * (len(s) / 4) - padding != len(result):
raise BinAsciiError("Non-base64 digit found")
return result
return builtin_decode(s, altchars, validate=False)
def b64decode_as_bytearray(
s: Any, altchars: Any = None, validate: bool = False
) -> bytearray:
"""Decode bytes encoded with the standard Base64 alphabet.
Argument ``s`` is a :term:`bytes-like object` or ASCII string to
decode.
Optional ``altchars`` must be a :term:`bytes-like object` or ASCII
string of length 2 which specifies the alternative alphabet used instead
of the '+' and '/' characters.
If ``validate`` is ``False`` (the default), characters that are neither in
the normal base-64 alphabet nor the alternative alphabet are discarded
prior to the padding check.
If ``validate`` is ``True``, these non-alphabet characters in the input
result in a :exc:`binascii.Error`.
The result is returned as a :class:`bytearray` object.
A :exc:`binascii.Error` is raised if ``s`` is incorrectly padded.
"""
return bytearray(b64decode(s, altchars=altchars, validate=validate))
def b64encode(s: Any, altchars: Any = None) -> bytes:
"""Encode bytes using the standard Base64 alphabet.
Argument ``s`` is a :term:`bytes-like object` to encode.
Optional ``altchars`` must be a byte string of length 2 which specifies
an alternative alphabet for the '+' and '/' characters. This allows an
application to e.g. generate url or filesystem safe Base64 strings.
The result is returned as a :class:`bytes` object.
"""
if altchars is not None:
altchars = _get_bytes(altchars)
assert len(altchars) == 2, repr(altchars)
return builtin_encode(s, altchars)
def b64encode_as_string(s: Any, altchars: Any = None) -> str:
"""Encode bytes using the standard Base64 alphabet.
Argument ``s`` is a :term:`bytes-like object` to encode.
Optional ``altchars`` must be a byte string of length 2 which specifies
an alternative alphabet for the '+' and '/' characters. This allows an
application to e.g. generate url or filesystem safe Base64 strings.
The result is returned as a :class:`str` object.
"""
return b64encode(s, altchars).decode("ascii")
def encodebytes(s: Any) -> bytes:
"""Encode bytes into a bytes object with newlines (b'\n') inserted after
every 76 bytes of output, and ensuring that there is a trailing newline,
as per :rfc:`2045` (MIME).
Argument ``s`` is a :term:`bytes-like object` to encode.
The result is returned as a :class:`bytes` object.
"""
return builtin_encodebytes(s)

View File

@@ -0,0 +1,61 @@
_license = """pybase64
===============================================================================
BSD 2-Clause License
Copyright (c) 2017-2019, Matthieu Darbois
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
===============================================================================
libbase64
===============================================================================
Copyright (c) 2005-2007, Nick Galbreath
Copyright (c) 2013-2019, Alfred Klomp
Copyright (c) 2015-2017, Wojciech Mula
Copyright (c) 2016-2017, Matthieu Darbois
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
- Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
==========================================================================""" \
+ "====="

View File

@@ -0,0 +1 @@
__version__ = "1.2.3"

View File

@@ -0,0 +1,210 @@
import os
import shutil
import sys
import tempfile
from contextlib import contextmanager
from distutils import log
from distutils.errors import CompileError, LinkError
__all__ = ["CCompilerCapabilities"]
@contextmanager
def chdir(path):
cwd = os.getcwd()
os.chdir(path)
try:
yield
finally:
os.chdir(cwd)
@contextmanager
def mkdtemp(suffix):
path = tempfile.mkdtemp(suffix)
try:
with chdir(path):
yield path
finally:
shutil.rmtree(path)
@contextmanager
def output(is_quiet):
if is_quiet: # pragma: no branch
devnull = open(os.devnull, "w")
oldstderr = os.dup(sys.stderr.fileno())
oldstdout = os.dup(sys.stdout.fileno())
os.dup2(devnull.fileno(), sys.stderr.fileno())
os.dup2(devnull.fileno(), sys.stdout.fileno())
try:
yield
finally:
if is_quiet: # pragma: no branch
os.dup2(oldstderr, sys.stderr.fileno())
os.dup2(oldstdout, sys.stdout.fileno())
devnull.close()
class CCompilerCapabilities:
SIMD_SSSE3 = 0
SIMD_SSE41 = 1
SIMD_SSE42 = 2
SIMD_AVX = 3
SIMD_AVX2 = 4
SIMD_NEON32 = 5
SIMD_NEON64 = 6
def __init__(self, compiler):
self.__capabilities = {}
self.__cflags = []
if compiler.compiler_type == "msvc":
self.__cflags = ["/WX", "/Od"] # pragma: no cover
else:
self.__cflags = ["-O0"]
self.__get_capabilities(compiler)
def __has_simd_support(self, compiler, flags, define, include, content):
quiet = True
with mkdtemp("pybase64simdtest") as dname:
fname = os.path.join(dname, "simd.c")
with open(fname, "w") as f:
f.write("""#include <%s>\n""" % include)
f.write(
"""\
int main (int argc, char **argv) {
%s
}
"""
% content
)
with output(quiet):
for flag in flags:
lflags = []
if not len(flag) == 0:
lflags = [flag]
try:
objects = compiler.compile(
["simd.c"],
output_dir=dname,
extra_postargs=lflags + self.__cflags,
)
except CompileError:
continue
try:
compiler.link_shared_lib(objects, "a.out", output_dir=dname)
except (LinkError, TypeError): # pragma: no cover
continue # pragma: no cover
return {"support": True, "flags": lflags + define}
return {"support": False, "flags": []}
def __get_capabilities(self, compiler):
log.info("getting compiler simd support")
self.__capabilities[CCompilerCapabilities.SIMD_SSSE3] = self.__has_simd_support(
compiler,
["", "-mssse3"],
["-D__SSSE3__"],
"tmmintrin.h",
"__m128i t = _mm_loadu_si128((const __m128i*)argv[0]);"
"t = _mm_shuffle_epi8(t, t);"
"return _mm_cvtsi128_si32(t);",
)
log.info(
"SSSE3: %s"
% str(self.__capabilities[CCompilerCapabilities.SIMD_SSSE3]["support"])
)
self.__capabilities[CCompilerCapabilities.SIMD_SSE41] = self.__has_simd_support(
compiler,
["", "-msse4.1"],
["-D__SSE4_1__"],
"smmintrin.h",
"__m128i t = _mm_loadu_si128((const __m128i*)argv[0]);"
"t = _mm_mpsadbw_epu8(t, t, 1);"
"return _mm_cvtsi128_si32(t);",
)
log.info(
"SSE41: %s"
% str(self.__capabilities[CCompilerCapabilities.SIMD_SSE41]["support"])
)
self.__capabilities[CCompilerCapabilities.SIMD_SSE42] = self.__has_simd_support(
compiler,
["", "-msse4.2"],
["-D__SSE4_2__"],
"nmmintrin.h",
"__m128i t = _mm_loadu_si128((const __m128i*)argv[0]);"
"return _mm_cmpistra(t, t, 0);",
)
log.info(
"SSE42: %s"
% str(self.__capabilities[CCompilerCapabilities.SIMD_SSE42]["support"])
)
self.__capabilities[CCompilerCapabilities.SIMD_AVX] = self.__has_simd_support(
compiler,
["", "-mavx", "/arch:AVX"],
["-D__AVX__"],
"immintrin.h",
"__m256i y = _mm256_loadu_si256((const __m256i*)argv[0]);"
"return _mm256_testz_si256(y, y);",
)
log.info(
"AVX: %s"
% str(self.__capabilities[CCompilerCapabilities.SIMD_AVX]["support"])
)
self.__capabilities[CCompilerCapabilities.SIMD_AVX2] = self.__has_simd_support(
compiler,
["", "-mavx2", "/arch:AVX2"],
["-D__AVX2__"],
"immintrin.h",
"__m256i y = _mm256_loadu_si256((const __m256i*)argv[0]);"
"y = _mm256_i32gather_epi32((int const*)argv[1], y, 2);"
"return _mm_cvtsi128_si32(_mm256_castsi256_si128(y));",
)
log.info(
"AVX2: %s"
% str(self.__capabilities[CCompilerCapabilities.SIMD_AVX2]["support"])
)
self.__capabilities[
CCompilerCapabilities.SIMD_NEON64
] = self.__has_simd_support(
compiler,
[""],
[],
"arm_neon.h",
"uint8x16_t t = vdupq_n_u8(1);"
"uint8x16x4_t t4 = {"
" .val[0]=t, .val[1]=t, .val[2]=t, .val[3]=t};"
"uint8x16_t o = vqtbx4q_u8(t, t4, t);"
"return vgetq_lane_s32(vreinterpretq_s32_u8(o), 0);",
)
log.info(
"NEON64: %s"
% str(self.__capabilities[CCompilerCapabilities.SIMD_NEON64]["support"])
)
self.__capabilities[
CCompilerCapabilities.SIMD_NEON32
] = self.__has_simd_support(
compiler,
[""],
[],
"arm_neon.h",
"uint8x16_t t = vdupq_n_u8(1);"
"return vgetq_lane_s32(vreinterpretq_s32_u8(t), 0);",
)
self.__capabilities[CCompilerCapabilities.SIMD_NEON32][
"support"
] &= not self.__capabilities[CCompilerCapabilities.SIMD_NEON64]["support"]
log.info(
"NEON32: %s"
% str(self.__capabilities[CCompilerCapabilities.SIMD_NEON32]["support"])
)
def has(self, what):
if what not in self.__capabilities:
return False
return self.__capabilities[what]["support"]
def flags(self, what):
if not self.has(what): # pragma: no branch
return self.__capabilities[666]["flags"] # pragma: no cover
return self.__capabilities[what]["flags"]

View File

@@ -0,0 +1,148 @@
import os
import re
import sys
import pytest
import pybase64
from pybase64.__main__ import main
@pytest.fixture
def emptyfile(tmpdir):
_file = os.path.join(tmpdir.strpath, "empty")
with open(_file, "wb"):
pass
yield _file
os.remove(_file)
@pytest.fixture
def hellofile(tmpdir):
_file = os.path.join(tmpdir.strpath, "helloworld")
with open(_file, "wb") as f:
f.write(b"hello world !/?\n")
yield _file
os.remove(_file)
def idfn_test_help(args):
if len(args) == 0:
return "(empty)"
return " ".join(args)
@pytest.mark.parametrize(
"args",
[
[],
["-h"],
["benchmark", "-h"],
["encode", "-h"],
["decode", "-h"],
],
ids=idfn_test_help,
)
def test_help(capsys, args):
if len(args) == 2:
usage = f"usage: pybase64 {args[0]} [-h]"
else:
usage = "usage: pybase64 [-h]"
with pytest.raises(SystemExit) as exit_info:
main(args)
captured = capsys.readouterr()
assert captured.err == ""
assert captured.out.startswith(usage)
assert exit_info.value.code == 0
def test_version(capsys):
with pytest.raises(SystemExit) as exit_info:
main(["-V"])
captured = capsys.readouterr()
assert captured.err == ""
assert captured.out.startswith("pybase64 " + pybase64.__version__)
assert exit_info.value.code == 0
def test_license(capsys):
restr = "\n".join(
x + "\n[=]+\n.*Copyright.*\n[=]+\n" for x in ["pybase64", "libbase64"]
)
regex = re.compile("^" + restr + "$", re.DOTALL)
with pytest.raises(SystemExit) as exit_info:
main(["--license"])
captured = capsys.readouterr()
assert captured.err == ""
assert regex.match(captured.out)
assert exit_info.value.code == 0
def test_benchmark(capsys, emptyfile):
main(["benchmark", "-d", "0.005", emptyfile])
captured = capsys.readouterr()
assert captured.err == ""
assert captured.out != ""
@pytest.mark.parametrize(
"args,expect",
[
([], b"aGVsbG8gd29ybGQgIS8/Cg=="),
(["-u"], b"aGVsbG8gd29ybGQgIS8_Cg=="),
(["-a", ":,"], b"aGVsbG8gd29ybGQgIS8,Cg=="),
],
ids=["0", "1", "2"],
)
def test_encode(capsysbinary, hellofile, args, expect):
main(["encode"] + args + [hellofile])
captured = capsysbinary.readouterr()
assert captured.err == b""
assert captured.out == expect
def test_encode_ouputfile(capsys, emptyfile, hellofile):
main(["encode", "-o", hellofile, emptyfile])
captured = capsys.readouterr()
assert captured.err == ""
assert captured.out == ""
with open(hellofile, "rb") as f:
data = f.read()
assert data == b""
@pytest.mark.parametrize(
"args,b64string",
[
[[], b"aGVsbG8gd29ybGQgIS8/Cg=="],
[["-u"], b"aGVsbG8gd29ybGQgIS8_Cg=="],
[["-a", ":,"], b"aGVsbG8gd29ybGQgIS8,Cg=="],
[["--no-validation"], b"aGVsbG8gd29yb GQgIS8/Cg==\n"],
],
ids=["0", "1", "2", "3"],
)
def test_decode(capsysbinary, tmpdir, args, b64string):
iname = os.path.join(tmpdir.strpath, "in")
with open(iname, "wb") as f:
f.write(b64string)
main(["decode"] + args + [iname])
captured = capsysbinary.readouterr()
assert captured.err == b""
assert captured.out == b"hello world !/?\n"
def test_subprocess():
import subprocess
process = subprocess.Popen(
[sys.executable, "-m", "pybase64", "encode", "-"],
bufsize=4096,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = process.communicate()
process.wait()
assert process.returncode == 0
assert out == b""
assert err == b""

View File

@@ -0,0 +1,455 @@
import base64
from base64 import encodebytes as b64encodebytes
from binascii import Error as BinAsciiError
import pytest
import pybase64
try:
from pybase64._pybase64 import (
_get_simd_flags_compile,
_get_simd_flags_runtime,
_get_simd_path,
_set_simd_path,
)
_has_extension = True
except ImportError:
_has_extension = False
STD = 0
URL = 1
ALT1 = 2
ALT2 = 3
ALT3 = 4
name_lut = ["standard", "urlsafe", "alternative", "alternative2", "alternative3"]
altchars_lut = [b"+/", b"-_", b"@&", b"+,", b";/"]
enc_helper_lut = [
pybase64.standard_b64encode,
pybase64.urlsafe_b64encode,
None,
None,
None,
]
ref_enc_helper_lut = [
pybase64.standard_b64encode,
pybase64.urlsafe_b64encode,
None,
None,
None,
]
dec_helper_lut = [
pybase64.standard_b64decode,
pybase64.urlsafe_b64decode,
None,
None,
None,
]
ref_dec_helper_lut = [
base64.standard_b64decode,
base64.urlsafe_b64decode,
None,
None,
None,
]
std = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/A"
std = std * (32 * 16)
std_len_minus_12 = len(std) - 12
test_vectors_b64_list = [
# rfc4648 test vectors
b"",
b"Zg==",
b"Zm8=",
b"Zm9v",
b"Zm9vYg==",
b"Zm9vYmE=",
b"Zm9vYmFy",
# custom test vectors
std[std_len_minus_12:],
std,
std[:72],
std[:76],
std[:80],
std[:148],
std[:152],
std[:156],
]
test_vectors_b64 = []
for altchars in altchars_lut:
trans = bytes.maketrans(b"+/", altchars)
test_vectors_b64.append(
[vector.translate(trans) for vector in test_vectors_b64_list]
)
test_vectors_bin = []
for altchars in altchars_lut:
test_vectors_bin.append(
[base64.b64decode(vector, altchars) for vector in test_vectors_b64_list]
)
compile_flags = [0]
runtime_flags = 0
if _has_extension:
runtime_flags = _get_simd_flags_runtime()
flags = _get_simd_flags_compile()
for i in range(31):
if flags & (1 << i):
compile_flags += [(1 << i)]
def get_simd_name(simd_id):
simd_name = None
if _has_extension:
simd_flag = compile_flags[simd_id]
if simd_flag == 0:
simd_name = "c"
elif simd_flag == 4:
simd_name = "ssse3"
elif simd_flag == 8:
simd_name = "sse41"
elif simd_flag == 16:
simd_name = "sse42"
elif simd_flag == 32:
simd_name = "avx"
elif simd_flag == 64:
simd_name = "avx2"
else: # pragma: no branch
simd_name = "unk" # pragma: no cover
else:
simd_name = "py"
return simd_name
def simd_setup(simd_id):
if _has_extension:
flag = compile_flags[simd_id]
if flag != 0 and not flag & runtime_flags: # pragma: no branch
pytest.skip("SIMD extension not available") # pragma: no cover
_set_simd_path(flag)
assert _get_simd_path() == flag
else:
assert 0 == simd_id
param_vector = pytest.mark.parametrize("vector_id", range(len(test_vectors_bin[0])))
param_validate = pytest.mark.parametrize(
"validate", [False, True], ids=["novalidate", "validate"]
)
param_altchars = pytest.mark.parametrize(
"altchars_id", [STD, URL, ALT1, ALT2, ALT3], ids=lambda x: name_lut[x]
)
param_altchars_helper = pytest.mark.parametrize(
"altchars_id", [STD, URL], ids=lambda x: name_lut[x]
)
param_simd = pytest.mark.parametrize(
"simd", range(len(compile_flags)), ids=lambda x: get_simd_name(x), indirect=True
)
param_encode_functions = pytest.mark.parametrize(
"efn, ecast",
[
(pybase64.b64encode, lambda x: x),
(pybase64.b64encode_as_string, lambda x: x.encode("ascii")),
],
)
param_decode_functions = pytest.mark.parametrize(
"dfn, dcast",
[
(pybase64.b64decode, lambda x: x),
(pybase64.b64decode_as_bytearray, lambda x: bytes(x)),
],
)
@pytest.fixture
def simd(request):
simd_setup(request.param)
return request.param
@param_simd
def test_version(simd):
assert pybase64.get_version().startswith(pybase64.__version__)
@param_simd
@param_vector
@param_altchars_helper
def test_enc_helper(altchars_id, vector_id, simd):
vector = test_vectors_bin[altchars_id][vector_id]
test = enc_helper_lut[altchars_id](vector)
base = ref_enc_helper_lut[altchars_id](vector)
assert test == base
@param_simd
@param_vector
@param_altchars_helper
def test_dec_helper(altchars_id, vector_id, simd):
vector = test_vectors_b64[altchars_id][vector_id]
test = dec_helper_lut[altchars_id](vector)
base = ref_dec_helper_lut[altchars_id](vector)
assert test == base
@param_simd
@param_vector
@param_altchars_helper
def test_dec_helper_unicode(altchars_id, vector_id, simd):
vector = test_vectors_b64[altchars_id][vector_id]
test = dec_helper_lut[altchars_id](str(vector, "utf-8"))
base = ref_dec_helper_lut[altchars_id](str(vector, "utf-8"))
assert test == base
@param_simd
@param_vector
@param_altchars_helper
def test_rnd_helper(altchars_id, vector_id, simd):
vector = test_vectors_b64[altchars_id][vector_id]
test = dec_helper_lut[altchars_id](vector)
test = enc_helper_lut[altchars_id](test)
assert test == vector
@param_simd
@param_vector
@param_altchars_helper
def test_rnd_helper_unicode(altchars_id, vector_id, simd):
vector = test_vectors_b64[altchars_id][vector_id]
test = dec_helper_lut[altchars_id](str(vector, "utf-8"))
test = enc_helper_lut[altchars_id](test)
assert test == vector
@param_simd
@param_vector
def test_encbytes(vector_id, simd):
vector = test_vectors_bin[STD][vector_id]
test = pybase64.encodebytes(vector)
base = b64encodebytes(vector)
assert test == base
@param_simd
@param_vector
@param_altchars
@param_encode_functions
def test_enc(efn, ecast, altchars_id, vector_id, simd):
vector = test_vectors_bin[altchars_id][vector_id]
altchars = altchars_lut[altchars_id]
test = ecast(efn(vector, altchars))
base = base64.b64encode(vector, altchars)
assert test == base
@param_simd
@param_vector
@param_altchars
@param_validate
@param_decode_functions
def test_dec(dfn, dcast, altchars_id, vector_id, validate, simd):
vector = test_vectors_b64[altchars_id][vector_id]
altchars = altchars_lut[altchars_id]
if validate:
base = base64.b64decode(vector, altchars, validate)
else:
base = base64.b64decode(vector, altchars)
test = dcast(dfn(vector, altchars, validate))
assert test == base
@param_simd
@param_vector
@param_altchars
@param_validate
@param_decode_functions
def test_dec_unicode(dfn, dcast, altchars_id, vector_id, validate, simd):
vector = test_vectors_b64[altchars_id][vector_id]
vector = str(vector, "utf-8")
altchars = altchars_lut[altchars_id]
if altchars_id == STD:
altchars = None
else:
altchars = str(altchars, "utf-8")
if validate:
base = base64.b64decode(vector, altchars, validate)
else:
base = base64.b64decode(vector, altchars)
test = dcast(dfn(vector, altchars, validate))
assert test == base
@param_simd
@param_vector
@param_altchars
@param_validate
@param_encode_functions
@param_decode_functions
def test_rnd(dfn, dcast, efn, ecast, altchars_id, vector_id, validate, simd):
vector = test_vectors_b64[altchars_id][vector_id]
altchars = altchars_lut[altchars_id]
test = dcast(dfn(vector, altchars, validate))
test = ecast(efn(test, altchars))
assert test == vector
@param_simd
@param_vector
@param_altchars
@param_validate
@param_encode_functions
@param_decode_functions
def test_rnd_unicode(dfn, dcast, efn, ecast, altchars_id, vector_id, validate, simd):
vector = test_vectors_b64[altchars_id][vector_id]
altchars = altchars_lut[altchars_id]
test = dcast(dfn(str(vector, "utf-8"), altchars, validate))
test = ecast(efn(test, altchars))
assert test == vector
@param_simd
@param_vector
@param_altchars
@param_validate
@param_decode_functions
def test_invalid_padding_dec(dfn, dcast, altchars_id, vector_id, validate, simd):
vector = test_vectors_b64[altchars_id][vector_id][1:]
if len(vector) > 0:
altchars = altchars_lut[altchars_id]
with pytest.raises(BinAsciiError):
dfn(vector, altchars, validate)
params_invalid_altchars = [
[b"", AssertionError],
[b"-", AssertionError],
[b"-__", AssertionError],
[3.0, TypeError],
["-€", ValueError],
]
params_invalid_altchars = pytest.mark.parametrize(
"altchars,exception",
params_invalid_altchars,
ids=[str(i) for i in range(len(params_invalid_altchars))],
)
@param_simd
@params_invalid_altchars
@param_encode_functions
def test_invalid_altchars_enc(efn, ecast, altchars, exception, simd):
with pytest.raises(exception):
efn(b"ABCD", altchars)
@param_simd
@params_invalid_altchars
@param_decode_functions
def test_invalid_altchars_dec(dfn, dcast, altchars, exception, simd):
with pytest.raises(exception):
dfn(b"ABCD", altchars)
@param_simd
@params_invalid_altchars
@param_decode_functions
def test_invalid_altchars_dec_validate(dfn, dcast, altchars, exception, simd):
with pytest.raises(exception):
dfn(b"ABCD", altchars, True)
params_invalid_data_novalidate = [
[b"A@@@@FG", None, BinAsciiError],
["ABC€", None, ValueError],
[3.0, None, TypeError],
]
params_invalid_data_validate = [
[b"\x00\x00\x00\x00", None, BinAsciiError],
[b"A@@@@FGHIJKLMNOPQRSTUVWXYZabcdef", b"-_", BinAsciiError],
[b"A@@@=FGHIJKLMNOPQRSTUVWXYZabcdef", b"-_", BinAsciiError],
[b"A@@=@FGHIJKLMNOPQRSTUVWXYZabcdef", b"-_", BinAsciiError],
[b"A@@@@FGHIJKLMNOPQRSTUVWXYZabcde@=", b"-_", BinAsciiError],
[b"A@@@@FGHIJKLMNOPQRSTUVWXYZabcd@==", b"-_", BinAsciiError],
[b"A@@@@FGH" * 10000, b"-_", BinAsciiError],
# [std, b'-_', BinAsciiError], TODO does no fail with base64 module
]
params_invalid_data_all = pytest.mark.parametrize(
"vector,altchars,exception",
params_invalid_data_novalidate + params_invalid_data_validate,
ids=[
str(i)
for i in range(
len(params_invalid_data_novalidate) + len(params_invalid_data_validate)
)
],
)
params_invalid_data_novalidate = pytest.mark.parametrize(
"vector,altchars,exception",
params_invalid_data_novalidate,
ids=[str(i) for i in range(len(params_invalid_data_novalidate))],
)
params_invalid_data_validate = pytest.mark.parametrize(
"vector,altchars,exception",
params_invalid_data_validate,
ids=[str(i) for i in range(len(params_invalid_data_validate))],
)
@param_simd
@params_invalid_data_novalidate
@param_decode_functions
def test_invalid_data_dec(dfn, dcast, vector, altchars, exception, simd):
with pytest.raises(exception):
dfn(vector, altchars)
@param_simd
@params_invalid_data_validate
@param_decode_functions
def test_invalid_data_dec_skip(dfn, dcast, vector, altchars, exception, simd):
test = dcast(dfn(vector, altchars))
base = base64.b64decode(vector, altchars)
assert test == base
@param_simd
@params_invalid_data_all
@param_decode_functions
def test_invalid_data_dec_validate(dfn, dcast, vector, altchars, exception, simd):
with pytest.raises(exception):
dfn(vector, altchars, True)
@param_encode_functions
def test_invalid_data_enc_0(efn, ecast):
with pytest.raises(TypeError):
efn("this is a test")
@param_encode_functions
def test_invalid_args_enc_0(efn, ecast):
with pytest.raises(TypeError):
efn()
@param_decode_functions
def test_invalid_args_dec_0(dfn, dcast):
with pytest.raises(TypeError):
dfn()