Created starter files for the project.

venv/Lib/site-packages/pip/_internal/network/__init__.py

@@ -0,0 +1,2 @@
+"""Contains purely network-related utilities.
+"""

venv/Lib/site-packages/pip/_internal/network/auth.py

@@ -0,0 +1,310 @@
+"""Network Authentication Helpers
+
+Contains interface (MultiDomainBasicAuth) and associated glue code for
+providing credentials in the context of network requests.
+"""
+
+import logging
+
+from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
+from pip._vendor.requests.utils import get_netrc_auth
+from pip._vendor.six.moves.urllib import parse as urllib_parse
+
+from pip._internal.utils.misc import (
+    ask,
+    ask_input,
+    ask_password,
+    remove_auth_from_url,
+    split_auth_netloc_from_url,
+)
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Dict, Optional, Tuple, List, Any
+
+    from pip._internal.vcs.versioncontrol import AuthInfo
+
+    from pip._vendor.requests.models import Response, Request
+
+    Credentials = Tuple[str, str, str]
+
+logger = logging.getLogger(__name__)
+
+try:
+    import keyring  # noqa
+except ImportError:
+    keyring = None
+except Exception as exc:
+    logger.warning(
+        "Keyring is skipped due to an exception: %s", str(exc),
+    )
+    keyring = None
+
+
+def get_keyring_auth(url, username):
+    # type: (str, str) -> Optional[AuthInfo]
+    """Return the tuple auth for a given url from keyring."""
+    global keyring
+    if not url or not keyring:
+        return None
+
+    try:
+        try:
+            get_credential = keyring.get_credential
+        except AttributeError:
+            pass
+        else:
+            logger.debug("Getting credentials from keyring for %s", url)
+            cred = get_credential(url, username)
+            if cred is not None:
+                return cred.username, cred.password
+            return None
+
+        if username:
+            logger.debug("Getting password from keyring for %s", url)
+            password = keyring.get_password(url, username)
+            if password:
+                return username, password
+
+    except Exception as exc:
+        logger.warning(
+            "Keyring is skipped due to an exception: %s", str(exc),
+        )
+        keyring = None
+    return None
+
+
+class MultiDomainBasicAuth(AuthBase):
+
+    def __init__(self, prompting=True, index_urls=None):
+        # type: (bool, Optional[List[str]]) -> None
+        self.prompting = prompting
+        self.index_urls = index_urls
+        self.passwords = {}  # type: Dict[str, AuthInfo]
+        # When the user is prompted to enter credentials and keyring is
+        # available, we will offer to save them. If the user accepts,
+        # this value is set to the credentials they entered. After the
+        # request authenticates, the caller should call
+        # ``save_credentials`` to save these.
+        self._credentials_to_save = None  # type: Optional[Credentials]
+
+    def _get_index_url(self, url):
+        # type: (str) -> Optional[str]
+        """Return the original index URL matching the requested URL.
+
+        Cached or dynamically generated credentials may work against
+        the original index URL rather than just the netloc.
+
+        The provided url should have had its username and password
+        removed already. If the original index url had credentials then
+        they will be included in the return value.
+
+        Returns None if no matching index was found, or if --no-index
+        was specified by the user.
+        """
+        if not url or not self.index_urls:
+            return None
+
+        for u in self.index_urls:
+            prefix = remove_auth_from_url(u).rstrip("/") + "/"
+            if url.startswith(prefix):
+                return u
+        return None
+
+    def _get_new_credentials(self, original_url, allow_netrc=True,
+                             allow_keyring=True):
+        # type: (str, bool, bool) -> AuthInfo
+        """Find and return credentials for the specified URL."""
+        # Split the credentials and netloc from the url.
+        url, netloc, url_user_password = split_auth_netloc_from_url(
+            original_url,
+        )
+
+        # Start with the credentials embedded in the url
+        username, password = url_user_password
+        if username is not None and password is not None:
+            logger.debug("Found credentials in url for %s", netloc)
+            return url_user_password
+
+        # Find a matching index url for this request
+        index_url = self._get_index_url(url)
+        if index_url:
+            # Split the credentials from the url.
+            index_info = split_auth_netloc_from_url(index_url)
+            if index_info:
+                index_url, _, index_url_user_password = index_info
+                logger.debug("Found index url %s", index_url)
+
+        # If an index URL was found, try its embedded credentials
+        if index_url and index_url_user_password[0] is not None:
+            username, password = index_url_user_password
+            if username is not None and password is not None:
+                logger.debug("Found credentials in index url for %s", netloc)
+                return index_url_user_password
+
+        # Get creds from netrc if we still don't have them
+        if allow_netrc:
+            netrc_auth = get_netrc_auth(original_url)
+            if netrc_auth:
+                logger.debug("Found credentials in netrc for %s", netloc)
+                return netrc_auth
+
+        # If we don't have a password and keyring is available, use it.
+        if allow_keyring:
+            # The index url is more specific than the netloc, so try it first
+            kr_auth = (
+                get_keyring_auth(index_url, username) or
+                get_keyring_auth(netloc, username)
+            )
+            if kr_auth:
+                logger.debug("Found credentials in keyring for %s", netloc)
+                return kr_auth
+
+        return username, password
+
+    def _get_url_and_credentials(self, original_url):
+        # type: (str) -> Tuple[str, Optional[str], Optional[str]]
+        """Return the credentials to use for the provided URL.
+
+        If allowed, netrc and keyring may be used to obtain the
+        correct credentials.
+
+        Returns (url_without_credentials, username, password). Note
+        that even if the original URL contains credentials, this
+        function may return a different username and password.
+        """
+        url, netloc, _ = split_auth_netloc_from_url(original_url)
+
+        # Use any stored credentials that we have for this netloc
+        username, password = self.passwords.get(netloc, (None, None))
+
+        if username is None and password is None:
+            # No stored credentials. Acquire new credentials without prompting
+            # the user. (e.g. from netrc, keyring, or the URL itself)
+            username, password = self._get_new_credentials(original_url)
+
+        if username is not None or password is not None:
+            # Convert the username and password if they're None, so that
+            # this netloc will show up as "cached" in the conditional above.
+            # Further, HTTPBasicAuth doesn't accept None, so it makes sense to
+            # cache the value that is going to be used.
+            username = username or ""
+            password = password or ""
+
+            # Store any acquired credentials.
+            self.passwords[netloc] = (username, password)
+
+        assert (
+            # Credentials were found
+            (username is not None and password is not None) or
+            # Credentials were not found
+            (username is None and password is None)
+        ), "Could not load credentials from url: {}".format(original_url)
+
+        return url, username, password
+
+    def __call__(self, req):
+        # type: (Request) -> Request
+        # Get credentials for this request
+        url, username, password = self._get_url_and_credentials(req.url)
+
+        # Set the url of the request to the url without any credentials
+        req.url = url
+
+        if username is not None and password is not None:
+            # Send the basic auth with this request
+            req = HTTPBasicAuth(username, password)(req)
+
+        # Attach a hook to handle 401 responses
+        req.register_hook("response", self.handle_401)
+
+        return req
+
+    # Factored out to allow for easy patching in tests
+    def _prompt_for_password(self, netloc):
+        # type: (str) -> Tuple[Optional[str], Optional[str], bool]
+        username = ask_input("User for {}: ".format(netloc))
+        if not username:
+            return None, None, False
+        auth = get_keyring_auth(netloc, username)
+        if auth and auth[0] is not None and auth[1] is not None:
+            return auth[0], auth[1], False
+        password = ask_password("Password: ")
+        return username, password, True
+
+    # Factored out to allow for easy patching in tests
+    def _should_save_password_to_keyring(self):
+        # type: () -> bool
+        if not keyring:
+            return False
+        return ask("Save credentials to keyring [y/N]: ", ["y", "n"]) == "y"
+
+    def handle_401(self, resp, **kwargs):
+        # type: (Response, **Any) -> Response
+        # We only care about 401 responses, anything else we want to just
+        #   pass through the actual response
+        if resp.status_code != 401:
+            return resp
+
+        # We are not able to prompt the user so simply return the response
+        if not self.prompting:
+            return resp
+
+        parsed = urllib_parse.urlparse(resp.url)
+
+        # Prompt the user for a new username and password
+        username, password, save = self._prompt_for_password(parsed.netloc)
+
+        # Store the new username and password to use for future requests
+        self._credentials_to_save = None
+        if username is not None and password is not None:
+            self.passwords[parsed.netloc] = (username, password)
+
+            # Prompt to save the password to keyring
+            if save and self._should_save_password_to_keyring():
+                self._credentials_to_save = (parsed.netloc, username, password)
+
+        # Consume content and release the original connection to allow our new
+        #   request to reuse the same one.
+        resp.content
+        resp.raw.release_conn()
+
+        # Add our new username and password to the request
+        req = HTTPBasicAuth(username or "", password or "")(resp.request)
+        req.register_hook("response", self.warn_on_401)
+
+        # On successful request, save the credentials that were used to
+        # keyring. (Note that if the user responded "no" above, this member
+        # is not set and nothing will be saved.)
+        if self._credentials_to_save:
+            req.register_hook("response", self.save_credentials)
+
+        # Send our new request
+        new_resp = resp.connection.send(req, **kwargs)
+        new_resp.history.append(resp)
+
+        return new_resp
+
+    def warn_on_401(self, resp, **kwargs):
+        # type: (Response, **Any) -> None
+        """Response callback to warn about incorrect credentials."""
+        if resp.status_code == 401:
+            logger.warning(
+                '401 Error, Credentials not correct for %s', resp.request.url,
+            )
+
+    def save_credentials(self, resp, **kwargs):
+        # type: (Response, **Any) -> None
+        """Response callback to save credentials on success."""
+        assert keyring is not None, "should never reach here without keyring"
+        if not keyring:
+            return
+
+        creds = self._credentials_to_save
+        self._credentials_to_save = None
+        if creds and resp.status_code < 400:
+            try:
+                logger.info('Saving credentials to keyring')
+                keyring.set_password(*creds)
+            except Exception:
+                logger.exception('Failed to save credentials')

venv/Lib/site-packages/pip/_internal/network/cache.py

@@ -0,0 +1,79 @@
+"""HTTP cache implementation.
+"""
+
+import os
+from contextlib import contextmanager
+
+from pip._vendor.cachecontrol.cache import BaseCache
+from pip._vendor.cachecontrol.caches import FileCache
+from pip._vendor.requests.models import Response
+
+from pip._internal.utils.filesystem import adjacent_tmp_file, replace
+from pip._internal.utils.misc import ensure_dir
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Optional, Iterator
+
+
+def is_from_cache(response):
+    # type: (Response) -> bool
+    return getattr(response, "from_cache", False)
+
+
+@contextmanager
+def suppressed_cache_errors():
+    # type: () -> Iterator[None]
+    """If we can't access the cache then we can just skip caching and process
+    requests as if caching wasn't enabled.
+    """
+    try:
+        yield
+    except (OSError, IOError):
+        pass
+
+
+class SafeFileCache(BaseCache):
+    """
+    A file based cache which is safe to use even when the target directory may
+    not be accessible or writable.
+    """
+
+    def __init__(self, directory):
+        # type: (str) -> None
+        assert directory is not None, "Cache directory must not be None."
+        super(SafeFileCache, self).__init__()
+        self.directory = directory
+
+    def _get_cache_path(self, name):
+        # type: (str) -> str
+        # From cachecontrol.caches.file_cache.FileCache._fn, brought into our
+        # class for backwards-compatibility and to avoid using a non-public
+        # method.
+        hashed = FileCache.encode(name)
+        parts = list(hashed[:5]) + [hashed]
+        return os.path.join(self.directory, *parts)
+
+    def get(self, key):
+        # type: (str) -> Optional[bytes]
+        path = self._get_cache_path(key)
+        with suppressed_cache_errors():
+            with open(path, 'rb') as f:
+                return f.read()
+
+    def set(self, key, value):
+        # type: (str, bytes) -> None
+        path = self._get_cache_path(key)
+        with suppressed_cache_errors():
+            ensure_dir(os.path.dirname(path))
+
+            with adjacent_tmp_file(path) as f:
+                f.write(value)
+
+            replace(f.name, path)
+
+    def delete(self, key):
+        # type: (str) -> None
+        path = self._get_cache_path(key)
+        with suppressed_cache_errors():
+            os.remove(path)

venv/Lib/site-packages/pip/_internal/network/download.py

@@ -0,0 +1,182 @@
+"""Download files with progress indicators.
+"""
+import cgi
+import logging
+import mimetypes
+import os
+
+from pip._vendor.requests.models import CONTENT_CHUNK_SIZE
+
+from pip._internal.cli.progress_bars import DownloadProgressProvider
+from pip._internal.exceptions import NetworkConnectionError
+from pip._internal.models.index import PyPI
+from pip._internal.network.cache import is_from_cache
+from pip._internal.network.utils import (
+    HEADERS,
+    raise_for_status,
+    response_chunks,
+)
+from pip._internal.utils.misc import (
+    format_size,
+    redact_auth_from_url,
+    splitext,
+)
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Iterable, Optional
+
+    from pip._vendor.requests.models import Response
+
+    from pip._internal.models.link import Link
+    from pip._internal.network.session import PipSession
+
+logger = logging.getLogger(__name__)
+
+
+def _get_http_response_size(resp):
+    # type: (Response) -> Optional[int]
+    try:
+        return int(resp.headers['content-length'])
+    except (ValueError, KeyError, TypeError):
+        return None
+
+
+def _prepare_download(
+    resp,  # type: Response
+    link,  # type: Link
+    progress_bar  # type: str
+):
+    # type: (...) -> Iterable[bytes]
+    total_length = _get_http_response_size(resp)
+
+    if link.netloc == PyPI.file_storage_domain:
+        url = link.show_url
+    else:
+        url = link.url_without_fragment
+
+    logged_url = redact_auth_from_url(url)
+
+    if total_length:
+        logged_url = '{} ({})'.format(logged_url, format_size(total_length))
+
+    if is_from_cache(resp):
+        logger.info("Using cached %s", logged_url)
+    else:
+        logger.info("Downloading %s", logged_url)
+
+    if logger.getEffectiveLevel() > logging.INFO:
+        show_progress = False
+    elif is_from_cache(resp):
+        show_progress = False
+    elif not total_length:
+        show_progress = True
+    elif total_length > (40 * 1000):
+        show_progress = True
+    else:
+        show_progress = False
+
+    chunks = response_chunks(resp, CONTENT_CHUNK_SIZE)
+
+    if not show_progress:
+        return chunks
+
+    return DownloadProgressProvider(
+        progress_bar, max=total_length
+    )(chunks)
+
+
+def sanitize_content_filename(filename):
+    # type: (str) -> str
+    """
+    Sanitize the "filename" value from a Content-Disposition header.
+    """
+    return os.path.basename(filename)
+
+
+def parse_content_disposition(content_disposition, default_filename):
+    # type: (str, str) -> str
+    """
+    Parse the "filename" value from a Content-Disposition header, and
+    return the default filename if the result is empty.
+    """
+    _type, params = cgi.parse_header(content_disposition)
+    filename = params.get('filename')
+    if filename:
+        # We need to sanitize the filename to prevent directory traversal
+        # in case the filename contains ".." path parts.
+        filename = sanitize_content_filename(filename)
+    return filename or default_filename
+
+
+def _get_http_response_filename(resp, link):
+    # type: (Response, Link) -> str
+    """Get an ideal filename from the given HTTP response, falling back to
+    the link filename if not provided.
+    """
+    filename = link.filename  # fallback
+    # Have a look at the Content-Disposition header for a better guess
+    content_disposition = resp.headers.get('content-disposition')
+    if content_disposition:
+        filename = parse_content_disposition(content_disposition, filename)
+    ext = splitext(filename)[1]  # type: Optional[str]
+    if not ext:
+        ext = mimetypes.guess_extension(
+            resp.headers.get('content-type', '')
+        )
+        if ext:
+            filename += ext
+    if not ext and link.url != resp.url:
+        ext = os.path.splitext(resp.url)[1]
+        if ext:
+            filename += ext
+    return filename
+
+
+def _http_get_download(session, link):
+    # type: (PipSession, Link) -> Response
+    target_url = link.url.split('#', 1)[0]
+    resp = session.get(target_url, headers=HEADERS, stream=True)
+    raise_for_status(resp)
+    return resp
+
+
+class Download(object):
+    def __init__(
+        self,
+        response,  # type: Response
+        filename,  # type: str
+        chunks,  # type: Iterable[bytes]
+    ):
+        # type: (...) -> None
+        self.response = response
+        self.filename = filename
+        self.chunks = chunks
+
+
+class Downloader(object):
+    def __init__(
+        self,
+        session,  # type: PipSession
+        progress_bar,  # type: str
+    ):
+        # type: (...) -> None
+        self._session = session
+        self._progress_bar = progress_bar
+
+    def __call__(self, link):
+        # type: (Link) -> Download
+        try:
+            resp = _http_get_download(self._session, link)
+        except NetworkConnectionError as e:
+            assert e.response is not None
+            logger.critical(
+                "HTTP error %s while getting %s", e.response.status_code, link
+            )
+            raise
+
+        return Download(
+            resp,
+            _get_http_response_filename(resp, link),
+            _prepare_download(resp, link, self._progress_bar),
+        )

venv/Lib/site-packages/pip/_internal/network/lazy_wheel.py

@@ -0,0 +1,235 @@
+"""Lazy ZIP over HTTP"""
+
+__all__ = ['HTTPRangeRequestUnsupported', 'dist_from_wheel_url']
+
+from bisect import bisect_left, bisect_right
+from contextlib import contextmanager
+from tempfile import NamedTemporaryFile
+from zipfile import BadZipfile, ZipFile
+
+from pip._vendor.requests.models import CONTENT_CHUNK_SIZE
+from pip._vendor.six.moves import range
+
+from pip._internal.network.utils import (
+    HEADERS,
+    raise_for_status,
+    response_chunks,
+)
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
+
+if MYPY_CHECK_RUNNING:
+    from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+    from pip._vendor.pkg_resources import Distribution
+    from pip._vendor.requests.models import Response
+
+    from pip._internal.network.session import PipSession
+
+
+class HTTPRangeRequestUnsupported(Exception):
+    pass
+
+
+def dist_from_wheel_url(name, url, session):
+    # type: (str, str, PipSession) -> Distribution
+    """Return a pkg_resources.Distribution from the given wheel URL.
+
+    This uses HTTP range requests to only fetch the potion of the wheel
+    containing metadata, just enough for the object to be constructed.
+    If such requests are not supported, HTTPRangeRequestUnsupported
+    is raised.
+    """
+    with LazyZipOverHTTP(url, session) as wheel:
+        # For read-only ZIP files, ZipFile only needs methods read,
+        # seek, seekable and tell, not the whole IO protocol.
+        zip_file = ZipFile(wheel)  # type: ignore
+        # After context manager exit, wheel.name
+        # is an invalid file by intention.
+        return pkg_resources_distribution_for_wheel(zip_file, name, wheel.name)
+
+
+class LazyZipOverHTTP(object):
+    """File-like object mapped to a ZIP file over HTTP.
+
+    This uses HTTP range requests to lazily fetch the file's content,
+    which is supposed to be fed to ZipFile.  If such requests are not
+    supported by the server, raise HTTPRangeRequestUnsupported
+    during initialization.
+    """
+
+    def __init__(self, url, session, chunk_size=CONTENT_CHUNK_SIZE):
+        # type: (str, PipSession, int) -> None
+        head = session.head(url, headers=HEADERS)
+        raise_for_status(head)
+        assert head.status_code == 200
+        self._session, self._url, self._chunk_size = session, url, chunk_size
+        self._length = int(head.headers['Content-Length'])
+        self._file = NamedTemporaryFile()
+        self.truncate(self._length)
+        self._left = []  # type: List[int]
+        self._right = []  # type: List[int]
+        if 'bytes' not in head.headers.get('Accept-Ranges', 'none'):
+            raise HTTPRangeRequestUnsupported('range request is not supported')
+        self._check_zip()
+
+    @property
+    def mode(self):
+        # type: () -> str
+        """Opening mode, which is always rb."""
+        return 'rb'
+
+    @property
+    def name(self):
+        # type: () -> str
+        """Path to the underlying file."""
+        return self._file.name
+
+    def seekable(self):
+        # type: () -> bool
+        """Return whether random access is supported, which is True."""
+        return True
+
+    def close(self):
+        # type: () -> None
+        """Close the file."""
+        self._file.close()
+
+    @property
+    def closed(self):
+        # type: () -> bool
+        """Whether the file is closed."""
+        return self._file.closed
+
+    def read(self, size=-1):
+        # type: (int) -> bytes
+        """Read up to size bytes from the object and return them.
+
+        As a convenience, if size is unspecified or -1,
+        all bytes until EOF are returned.  Fewer than
+        size bytes may be returned if EOF is reached.
+        """
+        download_size = max(size, self._chunk_size)
+        start, length = self.tell(), self._length
+        stop = length if size < 0 else min(start+download_size, length)
+        start = max(0, stop-download_size)
+        self._download(start, stop-1)
+        return self._file.read(size)
+
+    def readable(self):
+        # type: () -> bool
+        """Return whether the file is readable, which is True."""
+        return True
+
+    def seek(self, offset, whence=0):
+        # type: (int, int) -> int
+        """Change stream position and return the new absolute position.
+
+        Seek to offset relative position indicated by whence:
+        * 0: Start of stream (the default).  pos should be >= 0;
+        * 1: Current position - pos may be negative;
+        * 2: End of stream - pos usually negative.
+        """
+        return self._file.seek(offset, whence)
+
+    def tell(self):
+        # type: () -> int
+        """Return the current possition."""
+        return self._file.tell()
+
+    def truncate(self, size=None):
+        # type: (Optional[int]) -> int
+        """Resize the stream to the given size in bytes.
+
+        If size is unspecified resize to the current position.
+        The current stream position isn't changed.
+
+        Return the new file size.
+        """
+        return self._file.truncate(size)
+
+    def writable(self):
+        # type: () -> bool
+        """Return False."""
+        return False
+
+    def __enter__(self):
+        # type: () -> LazyZipOverHTTP
+        self._file.__enter__()
+        return self
+
+    def __exit__(self, *exc):
+        # type: (*Any) -> Optional[bool]
+        return self._file.__exit__(*exc)
+
+    @contextmanager
+    def _stay(self):
+        # type: ()-> Iterator[None]
+        """Return a context manager keeping the position.
+
+        At the end of the block, seek back to original position.
+        """
+        pos = self.tell()
+        try:
+            yield
+        finally:
+            self.seek(pos)
+
+    def _check_zip(self):
+        # type: () -> None
+        """Check and download until the file is a valid ZIP."""
+        end = self._length - 1
+        for start in reversed(range(0, end, self._chunk_size)):
+            self._download(start, end)
+            with self._stay():
+                try:
+                    # For read-only ZIP files, ZipFile only needs
+                    # methods read, seek, seekable and tell.
+                    ZipFile(self)  # type: ignore
+                except BadZipfile:
+                    pass
+                else:
+                    break
+
+    def _stream_response(self, start, end, base_headers=HEADERS):
+        # type: (int, int, Dict[str, str]) -> Response
+        """Return HTTP response to a range request from start to end."""
+        headers = base_headers.copy()
+        headers['Range'] = 'bytes={}-{}'.format(start, end)
+        # TODO: Get range requests to be correctly cached
+        headers['Cache-Control'] = 'no-cache'
+        return self._session.get(self._url, headers=headers, stream=True)
+
+    def _merge(self, start, end, left, right):
+        # type: (int, int, int, int) -> Iterator[Tuple[int, int]]
+        """Return an iterator of intervals to be fetched.
+
+        Args:
+            start (int): Start of needed interval
+            end (int): End of needed interval
+            left (int): Index of first overlapping downloaded data
+            right (int): Index after last overlapping downloaded data
+        """
+        lslice, rslice = self._left[left:right], self._right[left:right]
+        i = start = min([start]+lslice[:1])
+        end = max([end]+rslice[-1:])
+        for j, k in zip(lslice, rslice):
+            if j > i:
+                yield i, j-1
+            i = k + 1
+        if i <= end:
+            yield i, end
+        self._left[left:right], self._right[left:right] = [start], [end]
+
+    def _download(self, start, end):
+        # type: (int, int) -> None
+        """Download bytes from start to end inclusively."""
+        with self._stay():
+            left = bisect_left(self._right, start)
+            right = bisect_right(self._left, end)
+            for start, end in self._merge(start, end, left, right):
+                response = self._stream_response(start, end)
+                response.raise_for_status()
+                self.seek(start)
+                for chunk in response_chunks(response, self._chunk_size):
+                    self._file.write(chunk)

venv/Lib/site-packages/pip/_internal/network/session.py

@@ -0,0 +1,421 @@
+"""PipSession and supporting code, containing all pip-specific
+network request configuration and behavior.
+"""
+
+# The following comment should be removed at some point in the future.
+# mypy: disallow-untyped-defs=False
+
+import email.utils
+import json
+import logging
+import mimetypes
+import os
+import platform
+import sys
+import warnings
+
+from pip._vendor import requests, six, urllib3
+from pip._vendor.cachecontrol import CacheControlAdapter
+from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
+from pip._vendor.requests.models import Response
+from pip._vendor.requests.structures import CaseInsensitiveDict
+from pip._vendor.six.moves.urllib import parse as urllib_parse
+from pip._vendor.urllib3.exceptions import InsecureRequestWarning
+
+from pip import __version__
+from pip._internal.network.auth import MultiDomainBasicAuth
+from pip._internal.network.cache import SafeFileCache
+# Import ssl from compat so the initial import occurs in only one place.
+from pip._internal.utils.compat import has_tls, ipaddress
+from pip._internal.utils.glibc import libc_ver
+from pip._internal.utils.misc import (
+    build_url_from_netloc,
+    get_installed_version,
+    parse_netloc,
+)
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.urls import url_to_path
+
+if MYPY_CHECK_RUNNING:
+    from typing import (
+        Iterator, List, Optional, Tuple, Union,
+    )
+
+    from pip._internal.models.link import Link
+
+    SecureOrigin = Tuple[str, str, Optional[Union[int, str]]]
+
+
+logger = logging.getLogger(__name__)
+
+
+# Ignore warning raised when using --trusted-host.
+warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+
+
+SECURE_ORIGINS = [
+    # protocol, hostname, port
+    # Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC)
+    ("https", "*", "*"),
+    ("*", "localhost", "*"),
+    ("*", "127.0.0.0/8", "*"),
+    ("*", "::1/128", "*"),
+    ("file", "*", None),
+    # ssh is always secure.
+    ("ssh", "*", "*"),
+]  # type: List[SecureOrigin]
+
+
+# These are environment variables present when running under various
+# CI systems.  For each variable, some CI systems that use the variable
+# are indicated.  The collection was chosen so that for each of a number
+# of popular systems, at least one of the environment variables is used.
+# This list is used to provide some indication of and lower bound for
+# CI traffic to PyPI.  Thus, it is okay if the list is not comprehensive.
+# For more background, see: https://github.com/pypa/pip/issues/5499
+CI_ENVIRONMENT_VARIABLES = (
+    # Azure Pipelines
+    'BUILD_BUILDID',
+    # Jenkins
+    'BUILD_ID',
+    # AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI
+    'CI',
+    # Explicit environment variable.
+    'PIP_IS_CI',
+)
+
+
+def looks_like_ci():
+    # type: () -> bool
+    """
+    Return whether it looks like pip is running under CI.
+    """
+    # We don't use the method of checking for a tty (e.g. using isatty())
+    # because some CI systems mimic a tty (e.g. Travis CI).  Thus that
+    # method doesn't provide definitive information in either direction.
+    return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES)
+
+
+def user_agent():
+    """
+    Return a string representing the user agent.
+    """
+    data = {
+        "installer": {"name": "pip", "version": __version__},
+        "python": platform.python_version(),
+        "implementation": {
+            "name": platform.python_implementation(),
+        },
+    }
+
+    if data["implementation"]["name"] == 'CPython':
+        data["implementation"]["version"] = platform.python_version()
+    elif data["implementation"]["name"] == 'PyPy':
+        if sys.pypy_version_info.releaselevel == 'final':
+            pypy_version_info = sys.pypy_version_info[:3]
+        else:
+            pypy_version_info = sys.pypy_version_info
+        data["implementation"]["version"] = ".".join(
+            [str(x) for x in pypy_version_info]
+        )
+    elif data["implementation"]["name"] == 'Jython':
+        # Complete Guess
+        data["implementation"]["version"] = platform.python_version()
+    elif data["implementation"]["name"] == 'IronPython':
+        # Complete Guess
+        data["implementation"]["version"] = platform.python_version()
+
+    if sys.platform.startswith("linux"):
+        from pip._vendor import distro
+        distro_infos = dict(filter(
+            lambda x: x[1],
+            zip(["name", "version", "id"], distro.linux_distribution()),
+        ))
+        libc = dict(filter(
+            lambda x: x[1],
+            zip(["lib", "version"], libc_ver()),
+        ))
+        if libc:
+            distro_infos["libc"] = libc
+        if distro_infos:
+            data["distro"] = distro_infos
+
+    if sys.platform.startswith("darwin") and platform.mac_ver()[0]:
+        data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]}
+
+    if platform.system():
+        data.setdefault("system", {})["name"] = platform.system()
+
+    if platform.release():
+        data.setdefault("system", {})["release"] = platform.release()
+
+    if platform.machine():
+        data["cpu"] = platform.machine()
+
+    if has_tls():
+        import _ssl as ssl
+        data["openssl_version"] = ssl.OPENSSL_VERSION
+
+    setuptools_version = get_installed_version("setuptools")
+    if setuptools_version is not None:
+        data["setuptools_version"] = setuptools_version
+
+    # Use None rather than False so as not to give the impression that
+    # pip knows it is not being run under CI.  Rather, it is a null or
+    # inconclusive result.  Also, we include some value rather than no
+    # value to make it easier to know that the check has been run.
+    data["ci"] = True if looks_like_ci() else None
+
+    user_data = os.environ.get("PIP_USER_AGENT_USER_DATA")
+    if user_data is not None:
+        data["user_data"] = user_data
+
+    return "{data[installer][name]}/{data[installer][version]} {json}".format(
+        data=data,
+        json=json.dumps(data, separators=(",", ":"), sort_keys=True),
+    )
+
+
+class LocalFSAdapter(BaseAdapter):
+
+    def send(self, request, stream=None, timeout=None, verify=None, cert=None,
+             proxies=None):
+        pathname = url_to_path(request.url)
+
+        resp = Response()
+        resp.status_code = 200
+        resp.url = request.url
+
+        try:
+            stats = os.stat(pathname)
+        except OSError as exc:
+            resp.status_code = 404
+            resp.raw = exc
+        else:
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
+            resp.headers = CaseInsensitiveDict({
+                "Content-Type": content_type,
+                "Content-Length": stats.st_size,
+                "Last-Modified": modified,
+            })
+
+            resp.raw = open(pathname, "rb")
+            resp.close = resp.raw.close
+
+        return resp
+
+    def close(self):
+        pass
+
+
+class InsecureHTTPAdapter(HTTPAdapter):
+
+    def cert_verify(self, conn, url, verify, cert):
+        super(InsecureHTTPAdapter, self).cert_verify(
+            conn=conn, url=url, verify=False, cert=cert
+        )
+
+
+class InsecureCacheControlAdapter(CacheControlAdapter):
+
+    def cert_verify(self, conn, url, verify, cert):
+        super(InsecureCacheControlAdapter, self).cert_verify(
+            conn=conn, url=url, verify=False, cert=cert
+        )
+
+
+class PipSession(requests.Session):
+
+    timeout = None  # type: Optional[int]
+
+    def __init__(self, *args, **kwargs):
+        """
+        :param trusted_hosts: Domains not to emit warnings for when not using
+            HTTPS.
+        """
+        retries = kwargs.pop("retries", 0)
+        cache = kwargs.pop("cache", None)
+        trusted_hosts = kwargs.pop("trusted_hosts", [])  # type: List[str]
+        index_urls = kwargs.pop("index_urls", None)
+
+        super(PipSession, self).__init__(*args, **kwargs)
+
+        # Namespace the attribute with "pip_" just in case to prevent
+        # possible conflicts with the base class.
+        self.pip_trusted_origins = []  # type: List[Tuple[str, Optional[int]]]
+
+        # Attach our User Agent to the request
+        self.headers["User-Agent"] = user_agent()
+
+        # Attach our Authentication handler to the session
+        self.auth = MultiDomainBasicAuth(index_urls=index_urls)
+
+        # Create our urllib3.Retry instance which will allow us to customize
+        # how we handle retries.
+        retries = urllib3.Retry(
+            # Set the total number of retries that a particular request can
+            # have.
+            total=retries,
+
+            # A 503 error from PyPI typically means that the Fastly -> Origin
+            # connection got interrupted in some way. A 503 error in general
+            # is typically considered a transient error so we'll go ahead and
+            # retry it.
+            # A 500 may indicate transient error in Amazon S3
+            # A 520 or 527 - may indicate transient error in CloudFlare
+            status_forcelist=[500, 503, 520, 527],
+
+            # Add a small amount of back off between failed requests in
+            # order to prevent hammering the service.
+            backoff_factor=0.25,
+        )
+
+        # Our Insecure HTTPAdapter disables HTTPS validation. It does not
+        # support caching so we'll use it for all http:// URLs.
+        # If caching is disabled, we will also use it for
+        # https:// hosts that we've marked as ignoring
+        # TLS errors for (trusted-hosts).
+        insecure_adapter = InsecureHTTPAdapter(max_retries=retries)
+
+        # We want to _only_ cache responses on securely fetched origins or when
+        # the host is specified as trusted. We do this because
+        # we can't validate the response of an insecurely/untrusted fetched
+        # origin, and we don't want someone to be able to poison the cache and
+        # require manual eviction from the cache to fix it.
+        if cache:
+            secure_adapter = CacheControlAdapter(
+                cache=SafeFileCache(cache),
+                max_retries=retries,
+            )
+            self._trusted_host_adapter = InsecureCacheControlAdapter(
+                cache=SafeFileCache(cache),
+                max_retries=retries,
+            )
+        else:
+            secure_adapter = HTTPAdapter(max_retries=retries)
+            self._trusted_host_adapter = insecure_adapter
+
+        self.mount("https://", secure_adapter)
+        self.mount("http://", insecure_adapter)
+
+        # Enable file:// urls
+        self.mount("file://", LocalFSAdapter())
+
+        for host in trusted_hosts:
+            self.add_trusted_host(host, suppress_logging=True)
+
+    def add_trusted_host(self, host, source=None, suppress_logging=False):
+        # type: (str, Optional[str], bool) -> None
+        """
+        :param host: It is okay to provide a host that has previously been
+            added.
+        :param source: An optional source string, for logging where the host
+            string came from.
+        """
+        if not suppress_logging:
+            msg = 'adding trusted host: {!r}'.format(host)
+            if source is not None:
+                msg += ' (from {})'.format(source)
+            logger.info(msg)
+
+        host_port = parse_netloc(host)
+        if host_port not in self.pip_trusted_origins:
+            self.pip_trusted_origins.append(host_port)
+
+        self.mount(
+            build_url_from_netloc(host) + '/',
+            self._trusted_host_adapter
+        )
+        if not host_port[1]:
+            # Mount wildcard ports for the same host.
+            self.mount(
+                build_url_from_netloc(host) + ':',
+                self._trusted_host_adapter
+            )
+
+    def iter_secure_origins(self):
+        # type: () -> Iterator[SecureOrigin]
+        for secure_origin in SECURE_ORIGINS:
+            yield secure_origin
+        for host, port in self.pip_trusted_origins:
+            yield ('*', host, '*' if port is None else port)
+
+    def is_secure_origin(self, location):
+        # type: (Link) -> bool
+        # Determine if this url used a secure transport mechanism
+        parsed = urllib_parse.urlparse(str(location))
+        origin_protocol, origin_host, origin_port = (
+            parsed.scheme, parsed.hostname, parsed.port,
+        )
+
+        # The protocol to use to see if the protocol matches.
+        # Don't count the repository type as part of the protocol: in
+        # cases such as "git+ssh", only use "ssh". (I.e., Only verify against
+        # the last scheme.)
+        origin_protocol = origin_protocol.rsplit('+', 1)[-1]
+
+        # Determine if our origin is a secure origin by looking through our
+        # hardcoded list of secure origins, as well as any additional ones
+        # configured on this PackageFinder instance.
+        for secure_origin in self.iter_secure_origins():
+            secure_protocol, secure_host, secure_port = secure_origin
+            if origin_protocol != secure_protocol and secure_protocol != "*":
+                continue
+
+            try:
+                addr = ipaddress.ip_address(
+                    None
+                    if origin_host is None
+                    else six.ensure_text(origin_host)
+                )
+                network = ipaddress.ip_network(
+                    six.ensure_text(secure_host)
+                )
+            except ValueError:
+                # We don't have both a valid address or a valid network, so
+                # we'll check this origin against hostnames.
+                if (
+                    origin_host and
+                    origin_host.lower() != secure_host.lower() and
+                    secure_host != "*"
+                ):
+                    continue
+            else:
+                # We have a valid address and network, so see if the address
+                # is contained within the network.
+                if addr not in network:
+                    continue
+
+            # Check to see if the port matches.
+            if (
+                origin_port != secure_port and
+                secure_port != "*" and
+                secure_port is not None
+            ):
+                continue
+
+            # If we've gotten here, then this origin matches the current
+            # secure origin and we should return True
+            return True
+
+        # If we've gotten to this point, then the origin isn't secure and we
+        # will not accept it as a valid location to search. We will however
+        # log a warning that we are ignoring it.
+        logger.warning(
+            "The repository located at %s is not a trusted or secure host and "
+            "is being ignored. If this repository is available via HTTPS we "
+            "recommend you use HTTPS instead, otherwise you may silence "
+            "this warning and allow it anyway with '--trusted-host %s'.",
+            origin_host,
+            origin_host,
+        )
+
+        return False
+
+    def request(self, method, url, *args, **kwargs):
+        # Allow setting a default timeout on a session
+        kwargs.setdefault("timeout", self.timeout)
+
+        # Dispatch the actual request
+        return super(PipSession, self).request(method, url, *args, **kwargs)

venv/Lib/site-packages/pip/_internal/network/utils.py

@@ -0,0 +1,97 @@
+from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
+
+from pip._internal.exceptions import NetworkConnectionError
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Dict, Iterator
+
+# The following comments and HTTP headers were originally added by
+# Donald Stufft in git commit 22c562429a61bb77172039e480873fb239dd8c03.
+#
+# We use Accept-Encoding: identity here because requests defaults to
+# accepting compressed responses. This breaks in a variety of ways
+# depending on how the server is configured.
+# - Some servers will notice that the file isn't a compressible file
+#   and will leave the file alone and with an empty Content-Encoding
+# - Some servers will notice that the file is already compressed and
+#   will leave the file alone, adding a Content-Encoding: gzip header
+# - Some servers won't notice anything at all and will take a file
+#   that's already been compressed and compress it again, and set
+#   the Content-Encoding: gzip header
+# By setting this to request only the identity encoding we're hoping
+# to eliminate the third case.  Hopefully there does not exist a server
+# which when given a file will notice it is already compressed and that
+# you're not asking for a compressed file and will then decompress it
+# before sending because if that's the case I don't think it'll ever be
+# possible to make this work.
+HEADERS = {'Accept-Encoding': 'identity'}  # type: Dict[str, str]
+
+
+def raise_for_status(resp):
+    # type: (Response) -> None
+    http_error_msg = u''
+    if isinstance(resp.reason, bytes):
+        # We attempt to decode utf-8 first because some servers
+        # choose to localize their reason strings. If the string
+        # isn't utf-8, we fall back to iso-8859-1 for all other
+        # encodings.
+        try:
+            reason = resp.reason.decode('utf-8')
+        except UnicodeDecodeError:
+            reason = resp.reason.decode('iso-8859-1')
+    else:
+        reason = resp.reason
+
+    if 400 <= resp.status_code < 500:
+        http_error_msg = u'%s Client Error: %s for url: %s' % (
+            resp.status_code, reason, resp.url)
+
+    elif 500 <= resp.status_code < 600:
+        http_error_msg = u'%s Server Error: %s for url: %s' % (
+            resp.status_code, reason, resp.url)
+
+    if http_error_msg:
+        raise NetworkConnectionError(http_error_msg, response=resp)
+
+
+def response_chunks(response, chunk_size=CONTENT_CHUNK_SIZE):
+    # type: (Response, int) -> Iterator[bytes]
+    """Given a requests Response, provide the data chunks.
+    """
+    try:
+        # Special case for urllib3.
+        for chunk in response.raw.stream(
+            chunk_size,
+            # We use decode_content=False here because we don't
+            # want urllib3 to mess with the raw bytes we get
+            # from the server. If we decompress inside of
+            # urllib3 then we cannot verify the checksum
+            # because the checksum will be of the compressed
+            # file. This breakage will only occur if the
+            # server adds a Content-Encoding header, which
+            # depends on how the server was configured:
+            # - Some servers will notice that the file isn't a
+            #   compressible file and will leave the file alone
+            #   and with an empty Content-Encoding
+            # - Some servers will notice that the file is
+            #   already compressed and will leave the file
+            #   alone and will add a Content-Encoding: gzip
+            #   header
+            # - Some servers won't notice anything at all and
+            #   will take a file that's already been compressed
+            #   and compress it again and set the
+            #   Content-Encoding: gzip header
+            #
+            # By setting this not to decode automatically we
+            # hope to eliminate problems with the second case.
+            decode_content=False,
+        ):
+            yield chunk
+    except AttributeError:
+        # Standard file-like object.
+        while True:
+            chunk = response.raw.read(chunk_size)
+            if not chunk:
+                break
+            yield chunk

venv/Lib/site-packages/pip/_internal/network/xmlrpc.py

@@ -0,0 +1,52 @@
+"""xmlrpclib.Transport implementation
+"""
+
+import logging
+
+# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
+#       why we ignore the type on this import
+from pip._vendor.six.moves import xmlrpc_client  # type: ignore
+from pip._vendor.six.moves.urllib import parse as urllib_parse
+
+from pip._internal.exceptions import NetworkConnectionError
+from pip._internal.network.utils import raise_for_status
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import Dict
+    from pip._internal.network.session import PipSession
+
+
+logger = logging.getLogger(__name__)
+
+
+class PipXmlrpcTransport(xmlrpc_client.Transport):
+    """Provide a `xmlrpclib.Transport` implementation via a `PipSession`
+    object.
+    """
+
+    def __init__(self, index_url, session, use_datetime=False):
+        # type: (str, PipSession, bool) -> None
+        xmlrpc_client.Transport.__init__(self, use_datetime)
+        index_parts = urllib_parse.urlparse(index_url)
+        self._scheme = index_parts.scheme
+        self._session = session
+
+    def request(self, host, handler, request_body, verbose=False):
+        # type: (str, str, Dict[str, str], bool) -> None
+        parts = (self._scheme, host, handler, None, None, None)
+        url = urllib_parse.urlunparse(parts)
+        try:
+            headers = {'Content-Type': 'text/xml'}
+            response = self._session.post(url, data=request_body,
+                                          headers=headers, stream=True)
+            raise_for_status(response)
+            self.verbose = verbose
+            return self.parse_response(response.raw)
+        except NetworkConnectionError as exc:
+            assert exc.response
+            logger.critical(
+                "HTTP error %s while getting %s",
+                exc.response.status_code, url,
+            )
+            raise