feat(ST2.GitPackages): bump up all packages
- Refresh PackageCache with latest versions of everything
This commit is contained in:
Iristyle
@@ -0,0 +1,390 @@
|
||||
# python3-compatible git library from https://github.com/kemayo/sublime-text-2-git
|
||||
# including this temporarily until a Package Control installable version of Git
|
||||
# is available
|
||||
import os
|
||||
import sublime
|
||||
import sublime_plugin
|
||||
import threading
|
||||
import subprocess
|
||||
import functools
|
||||
import os.path
|
||||
import time
|
||||
|
||||
# In a complete inversion from ST2, in ST3 when a plugin is loaded we
|
||||
# actually can trust __file__.
|
||||
# Goal is to get: "Packages/Git", allowing for people who rename things
|
||||
FULL_PLUGIN_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
|
||||
PLUGIN_DIRECTORY = FULL_PLUGIN_DIRECTORY.replace(os.path.normpath(os.path.join(FULL_PLUGIN_DIRECTORY, '..', '..')) + os.path.sep, '').replace(os.path.sep, '/')
|
||||
|
||||
git_root_cache = {}
|
||||
|
||||
|
||||
def main_thread(callback, *args, **kwargs):
|
||||
# sublime.set_timeout gets used to send things onto the main thread
|
||||
# most sublime.[something] calls need to be on the main thread
|
||||
sublime.set_timeout(functools.partial(callback, *args, **kwargs), 0)
|
||||
|
||||
|
||||
def open_url(url):
|
||||
sublime.active_window().run_command('open_url', {"url": url})
|
||||
|
||||
|
||||
def git_root(directory):
|
||||
global git_root_cache
|
||||
|
||||
retval = False
|
||||
leaf_dir = directory
|
||||
|
||||
if leaf_dir in git_root_cache and git_root_cache[leaf_dir]['expires'] > time.time():
|
||||
return git_root_cache[leaf_dir]['retval']
|
||||
|
||||
while directory:
|
||||
if os.path.exists(os.path.join(directory, '.git')):
|
||||
retval = directory
|
||||
break
|
||||
parent = os.path.realpath(os.path.join(directory, os.path.pardir))
|
||||
if parent == directory:
|
||||
# /.. == /
|
||||
retval = False
|
||||
break
|
||||
directory = parent
|
||||
|
||||
git_root_cache[leaf_dir] = {
|
||||
'retval': retval,
|
||||
'expires': time.time() + 5
|
||||
}
|
||||
|
||||
return retval
|
||||
|
||||
|
||||
# for readability code
|
||||
def git_root_exist(directory):
|
||||
return git_root(directory)
|
||||
|
||||
|
||||
def view_contents(view):
|
||||
region = sublime.Region(0, view.size())
|
||||
return view.substr(region)
|
||||
|
||||
|
||||
def plugin_file(name):
|
||||
return os.path.join(PLUGIN_DIRECTORY, name)
|
||||
|
||||
|
||||
def do_when(conditional, callback, *args, **kwargs):
|
||||
if conditional():
|
||||
return callback(*args, **kwargs)
|
||||
sublime.set_timeout(functools.partial(do_when, conditional, callback, *args, **kwargs), 50)
|
||||
|
||||
|
||||
def _make_text_safeish(text, fallback_encoding, method='decode'):
|
||||
# The unicode decode here is because sublime converts to unicode inside
|
||||
# insert in such a way that unknown characters will cause errors, which is
|
||||
# distinctly non-ideal... and there's no way to tell what's coming out of
|
||||
# git in output. So...
|
||||
try:
|
||||
unitext = getattr(text, method)('utf-8')
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
unitext = getattr(text, method)(fallback_encoding)
|
||||
except AttributeError:
|
||||
# strongly implies we're already unicode, but just in case let's cast
|
||||
# to string
|
||||
unitext = str(text)
|
||||
return unitext
|
||||
|
||||
|
||||
def _test_paths_for_executable(paths, test_file):
|
||||
for directory in paths:
|
||||
file_path = os.path.join(directory, test_file)
|
||||
if os.path.exists(file_path) and os.access(file_path, os.X_OK):
|
||||
return file_path
|
||||
def find_git():
|
||||
# It turns out to be difficult to reliably run git, with varying paths
|
||||
# and subprocess environments across different platforms. So. Let's hack
|
||||
# this a bit.
|
||||
# (Yes, I could fall back on a hardline "set your system path properly"
|
||||
# attitude. But that involves a lot more arguing with people.)
|
||||
path = os.environ.get('PATH', '').split(os.pathsep)
|
||||
if os.name == 'nt':
|
||||
git_cmd = 'git.exe'
|
||||
else:
|
||||
git_cmd = 'git'
|
||||
|
||||
git_path = _test_paths_for_executable(path, git_cmd)
|
||||
|
||||
if not git_path:
|
||||
# /usr/local/bin:/usr/local/git/bin
|
||||
if os.name == 'nt':
|
||||
extra_paths = (
|
||||
os.path.join(os.environ["ProgramFiles"], "Git", "bin"),
|
||||
os.path.join(os.environ["ProgramFiles(x86)"], "Git", "bin"),
|
||||
)
|
||||
else:
|
||||
extra_paths = (
|
||||
'/usr/local/bin',
|
||||
'/usr/local/git/bin',
|
||||
)
|
||||
git_path = _test_paths_for_executable(extra_paths, git_cmd)
|
||||
return git_path
|
||||
GIT = find_git()
|
||||
|
||||
|
||||
class CommandThread(threading.Thread):
|
||||
def __init__(self, command, on_done, working_dir="", fallback_encoding="", **kwargs):
|
||||
threading.Thread.__init__(self)
|
||||
self.command = command
|
||||
self.on_done = on_done
|
||||
self.working_dir = working_dir
|
||||
if "stdin" in kwargs:
|
||||
self.stdin = kwargs["stdin"].encode()
|
||||
else:
|
||||
self.stdin = None
|
||||
if "stdout" in kwargs:
|
||||
self.stdout = kwargs["stdout"]
|
||||
else:
|
||||
self.stdout = subprocess.PIPE
|
||||
self.fallback_encoding = fallback_encoding
|
||||
self.kwargs = kwargs
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
# Ignore directories that no longer exist
|
||||
if not os.path.isdir(self.working_dir):
|
||||
return
|
||||
|
||||
if self.working_dir != "":
|
||||
os.chdir(self.working_dir)
|
||||
|
||||
# Windows needs startupinfo in order to start process in background
|
||||
startupinfo = None
|
||||
if os.name == 'nt':
|
||||
startupinfo = subprocess.STARTUPINFO()
|
||||
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
||||
|
||||
# universal_newlines seems to break `log` in python3
|
||||
proc = subprocess.Popen(self.command,
|
||||
stdout=self.stdout, stderr=subprocess.STDOUT,
|
||||
stdin=subprocess.PIPE, startupinfo=startupinfo,
|
||||
shell=False, universal_newlines=False)
|
||||
output = proc.communicate(self.stdin)[0]
|
||||
if not output:
|
||||
output = ''
|
||||
|
||||
main_thread(self.on_done,
|
||||
_make_text_safeish(output, self.fallback_encoding), **self.kwargs)
|
||||
except subprocess.CalledProcessError as e:
|
||||
main_thread(self.on_done, e.returncode)
|
||||
except OSError as e:
|
||||
if e.errno == 2:
|
||||
main_thread(sublime.error_message, "Git binary could not be found in PATH\n\nConsider using the git_command setting for the Git plugin\n\nPATH is: %s" % os.environ['PATH'])
|
||||
else:
|
||||
raise e
|
||||
|
||||
|
||||
class GitScratchOutputCommand(sublime_plugin.TextCommand):
|
||||
def run(self, edit, output = '', output_file = None, clear = False):
|
||||
if clear:
|
||||
region = sublime.Region(0, self.view.size())
|
||||
self.view.erase(edit, region)
|
||||
self.view.insert(edit, 0, output)
|
||||
|
||||
|
||||
# A base for all commands
|
||||
class GitCommand(object):
|
||||
may_change_files = False
|
||||
|
||||
def run_command(self, command, callback=None, show_status=True,
|
||||
filter_empty_args=True, no_save=False, **kwargs):
|
||||
if filter_empty_args:
|
||||
command = [arg for arg in command if arg]
|
||||
if 'working_dir' not in kwargs:
|
||||
kwargs['working_dir'] = self.get_working_dir()
|
||||
if 'fallback_encoding' not in kwargs and self.active_view() and self.active_view().settings().get('fallback_encoding'):
|
||||
kwargs['fallback_encoding'] = self.active_view().settings().get('fallback_encoding').rpartition('(')[2].rpartition(')')[0]
|
||||
|
||||
s = sublime.load_settings("Git.sublime-settings")
|
||||
if s.get('save_first') and self.active_view() and self.active_view().is_dirty() and not no_save:
|
||||
self.active_view().run_command('save')
|
||||
if command[0] == 'git':
|
||||
if s.get('git_command'):
|
||||
command[0] = s.get('git_command')
|
||||
elif GIT:
|
||||
command[0] = GIT
|
||||
if command[0] == 'git-flow' and s.get('git_flow_command'):
|
||||
command[0] = s.get('git_flow_command')
|
||||
if not callback:
|
||||
callback = self.generic_done
|
||||
|
||||
thread = CommandThread(command, callback, **kwargs)
|
||||
thread.start()
|
||||
|
||||
if show_status:
|
||||
message = kwargs.get('status_message', False) or ' '.join(command)
|
||||
sublime.status_message(message)
|
||||
|
||||
def generic_done(self, result):
|
||||
if self.may_change_files and self.active_view() and self.active_view().file_name():
|
||||
if self.active_view().is_dirty():
|
||||
result = "WARNING: Current view is dirty.\n\n"
|
||||
else:
|
||||
# just asking the current file to be re-opened doesn't do anything
|
||||
print("reverting")
|
||||
position = self.active_view().viewport_position()
|
||||
self.active_view().run_command('revert')
|
||||
do_when(lambda: not self.active_view().is_loading(), lambda: self.active_view().set_viewport_position(position, False))
|
||||
# self.active_view().show(position)
|
||||
|
||||
view = self.active_view()
|
||||
if view and view.settings().get('live_git_annotations'):
|
||||
self.view.run_command('git_annotate')
|
||||
|
||||
if not result.strip():
|
||||
return
|
||||
self.panel(result)
|
||||
|
||||
def _output_to_view(self, output_file, output, clear=False,
|
||||
syntax="Packages/Diff/Diff.tmLanguage", **kwargs):
|
||||
output_file.set_syntax_file(syntax)
|
||||
args = {
|
||||
'output': output,
|
||||
'clear': clear
|
||||
}
|
||||
output_file.run_command('git_scratch_output', args)
|
||||
|
||||
def scratch(self, output, title=False, position=None, **kwargs):
|
||||
scratch_file = self.get_window().new_file()
|
||||
if title:
|
||||
scratch_file.set_name(title)
|
||||
scratch_file.set_scratch(True)
|
||||
self._output_to_view(scratch_file, output, **kwargs)
|
||||
scratch_file.set_read_only(True)
|
||||
if position:
|
||||
sublime.set_timeout(lambda: scratch_file.set_viewport_position(position), 0)
|
||||
return scratch_file
|
||||
|
||||
def panel(self, output, **kwargs):
|
||||
if not hasattr(self, 'output_view'):
|
||||
self.output_view = self.get_window().get_output_panel("git")
|
||||
self.output_view.set_read_only(False)
|
||||
self._output_to_view(self.output_view, output, clear=True, **kwargs)
|
||||
self.output_view.set_read_only(True)
|
||||
self.get_window().run_command("show_panel", {"panel": "output.git"})
|
||||
|
||||
def quick_panel(self, *args, **kwargs):
|
||||
self.get_window().show_quick_panel(*args, **kwargs)
|
||||
|
||||
|
||||
# A base for all git commands that work with the entire repository
|
||||
class GitWindowCommand(GitCommand, sublime_plugin.WindowCommand):
|
||||
def active_view(self):
|
||||
return self.window.active_view()
|
||||
|
||||
def _active_file_name(self):
|
||||
view = self.active_view()
|
||||
if view and view.file_name() and len(view.file_name()) > 0:
|
||||
return view.file_name()
|
||||
|
||||
@property
|
||||
def fallback_encoding(self):
|
||||
if self.active_view() and self.active_view().settings().get('fallback_encoding'):
|
||||
return self.active_view().settings().get('fallback_encoding').rpartition('(')[2].rpartition(')')[0]
|
||||
|
||||
# If there's no active view or the active view is not a file on the
|
||||
# filesystem (e.g. a search results view), we can infer the folder
|
||||
# that the user intends Git commands to run against when there's only
|
||||
# only one.
|
||||
def is_enabled(self):
|
||||
if self._active_file_name() or len(self.window.folders()) == 1:
|
||||
return bool(git_root(self.get_working_dir()))
|
||||
return False
|
||||
|
||||
def get_file_name(self):
|
||||
return ''
|
||||
|
||||
def get_relative_file_name(self):
|
||||
return ''
|
||||
|
||||
# If there is a file in the active view use that file's directory to
|
||||
# search for the Git root. Otherwise, use the only folder that is
|
||||
# open.
|
||||
def get_working_dir(self):
|
||||
file_name = self._active_file_name()
|
||||
if file_name:
|
||||
return os.path.realpath(os.path.dirname(file_name))
|
||||
else:
|
||||
try: # handle case with no open folder
|
||||
return self.window.folders()[0]
|
||||
except IndexError:
|
||||
return ''
|
||||
|
||||
def get_window(self):
|
||||
return self.window
|
||||
|
||||
|
||||
# A base for all git commands that work with the file in the active view
|
||||
class GitTextCommand(GitCommand, sublime_plugin.TextCommand):
|
||||
def active_view(self):
|
||||
return self.view
|
||||
|
||||
def is_enabled(self):
|
||||
# First, is this actually a file on the file system?
|
||||
if self.view.file_name() and len(self.view.file_name()) > 0:
|
||||
return bool(git_root(self.get_working_dir()))
|
||||
return False
|
||||
|
||||
def get_file_name(self):
|
||||
return os.path.basename(self.view.file_name())
|
||||
|
||||
def get_relative_file_name(self):
|
||||
working_dir = self.get_working_dir()
|
||||
file_path = working_dir.replace(git_root(working_dir), '')[1:]
|
||||
file_name = os.path.join(file_path, self.get_file_name())
|
||||
return file_name.replace('\\', '/') # windows issues
|
||||
|
||||
def get_working_dir(self):
|
||||
return os.path.realpath(os.path.dirname(self.view.file_name()))
|
||||
|
||||
def get_window(self):
|
||||
# Fun discovery: if you switch tabs while a command is working,
|
||||
# self.view.window() is None. (Admittedly this is a consequence
|
||||
# of my deciding to do async command processing... but, hey,
|
||||
# got to live with that now.)
|
||||
# I did try tracking the window used at the start of the command
|
||||
# and using it instead of view.window() later, but that results
|
||||
# panels on a non-visible window, which is especially useless in
|
||||
# the case of the quick panel.
|
||||
# So, this is not necessarily ideal, but it does work.
|
||||
return self.view.window() or sublime.active_window()
|
||||
|
||||
|
||||
# A few miscellaneous commands
|
||||
|
||||
|
||||
class GitCustomCommand(GitWindowCommand):
|
||||
may_change_files = True
|
||||
|
||||
def run(self):
|
||||
self.get_window().show_input_panel("Git command", "",
|
||||
self.on_input, None, None)
|
||||
|
||||
def on_input(self, command):
|
||||
command = str(command) # avoiding unicode
|
||||
if command.strip() == "":
|
||||
self.panel("No git command provided")
|
||||
return
|
||||
import shlex
|
||||
command_splitted = ['git'] + shlex.split(command)
|
||||
print(command_splitted)
|
||||
self.run_command(command_splitted)
|
||||
|
||||
|
||||
class GitGuiCommand(GitTextCommand):
|
||||
def run(self, edit):
|
||||
command = ['git', 'gui']
|
||||
self.run_command(command)
|
||||
|
||||
|
||||
class GitGitkCommand(GitTextCommand):
|
||||
def run(self, edit):
|
||||
command = ['gitk']
|
||||
self.run_command(command)
|
||||
@@ -6,25 +6,57 @@
|
||||
# /
|
||||
|
||||
"""
|
||||
requests
|
||||
~~~~~~~~
|
||||
requests HTTP library
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
:copyright: (c) 2012 by Kenneth Reitz.
|
||||
:license: ISC, see LICENSE for more details.
|
||||
Requests is an HTTP library, written in Python, for human beings. Basic GET
|
||||
usage:
|
||||
|
||||
>>> import requests
|
||||
>>> r = requests.get('http://python.org')
|
||||
>>> r.status_code
|
||||
200
|
||||
>>> 'Python is a programming language' in r.content
|
||||
True
|
||||
|
||||
... or POST:
|
||||
|
||||
>>> payload = dict(key1='value1', key2='value2')
|
||||
>>> r = requests.post("http://httpbin.org/post", data=payload)
|
||||
>>> print r.text
|
||||
{
|
||||
...
|
||||
"form": {
|
||||
"key2": "value2",
|
||||
"key1": "value1"
|
||||
},
|
||||
...
|
||||
}
|
||||
|
||||
The other HTTP methods are supported - see `requests.api`. Full documentation
|
||||
is at <http://python-requests.org>.
|
||||
|
||||
:copyright: (c) 2013 by Kenneth Reitz.
|
||||
:license: Apache 2.0, see LICENSE for more details.
|
||||
|
||||
"""
|
||||
|
||||
__title__ = 'requests'
|
||||
__version__ = '0.10.2'
|
||||
__build__ = 0x001002
|
||||
__version__ = '1.2.2'
|
||||
__build__ = 0x010202
|
||||
__author__ = 'Kenneth Reitz'
|
||||
__license__ = 'ISC'
|
||||
__copyright__ = 'Copyright 2012 Kenneth Reitz'
|
||||
|
||||
__license__ = 'Apache 2.0'
|
||||
__copyright__ = 'Copyright 2013 Kenneth Reitz'
|
||||
|
||||
# Attempt to enable urllib3's SNI support, if possible
|
||||
try:
|
||||
from requests.packages.urllib3.contrib import pyopenssl
|
||||
pyopenssl.inject_into_urllib3()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from . import utils
|
||||
from .models import Request, Response
|
||||
from .models import Request, Response, PreparedRequest
|
||||
from .api import request, get, head, post, patch, put, delete, options
|
||||
from .sessions import session, Session
|
||||
from .status_codes import codes
|
||||
@@ -32,3 +64,14 @@ from .exceptions import (
|
||||
RequestException, Timeout, URLRequired,
|
||||
TooManyRedirects, HTTPError, ConnectionError
|
||||
)
|
||||
|
||||
# Set default logging handler to avoid "No handler found" warnings.
|
||||
import logging
|
||||
try: # Python 2.7+
|
||||
from logging import NullHandler
|
||||
except ImportError:
|
||||
class NullHandler(logging.Handler):
|
||||
def emit(self, record):
|
||||
pass
|
||||
|
||||
logging.getLogger(__name__).addHandler(NullHandler())
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
requests.adapters
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
This module contains the transport adapters that Requests uses to define
|
||||
and maintain connections.
|
||||
"""
|
||||
|
||||
import socket
|
||||
|
||||
from .models import Response
|
||||
from .packages.urllib3.poolmanager import PoolManager, ProxyManager
|
||||
from .packages.urllib3.response import HTTPResponse
|
||||
from .compat import urlparse, basestring, urldefrag, unquote
|
||||
from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
|
||||
prepend_scheme_if_needed, get_auth_from_url)
|
||||
from .structures import CaseInsensitiveDict
|
||||
from .packages.urllib3.exceptions import MaxRetryError
|
||||
from .packages.urllib3.exceptions import TimeoutError
|
||||
from .packages.urllib3.exceptions import SSLError as _SSLError
|
||||
from .packages.urllib3.exceptions import HTTPError as _HTTPError
|
||||
from .cookies import extract_cookies_to_jar
|
||||
from .exceptions import ConnectionError, Timeout, SSLError
|
||||
from .auth import _basic_auth_str
|
||||
|
||||
DEFAULT_POOLBLOCK = False
|
||||
DEFAULT_POOLSIZE = 10
|
||||
DEFAULT_RETRIES = 0
|
||||
|
||||
|
||||
class BaseAdapter(object):
|
||||
"""The Base Transport Adapter"""
|
||||
|
||||
def __init__(self):
|
||||
super(BaseAdapter, self).__init__()
|
||||
|
||||
def send(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def close(self):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class HTTPAdapter(BaseAdapter):
|
||||
"""The built-in HTTP Adapter for urllib3.
|
||||
|
||||
Provides a general-case interface for Requests sessions to contact HTTP and
|
||||
HTTPS urls by implementing the Transport Adapter interface. This class will
|
||||
usually be created by the :class:`Session <Session>` class under the
|
||||
covers.
|
||||
|
||||
:param pool_connections: The number of urllib3 connection pools to cache.
|
||||
:param pool_maxsize: The maximum number of connections to save in the pool.
|
||||
:param max_retries: The maximum number of retries each connection should attempt.
|
||||
:param pool_block: Whether the connection pool should block for connections.
|
||||
|
||||
Usage::
|
||||
|
||||
>>> import requests
|
||||
>>> s = requests.Session()
|
||||
>>> a = requests.adapters.HTTPAdapter()
|
||||
>>> s.mount('http://', a)
|
||||
"""
|
||||
__attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize',
|
||||
'_pool_block']
|
||||
|
||||
def __init__(self, pool_connections=DEFAULT_POOLSIZE,
|
||||
pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,
|
||||
pool_block=DEFAULT_POOLBLOCK):
|
||||
self.max_retries = max_retries
|
||||
self.config = {}
|
||||
|
||||
super(HTTPAdapter, self).__init__()
|
||||
|
||||
self._pool_connections = pool_connections
|
||||
self._pool_maxsize = pool_maxsize
|
||||
self._pool_block = pool_block
|
||||
|
||||
self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
|
||||
|
||||
def __getstate__(self):
|
||||
return dict((attr, getattr(self, attr, None)) for attr in
|
||||
self.__attrs__)
|
||||
|
||||
def __setstate__(self, state):
|
||||
for attr, value in state.items():
|
||||
setattr(self, attr, value)
|
||||
|
||||
self.init_poolmanager(self._pool_connections, self._pool_maxsize,
|
||||
block=self._pool_block)
|
||||
|
||||
def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK):
|
||||
"""Initializes a urllib3 PoolManager. This method should not be called
|
||||
from user code, and is only exposed for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param connections: The number of urllib3 connection pools to cache.
|
||||
:param maxsize: The maximum number of connections to save in the pool.
|
||||
:param block: Block when no free connections are available.
|
||||
"""
|
||||
# save these values for pickling
|
||||
self._pool_connections = connections
|
||||
self._pool_maxsize = maxsize
|
||||
self._pool_block = block
|
||||
|
||||
self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
|
||||
block=block)
|
||||
|
||||
def cert_verify(self, conn, url, verify, cert):
|
||||
"""Verify a SSL certificate. This method should not be called from user
|
||||
code, and is only exposed for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param conn: The urllib3 connection object associated with the cert.
|
||||
:param url: The requested URL.
|
||||
:param verify: Whether we should actually verify the certificate.
|
||||
:param cert: The SSL certificate to verify.
|
||||
"""
|
||||
if url.startswith('https') and verify:
|
||||
|
||||
cert_loc = None
|
||||
|
||||
# Allow self-specified cert location.
|
||||
if verify is not True:
|
||||
cert_loc = verify
|
||||
|
||||
if not cert_loc:
|
||||
cert_loc = DEFAULT_CA_BUNDLE_PATH
|
||||
|
||||
if not cert_loc:
|
||||
raise Exception("Could not find a suitable SSL CA certificate bundle.")
|
||||
|
||||
conn.cert_reqs = 'CERT_REQUIRED'
|
||||
conn.ca_certs = cert_loc
|
||||
else:
|
||||
conn.cert_reqs = 'CERT_NONE'
|
||||
conn.ca_certs = None
|
||||
|
||||
if cert:
|
||||
if not isinstance(cert, basestring):
|
||||
conn.cert_file = cert[0]
|
||||
conn.key_file = cert[1]
|
||||
else:
|
||||
conn.cert_file = cert
|
||||
|
||||
def build_response(self, req, resp):
|
||||
"""Builds a :class:`Response <requests.Response>` object from a urllib3
|
||||
response. This should not be called from user code, and is only exposed
|
||||
for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
|
||||
|
||||
:param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
|
||||
:param resp: The urllib3 response object.
|
||||
"""
|
||||
response = Response()
|
||||
|
||||
# Fallback to None if there's no status_code, for whatever reason.
|
||||
response.status_code = getattr(resp, 'status', None)
|
||||
|
||||
# Make headers case-insensitive.
|
||||
response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
|
||||
|
||||
# Set encoding.
|
||||
response.encoding = get_encoding_from_headers(response.headers)
|
||||
response.raw = resp
|
||||
response.reason = response.raw.reason
|
||||
|
||||
if isinstance(req.url, bytes):
|
||||
response.url = req.url.decode('utf-8')
|
||||
else:
|
||||
response.url = req.url
|
||||
|
||||
# Add new cookies from the server.
|
||||
extract_cookies_to_jar(response.cookies, req, resp)
|
||||
|
||||
# Give the Response some context.
|
||||
response.request = req
|
||||
response.connection = self
|
||||
|
||||
return response
|
||||
|
||||
def get_connection(self, url, proxies=None):
|
||||
"""Returns a urllib3 connection for the given URL. This should not be
|
||||
called from user code, and is only exposed for use when subclassing the
|
||||
:class:`HTTPAdapter <reqeusts.adapters.HTTPAdapter>`.
|
||||
|
||||
:param url: The URL to connect to.
|
||||
:param proxies: (optional) A Requests-style dictionary of proxies used on this request.
|
||||
"""
|
||||
proxies = proxies or {}
|
||||
proxy = proxies.get(urlparse(url).scheme)
|
||||
|
||||
if proxy:
|
||||
proxy = prepend_scheme_if_needed(proxy, urlparse(url).scheme)
|
||||
conn = ProxyManager(self.poolmanager.connection_from_url(proxy))
|
||||
else:
|
||||
conn = self.poolmanager.connection_from_url(url)
|
||||
|
||||
return conn
|
||||
|
||||
def close(self):
|
||||
"""Disposes of any internal state.
|
||||
|
||||
Currently, this just closes the PoolManager, which closes pooled
|
||||
connections.
|
||||
"""
|
||||
self.poolmanager.clear()
|
||||
|
||||
def request_url(self, request, proxies):
|
||||
"""Obtain the url to use when making the final request.
|
||||
|
||||
If the message is being sent through a proxy, the full URL has to be
|
||||
used. Otherwise, we should only use the path portion of the URL.
|
||||
|
||||
This shoudl not be called from user code, and is only exposed for use
|
||||
when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||||
:param proxies: A dictionary of schemes to proxy URLs.
|
||||
"""
|
||||
proxies = proxies or {}
|
||||
proxy = proxies.get(urlparse(request.url).scheme)
|
||||
|
||||
if proxy:
|
||||
url, _ = urldefrag(request.url)
|
||||
else:
|
||||
url = request.path_url
|
||||
|
||||
return url
|
||||
|
||||
def add_headers(self, request, **kwargs):
|
||||
"""Add any headers needed by the connection. Currently this adds a
|
||||
Proxy-Authorization header.
|
||||
|
||||
This should not be called from user code, and is only exposed for use
|
||||
when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
|
||||
:param kwargs: The keyword arguments from the call to send().
|
||||
"""
|
||||
proxies = kwargs.get('proxies', {})
|
||||
|
||||
if proxies is None:
|
||||
proxies = {}
|
||||
|
||||
proxy = proxies.get(urlparse(request.url).scheme)
|
||||
username, password = get_auth_from_url(proxy)
|
||||
|
||||
if username and password:
|
||||
# Proxy auth usernames and passwords will be urlencoded, we need
|
||||
# to decode them.
|
||||
username = unquote(username)
|
||||
password = unquote(password)
|
||||
request.headers['Proxy-Authorization'] = _basic_auth_str(username,
|
||||
password)
|
||||
|
||||
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
|
||||
"""Sends PreparedRequest object. Returns Response object.
|
||||
|
||||
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||||
:param stream: (optional) Whether to stream the request content.
|
||||
:param timeout: (optional) The timeout on the request.
|
||||
:param verify: (optional) Whether to verify SSL certificates.
|
||||
:param vert: (optional) Any user-provided SSL certificate to be trusted.
|
||||
:param proxies: (optional) The proxies dictionary to apply to the request.
|
||||
"""
|
||||
|
||||
conn = self.get_connection(request.url, proxies)
|
||||
|
||||
self.cert_verify(conn, request.url, verify, cert)
|
||||
url = self.request_url(request, proxies)
|
||||
self.add_headers(request, proxies=proxies)
|
||||
|
||||
chunked = not (request.body is None or 'Content-Length' in request.headers)
|
||||
|
||||
try:
|
||||
if not chunked:
|
||||
resp = conn.urlopen(
|
||||
method=request.method,
|
||||
url=url,
|
||||
body=request.body,
|
||||
headers=request.headers,
|
||||
redirect=False,
|
||||
assert_same_host=False,
|
||||
preload_content=False,
|
||||
decode_content=False,
|
||||
retries=self.max_retries,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
# Send the request.
|
||||
else:
|
||||
if hasattr(conn, 'proxy_pool'):
|
||||
conn = conn.proxy_pool
|
||||
|
||||
low_conn = conn._get_conn(timeout=timeout)
|
||||
low_conn.putrequest(request.method, url, skip_accept_encoding=True)
|
||||
|
||||
for header, value in request.headers.items():
|
||||
low_conn.putheader(header, value)
|
||||
|
||||
low_conn.endheaders()
|
||||
|
||||
for i in request.body:
|
||||
low_conn.send(hex(len(i))[2:].encode('utf-8'))
|
||||
low_conn.send(b'\r\n')
|
||||
low_conn.send(i)
|
||||
low_conn.send(b'\r\n')
|
||||
low_conn.send(b'0\r\n\r\n')
|
||||
|
||||
r = low_conn.getresponse()
|
||||
resp = HTTPResponse.from_httplib(r,
|
||||
pool=conn,
|
||||
connection=low_conn,
|
||||
preload_content=False,
|
||||
decode_content=False
|
||||
)
|
||||
|
||||
except socket.error as sockerr:
|
||||
raise ConnectionError(sockerr)
|
||||
|
||||
except MaxRetryError as e:
|
||||
raise ConnectionError(e)
|
||||
|
||||
except (_SSLError, _HTTPError) as e:
|
||||
if isinstance(e, _SSLError):
|
||||
raise SSLError(e)
|
||||
elif isinstance(e, TimeoutError):
|
||||
raise Timeout(e)
|
||||
else:
|
||||
raise
|
||||
|
||||
r = self.build_response(request, resp)
|
||||
|
||||
if not stream:
|
||||
r.content
|
||||
|
||||
return r
|
||||
@@ -7,12 +7,13 @@ requests.api
|
||||
This module implements the Requests API.
|
||||
|
||||
:copyright: (c) 2012 by Kenneth Reitz.
|
||||
:license: ISC, see LICENSE for more details.
|
||||
:license: Apache2, see LICENSE for more details.
|
||||
|
||||
"""
|
||||
|
||||
from . import sessions
|
||||
|
||||
|
||||
def request(method, url, **kwargs):
|
||||
"""Constructs and sends a :class:`Request <Request>`.
|
||||
Returns :class:`Response <Response>` object.
|
||||
@@ -20,7 +21,7 @@ def request(method, url, **kwargs):
|
||||
:param method: method for the new :class:`Request` object.
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload.
|
||||
@@ -28,23 +29,26 @@ def request(method, url, **kwargs):
|
||||
:param timeout: (optional) Float describing the timeout of the request.
|
||||
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param return_response: (optional) If False, an un-sent Request object will returned.
|
||||
:param session: (optional) A :class:`Session` object to be used for the request.
|
||||
:param config: (optional) A configuration dictionary.
|
||||
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
|
||||
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
|
||||
:param stream: (optional) if ``False``, the response content will be immediately downloaded.
|
||||
:param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
|
||||
|
||||
Usage::
|
||||
|
||||
>>> import requests
|
||||
>>> req = requests.request('GET', 'http://httpbin.org/get')
|
||||
<Response [200]>
|
||||
"""
|
||||
|
||||
s = kwargs.pop('session') if 'session' in kwargs else sessions.session()
|
||||
return s.request(method=method, url=url, **kwargs)
|
||||
|
||||
session = sessions.Session()
|
||||
return session.request(method=method, url=url, **kwargs)
|
||||
|
||||
|
||||
def get(url, **kwargs):
|
||||
"""Sends a GET request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
@@ -55,7 +59,7 @@ def options(url, **kwargs):
|
||||
"""Sends a OPTIONS request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
@@ -66,10 +70,10 @@ def head(url, **kwargs):
|
||||
"""Sends a HEAD request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
kwargs.setdefault('allow_redirects', False)
|
||||
return request('head', url, **kwargs)
|
||||
|
||||
|
||||
@@ -77,8 +81,8 @@ def post(url, data=None, **kwargs):
|
||||
"""Sends a POST request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return request('post', url, data=data, **kwargs)
|
||||
@@ -88,8 +92,8 @@ def put(url, data=None, **kwargs):
|
||||
"""Sends a PUT request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return request('put', url, data=data, **kwargs)
|
||||
@@ -99,8 +103,8 @@ def patch(url, data=None, **kwargs):
|
||||
"""Sends a PATCH request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return request('patch', url, data=data, **kwargs)
|
||||
@@ -110,7 +114,7 @@ def delete(url, **kwargs):
|
||||
"""Sends a DELETE request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return request('delete', url, **kwargs)
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
requests.async
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
This module contains an asynchronous replica of ``requests.api``, powered
|
||||
by gevent. All API methods return a ``Request`` instance (as opposed to
|
||||
``Response``). A list of requests can be sent with ``map()``.
|
||||
"""
|
||||
|
||||
try:
|
||||
import gevent
|
||||
from gevent import monkey as curious_george
|
||||
from gevent.pool import Pool
|
||||
except ImportError:
|
||||
raise RuntimeError('Gevent is required for requests.async.')
|
||||
|
||||
# Monkey-patch.
|
||||
curious_george.patch_all(thread=False)
|
||||
|
||||
from . import api
|
||||
|
||||
|
||||
__all__ = (
|
||||
'map',
|
||||
'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request'
|
||||
)
|
||||
|
||||
|
||||
def patched(f):
|
||||
"""Patches a given API function to not send."""
|
||||
|
||||
def wrapped(*args, **kwargs):
|
||||
|
||||
kwargs['return_response'] = False
|
||||
kwargs['prefetch'] = True
|
||||
|
||||
config = kwargs.get('config', {})
|
||||
config.update(safe_mode=True)
|
||||
|
||||
kwargs['config'] = config
|
||||
|
||||
return f(*args, **kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
def send(r, pool=None, prefetch=False):
|
||||
"""Sends the request object using the specified pool. If a pool isn't
|
||||
specified this method blocks. Pools are useful because you can specify size
|
||||
and can hence limit concurrency."""
|
||||
|
||||
if pool != None:
|
||||
return pool.spawn(r.send, prefetch=prefetch)
|
||||
|
||||
return gevent.spawn(r.send, prefetch=prefetch)
|
||||
|
||||
|
||||
# Patched requests.api functions.
|
||||
get = patched(api.get)
|
||||
options = patched(api.options)
|
||||
head = patched(api.head)
|
||||
post = patched(api.post)
|
||||
put = patched(api.put)
|
||||
patch = patched(api.patch)
|
||||
delete = patched(api.delete)
|
||||
request = patched(api.request)
|
||||
|
||||
|
||||
def map(requests, prefetch=True, size=None):
|
||||
"""Concurrently converts a list of Requests to Responses.
|
||||
|
||||
:param requests: a collection of Request objects.
|
||||
:param prefetch: If False, the content will not be downloaded immediately.
|
||||
:param size: Specifies the number of requests to make at a time. If None, no throttling occurs.
|
||||
"""
|
||||
|
||||
requests = list(requests)
|
||||
|
||||
pool = Pool(size) if size else None
|
||||
jobs = [send(r, pool, prefetch=prefetch) for r in requests]
|
||||
gevent.joinall(jobs)
|
||||
|
||||
return [r.response for r in requests]
|
||||
@@ -7,21 +7,28 @@ requests.auth
|
||||
This module contains the authentication handlers for Requests.
|
||||
"""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import hashlib
|
||||
import logging
|
||||
|
||||
from base64 import b64encode
|
||||
from .compat import urlparse, str, bytes
|
||||
from .utils import randombytes, parse_dict_header
|
||||
|
||||
from .compat import urlparse, str
|
||||
from .utils import parse_dict_header
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
|
||||
CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
|
||||
|
||||
|
||||
def _basic_auth_str(username, password):
|
||||
"""Returns a Basic Auth string."""
|
||||
|
||||
return 'Basic ' + b64encode(("%s:%s" % (username, password)).encode('utf-8')).strip().decode('utf-8')
|
||||
return 'Basic ' + b64encode(('%s:%s' % (username, password)).encode('latin1')).strip().decode('latin1')
|
||||
|
||||
|
||||
class AuthBase(object):
|
||||
@@ -43,7 +50,7 @@ class HTTPBasicAuth(AuthBase):
|
||||
|
||||
|
||||
class HTTPProxyAuth(HTTPBasicAuth):
|
||||
"""Attaches HTTP Proxy Authenetication to a given Request object."""
|
||||
"""Attaches HTTP Proxy Authentication to a given Request object."""
|
||||
def __call__(self, r):
|
||||
r.headers['Proxy-Authorization'] = _basic_auth_str(self.username, self.password)
|
||||
return r
|
||||
@@ -54,97 +61,117 @@ class HTTPDigestAuth(AuthBase):
|
||||
def __init__(self, username, password):
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.last_nonce = ''
|
||||
self.nonce_count = 0
|
||||
self.chal = {}
|
||||
|
||||
def handle_401(self, r):
|
||||
def build_digest_header(self, method, url):
|
||||
|
||||
realm = self.chal['realm']
|
||||
nonce = self.chal['nonce']
|
||||
qop = self.chal.get('qop')
|
||||
algorithm = self.chal.get('algorithm')
|
||||
opaque = self.chal.get('opaque')
|
||||
|
||||
if algorithm is None:
|
||||
_algorithm = 'MD5'
|
||||
else:
|
||||
_algorithm = algorithm.upper()
|
||||
# lambdas assume digest modules are imported at the top level
|
||||
if _algorithm == 'MD5':
|
||||
def md5_utf8(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode('utf-8')
|
||||
return hashlib.md5(x).hexdigest()
|
||||
hash_utf8 = md5_utf8
|
||||
elif _algorithm == 'SHA':
|
||||
def sha_utf8(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode('utf-8')
|
||||
return hashlib.sha1(x).hexdigest()
|
||||
hash_utf8 = sha_utf8
|
||||
# XXX MD5-sess
|
||||
KD = lambda s, d: hash_utf8("%s:%s" % (s, d))
|
||||
|
||||
if hash_utf8 is None:
|
||||
return None
|
||||
|
||||
# XXX not implemented yet
|
||||
entdig = None
|
||||
p_parsed = urlparse(url)
|
||||
path = p_parsed.path
|
||||
if p_parsed.query:
|
||||
path += '?' + p_parsed.query
|
||||
|
||||
A1 = '%s:%s:%s' % (self.username, realm, self.password)
|
||||
A2 = '%s:%s' % (method, path)
|
||||
|
||||
if qop == 'auth':
|
||||
if nonce == self.last_nonce:
|
||||
self.nonce_count += 1
|
||||
else:
|
||||
self.nonce_count = 1
|
||||
|
||||
ncvalue = '%08x' % self.nonce_count
|
||||
s = str(self.nonce_count).encode('utf-8')
|
||||
s += nonce.encode('utf-8')
|
||||
s += time.ctime().encode('utf-8')
|
||||
s += os.urandom(8)
|
||||
|
||||
cnonce = (hashlib.sha1(s).hexdigest()[:16])
|
||||
noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, hash_utf8(A2))
|
||||
respdig = KD(hash_utf8(A1), noncebit)
|
||||
elif qop is None:
|
||||
respdig = KD(hash_utf8(A1), "%s:%s" % (nonce, hash_utf8(A2)))
|
||||
else:
|
||||
# XXX handle auth-int.
|
||||
return None
|
||||
|
||||
self.last_nonce = nonce
|
||||
|
||||
# XXX should the partial digests be encoded too?
|
||||
base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
|
||||
'response="%s"' % (self.username, realm, nonce, path, respdig)
|
||||
if opaque:
|
||||
base += ', opaque="%s"' % opaque
|
||||
if algorithm:
|
||||
base += ', algorithm="%s"' % algorithm
|
||||
if entdig:
|
||||
base += ', digest="%s"' % entdig
|
||||
if qop:
|
||||
base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
|
||||
|
||||
return 'Digest %s' % (base)
|
||||
|
||||
def handle_401(self, r, **kwargs):
|
||||
"""Takes the given response and tries digest-auth, if needed."""
|
||||
|
||||
num_401_calls = getattr(self, 'num_401_calls', 1)
|
||||
s_auth = r.headers.get('www-authenticate', '')
|
||||
|
||||
if 'digest' in s_auth.lower():
|
||||
if 'digest' in s_auth.lower() and num_401_calls < 2:
|
||||
|
||||
last_nonce = ''
|
||||
nonce_count = 0
|
||||
setattr(self, 'num_401_calls', num_401_calls + 1)
|
||||
pat = re.compile(r'digest ', flags=re.IGNORECASE)
|
||||
self.chal = parse_dict_header(pat.sub('', s_auth, count=1))
|
||||
|
||||
chal = parse_dict_header(s_auth.replace('Digest ', ''))
|
||||
# Consume content and release the original connection
|
||||
# to allow our new request to reuse the same one.
|
||||
r.content
|
||||
r.raw.release_conn()
|
||||
|
||||
realm = chal['realm']
|
||||
nonce = chal['nonce']
|
||||
qop = chal.get('qop')
|
||||
algorithm = chal.get('algorithm', 'MD5')
|
||||
opaque = chal.get('opaque', None)
|
||||
|
||||
algorithm = algorithm.upper()
|
||||
# lambdas assume digest modules are imported at the top level
|
||||
if algorithm == 'MD5':
|
||||
def h(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode('utf-8')
|
||||
return hashlib.md5(x).hexdigest()
|
||||
H = h
|
||||
elif algorithm == 'SHA':
|
||||
def h(x):
|
||||
if isinstance(x, str):
|
||||
x = x.encode('utf-8')
|
||||
return hashlib.sha1(x).hexdigest()
|
||||
H = h
|
||||
# XXX MD5-sess
|
||||
KD = lambda s, d: H("%s:%s" % (s, d))
|
||||
|
||||
if H is None:
|
||||
return None
|
||||
|
||||
# XXX not implemented yet
|
||||
entdig = None
|
||||
p_parsed = urlparse(r.request.url)
|
||||
path = p_parsed.path
|
||||
if p_parsed.query:
|
||||
path += '?' + p_parsed.query
|
||||
|
||||
A1 = '%s:%s:%s' % (self.username, realm, self.password)
|
||||
A2 = '%s:%s' % (r.request.method, path)
|
||||
|
||||
if qop == 'auth':
|
||||
if nonce == last_nonce:
|
||||
nonce_count += 1
|
||||
else:
|
||||
nonce_count = 1
|
||||
last_nonce = nonce
|
||||
|
||||
ncvalue = '%08x' % nonce_count
|
||||
s = str(nonce_count).encode('utf-8')
|
||||
s += nonce.encode('utf-8')
|
||||
s += time.ctime().encode('utf-8')
|
||||
s += randombytes(8)
|
||||
|
||||
cnonce = (hashlib.sha1(s).hexdigest()[:16])
|
||||
noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
|
||||
respdig = KD(H(A1), noncebit)
|
||||
elif qop is None:
|
||||
respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
|
||||
else:
|
||||
# XXX handle auth-int.
|
||||
return None
|
||||
|
||||
# XXX should the partial digests be encoded too?
|
||||
base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
|
||||
'response="%s"' % (self.username, realm, nonce, path, respdig)
|
||||
if opaque:
|
||||
base += ', opaque="%s"' % opaque
|
||||
if entdig:
|
||||
base += ', digest="%s"' % entdig
|
||||
base += ', algorithm="%s"' % algorithm
|
||||
if qop:
|
||||
base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
|
||||
|
||||
r.request.headers['Authorization'] = 'Digest %s' % (base)
|
||||
r.request.send(anyway=True)
|
||||
_r = r.request.response
|
||||
r.request.headers['Authorization'] = self.build_digest_header(r.request.method, r.request.url)
|
||||
_r = r.connection.send(r.request, **kwargs)
|
||||
_r.history.append(r)
|
||||
|
||||
return _r
|
||||
|
||||
setattr(self, 'num_401_calls', 1)
|
||||
return r
|
||||
|
||||
def __call__(self, r):
|
||||
# If we have a saved nonce, skip the 401
|
||||
if self.last_nonce:
|
||||
r.headers['Authorization'] = self.build_digest_header(r.method, r.url)
|
||||
r.register_hook('response', self.handle_401)
|
||||
return r
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
certs.py
|
||||
~~~~~~~~
|
||||
|
||||
This module returns the preferred default CA certificate bundle.
|
||||
|
||||
If you are packaging Requests, e.g., for a Linux distribution or a managed
|
||||
environment, you can change the definition of where() to return a separately
|
||||
packaged CA bundle.
|
||||
"""
|
||||
|
||||
import os.path
|
||||
|
||||
|
||||
def where():
|
||||
"""Return the preferred certificate bundle."""
|
||||
# vendored bundle inside Requests
|
||||
return os.path.join(os.path.dirname(__file__), 'cacert.pem')
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(where())
|
||||
@@ -4,6 +4,7 @@
|
||||
pythoncompat
|
||||
"""
|
||||
|
||||
from .packages import charade as chardet
|
||||
|
||||
import sys
|
||||
|
||||
@@ -72,34 +73,41 @@ is_osx = ('darwin' in str(sys.platform).lower())
|
||||
is_hpux = ('hpux' in str(sys.platform).lower()) # Complete guess.
|
||||
is_solaris = ('solar==' in str(sys.platform).lower()) # Complete guess.
|
||||
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
# ---------
|
||||
# Specifics
|
||||
# ---------
|
||||
|
||||
|
||||
if is_py2:
|
||||
from urllib import quote, unquote, urlencode
|
||||
from urlparse import urlparse, urlunparse, urljoin, urlsplit
|
||||
from urllib import quote, unquote, quote_plus, unquote_plus, urlencode
|
||||
from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag
|
||||
from urllib2 import parse_http_list
|
||||
import cookielib
|
||||
from .packages.oreos.monkeys import SimpleCookie
|
||||
from Cookie import Morsel
|
||||
from StringIO import StringIO
|
||||
from .packages.urllib3.packages.ordered_dict import OrderedDict
|
||||
|
||||
builtin_str = str
|
||||
bytes = str
|
||||
str = unicode
|
||||
basestring = basestring
|
||||
|
||||
numeric_types = (int, long, float)
|
||||
|
||||
|
||||
elif is_py3:
|
||||
from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote
|
||||
from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag
|
||||
from urllib.request import parse_http_list
|
||||
from http import cookiejar as cookielib
|
||||
from http.cookies import SimpleCookie
|
||||
from http.cookies import Morsel
|
||||
from io import StringIO
|
||||
from collections import OrderedDict
|
||||
|
||||
builtin_str = str
|
||||
str = str
|
||||
bytes = bytes
|
||||
basestring = (str,bytes)
|
||||
|
||||
basestring = (str, bytes)
|
||||
numeric_types = (int, float)
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
Compatibility code to be able to use `cookielib.CookieJar` with requests.
|
||||
|
||||
requests.utils imports from here, so be careful with imports.
|
||||
"""
|
||||
|
||||
import collections
|
||||
from .compat import cookielib, urlparse, Morsel
|
||||
|
||||
try:
|
||||
import threading
|
||||
# grr, pyflakes: this fixes "redefinition of unused 'threading'"
|
||||
threading
|
||||
except ImportError:
|
||||
import dummy_threading as threading
|
||||
|
||||
|
||||
class MockRequest(object):
|
||||
"""Wraps a `requests.Request` to mimic a `urllib2.Request`.
|
||||
|
||||
The code in `cookielib.CookieJar` expects this interface in order to correctly
|
||||
manage cookie policies, i.e., determine whether a cookie can be set, given the
|
||||
domains of the request and the cookie.
|
||||
|
||||
The original request object is read-only. The client is responsible for collecting
|
||||
the new headers via `get_new_headers()` and interpreting them appropriately. You
|
||||
probably want `get_cookie_header`, defined below.
|
||||
"""
|
||||
|
||||
def __init__(self, request):
|
||||
self._r = request
|
||||
self._new_headers = {}
|
||||
self.type = urlparse(self._r.url).scheme
|
||||
|
||||
def get_type(self):
|
||||
return self.type
|
||||
|
||||
def get_host(self):
|
||||
return urlparse(self._r.url).netloc
|
||||
|
||||
def get_origin_req_host(self):
|
||||
return self.get_host()
|
||||
|
||||
def get_full_url(self):
|
||||
return self._r.url
|
||||
|
||||
def is_unverifiable(self):
|
||||
return True
|
||||
|
||||
def has_header(self, name):
|
||||
return name in self._r.headers or name in self._new_headers
|
||||
|
||||
def get_header(self, name, default=None):
|
||||
return self._r.headers.get(name, self._new_headers.get(name, default))
|
||||
|
||||
def add_header(self, key, val):
|
||||
"""cookielib has no legitimate use for this method; add it back if you find one."""
|
||||
raise NotImplementedError("Cookie headers should be added with add_unredirected_header()")
|
||||
|
||||
def add_unredirected_header(self, name, value):
|
||||
self._new_headers[name] = value
|
||||
|
||||
def get_new_headers(self):
|
||||
return self._new_headers
|
||||
|
||||
@property
|
||||
def unverifiable(self):
|
||||
return self.is_unverifiable()
|
||||
|
||||
@property
|
||||
def origin_req_host(self):
|
||||
return self.get_origin_req_host()
|
||||
|
||||
|
||||
class MockResponse(object):
|
||||
"""Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`.
|
||||
|
||||
...what? Basically, expose the parsed HTTP headers from the server response
|
||||
the way `cookielib` expects to see them.
|
||||
"""
|
||||
|
||||
def __init__(self, headers):
|
||||
"""Make a MockResponse for `cookielib` to read.
|
||||
|
||||
:param headers: a httplib.HTTPMessage or analogous carrying the headers
|
||||
"""
|
||||
self._headers = headers
|
||||
|
||||
def info(self):
|
||||
return self._headers
|
||||
|
||||
def getheaders(self, name):
|
||||
self._headers.getheaders(name)
|
||||
|
||||
|
||||
def extract_cookies_to_jar(jar, request, response):
|
||||
"""Extract the cookies from the response into a CookieJar.
|
||||
|
||||
:param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar)
|
||||
:param request: our own requests.Request object
|
||||
:param response: urllib3.HTTPResponse object
|
||||
"""
|
||||
# the _original_response field is the wrapped httplib.HTTPResponse object,
|
||||
req = MockRequest(request)
|
||||
# pull out the HTTPMessage with the headers and put it in the mock:
|
||||
res = MockResponse(response._original_response.msg)
|
||||
jar.extract_cookies(res, req)
|
||||
|
||||
|
||||
def get_cookie_header(jar, request):
|
||||
"""Produce an appropriate Cookie header string to be sent with `request`, or None."""
|
||||
r = MockRequest(request)
|
||||
jar.add_cookie_header(r)
|
||||
return r.get_new_headers().get('Cookie')
|
||||
|
||||
|
||||
def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
|
||||
"""Unsets a cookie by name, by default over all domains and paths.
|
||||
|
||||
Wraps CookieJar.clear(), is O(n).
|
||||
"""
|
||||
clearables = []
|
||||
for cookie in cookiejar:
|
||||
if cookie.name == name:
|
||||
if domain is None or domain == cookie.domain:
|
||||
if path is None or path == cookie.path:
|
||||
clearables.append((cookie.domain, cookie.path, cookie.name))
|
||||
|
||||
for domain, path, name in clearables:
|
||||
cookiejar.clear(domain, path, name)
|
||||
|
||||
|
||||
class CookieConflictError(RuntimeError):
|
||||
"""There are two cookies that meet the criteria specified in the cookie jar.
|
||||
Use .get and .set and include domain and path args in order to be more specific."""
|
||||
|
||||
|
||||
class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
|
||||
"""Compatibility class; is a cookielib.CookieJar, but exposes a dict interface.
|
||||
|
||||
This is the CookieJar we create by default for requests and sessions that
|
||||
don't specify one, since some clients may expect response.cookies and
|
||||
session.cookies to support dict operations.
|
||||
|
||||
Don't use the dict interface internally; it's just for compatibility with
|
||||
with external client code. All `requests` code should work out of the box
|
||||
with externally provided instances of CookieJar, e.g., LWPCookieJar and
|
||||
FileCookieJar.
|
||||
|
||||
Caution: dictionary operations that are normally O(1) may be O(n).
|
||||
|
||||
Unlike a regular CookieJar, this class is pickleable.
|
||||
"""
|
||||
|
||||
def get(self, name, default=None, domain=None, path=None):
|
||||
"""Dict-like get() that also supports optional domain and path args in
|
||||
order to resolve naming collisions from using one cookie jar over
|
||||
multiple domains. Caution: operation is O(n), not O(1)."""
|
||||
try:
|
||||
return self._find_no_duplicates(name, domain, path)
|
||||
except KeyError:
|
||||
return default
|
||||
|
||||
def set(self, name, value, **kwargs):
|
||||
"""Dict-like set() that also supports optional domain and path args in
|
||||
order to resolve naming collisions from using one cookie jar over
|
||||
multiple domains."""
|
||||
# support client code that unsets cookies by assignment of a None value:
|
||||
if value is None:
|
||||
remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path'))
|
||||
return
|
||||
|
||||
if isinstance(value, Morsel):
|
||||
c = morsel_to_cookie(value)
|
||||
else:
|
||||
c = create_cookie(name, value, **kwargs)
|
||||
self.set_cookie(c)
|
||||
return c
|
||||
|
||||
def keys(self):
|
||||
"""Dict-like keys() that returns a list of names of cookies from the jar.
|
||||
See values() and items()."""
|
||||
keys = []
|
||||
for cookie in iter(self):
|
||||
keys.append(cookie.name)
|
||||
return keys
|
||||
|
||||
def values(self):
|
||||
"""Dict-like values() that returns a list of values of cookies from the jar.
|
||||
See keys() and items()."""
|
||||
values = []
|
||||
for cookie in iter(self):
|
||||
values.append(cookie.value)
|
||||
return values
|
||||
|
||||
def items(self):
|
||||
"""Dict-like items() that returns a list of name-value tuples from the jar.
|
||||
See keys() and values(). Allows client-code to call "dict(RequestsCookieJar)
|
||||
and get a vanilla python dict of key value pairs."""
|
||||
items = []
|
||||
for cookie in iter(self):
|
||||
items.append((cookie.name, cookie.value))
|
||||
return items
|
||||
|
||||
def list_domains(self):
|
||||
"""Utility method to list all the domains in the jar."""
|
||||
domains = []
|
||||
for cookie in iter(self):
|
||||
if cookie.domain not in domains:
|
||||
domains.append(cookie.domain)
|
||||
return domains
|
||||
|
||||
def list_paths(self):
|
||||
"""Utility method to list all the paths in the jar."""
|
||||
paths = []
|
||||
for cookie in iter(self):
|
||||
if cookie.path not in paths:
|
||||
paths.append(cookie.path)
|
||||
return paths
|
||||
|
||||
def multiple_domains(self):
|
||||
"""Returns True if there are multiple domains in the jar.
|
||||
Returns False otherwise."""
|
||||
domains = []
|
||||
for cookie in iter(self):
|
||||
if cookie.domain is not None and cookie.domain in domains:
|
||||
return True
|
||||
domains.append(cookie.domain)
|
||||
return False # there is only one domain in jar
|
||||
|
||||
def get_dict(self, domain=None, path=None):
|
||||
"""Takes as an argument an optional domain and path and returns a plain old
|
||||
Python dict of name-value pairs of cookies that meet the requirements."""
|
||||
dictionary = {}
|
||||
for cookie in iter(self):
|
||||
if (domain is None or cookie.domain == domain) and (path is None
|
||||
or cookie.path == path):
|
||||
dictionary[cookie.name] = cookie.value
|
||||
return dictionary
|
||||
|
||||
def __getitem__(self, name):
|
||||
"""Dict-like __getitem__() for compatibility with client code. Throws exception
|
||||
if there are more than one cookie with name. In that case, use the more
|
||||
explicit get() method instead. Caution: operation is O(n), not O(1)."""
|
||||
|
||||
return self._find_no_duplicates(name)
|
||||
|
||||
def __setitem__(self, name, value):
|
||||
"""Dict-like __setitem__ for compatibility with client code. Throws exception
|
||||
if there is already a cookie of that name in the jar. In that case, use the more
|
||||
explicit set() method instead."""
|
||||
|
||||
self.set(name, value)
|
||||
|
||||
def __delitem__(self, name):
|
||||
"""Deletes a cookie given a name. Wraps cookielib.CookieJar's remove_cookie_by_name()."""
|
||||
remove_cookie_by_name(self, name)
|
||||
|
||||
def update(self, other):
|
||||
"""Updates this jar with cookies from another CookieJar or dict-like"""
|
||||
if isinstance(other, cookielib.CookieJar):
|
||||
for cookie in other:
|
||||
self.set_cookie(cookie)
|
||||
else:
|
||||
super(RequestsCookieJar, self).update(other)
|
||||
|
||||
def _find(self, name, domain=None, path=None):
|
||||
"""Requests uses this method internally to get cookie values. Takes as args name
|
||||
and optional domain and path. Returns a cookie.value. If there are conflicting cookies,
|
||||
_find arbitrarily chooses one. See _find_no_duplicates if you want an exception thrown
|
||||
if there are conflicting cookies."""
|
||||
for cookie in iter(self):
|
||||
if cookie.name == name:
|
||||
if domain is None or cookie.domain == domain:
|
||||
if path is None or cookie.path == path:
|
||||
return cookie.value
|
||||
|
||||
raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))
|
||||
|
||||
def _find_no_duplicates(self, name, domain=None, path=None):
|
||||
"""__get_item__ and get call _find_no_duplicates -- never used in Requests internally.
|
||||
Takes as args name and optional domain and path. Returns a cookie.value.
|
||||
Throws KeyError if cookie is not found and CookieConflictError if there are
|
||||
multiple cookies that match name and optionally domain and path."""
|
||||
toReturn = None
|
||||
for cookie in iter(self):
|
||||
if cookie.name == name:
|
||||
if domain is None or cookie.domain == domain:
|
||||
if path is None or cookie.path == path:
|
||||
if toReturn is not None: # if there are multiple cookies that meet passed in criteria
|
||||
raise CookieConflictError('There are multiple cookies with name, %r' % (name))
|
||||
toReturn = cookie.value # we will eventually return this as long as no cookie conflict
|
||||
|
||||
if toReturn:
|
||||
return toReturn
|
||||
raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))
|
||||
|
||||
def __getstate__(self):
|
||||
"""Unlike a normal CookieJar, this class is pickleable."""
|
||||
state = self.__dict__.copy()
|
||||
# remove the unpickleable RLock object
|
||||
state.pop('_cookies_lock')
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
"""Unlike a normal CookieJar, this class is pickleable."""
|
||||
self.__dict__.update(state)
|
||||
if '_cookies_lock' not in self.__dict__:
|
||||
self._cookies_lock = threading.RLock()
|
||||
|
||||
def copy(self):
|
||||
"""Return a copy of this RequestsCookieJar."""
|
||||
new_cj = RequestsCookieJar()
|
||||
new_cj.update(self)
|
||||
return new_cj
|
||||
|
||||
|
||||
def create_cookie(name, value, **kwargs):
|
||||
"""Make a cookie from underspecified parameters.
|
||||
|
||||
By default, the pair of `name` and `value` will be set for the domain ''
|
||||
and sent on every request (this is sometimes called a "supercookie").
|
||||
"""
|
||||
result = dict(
|
||||
version=0,
|
||||
name=name,
|
||||
value=value,
|
||||
port=None,
|
||||
domain='',
|
||||
path='/',
|
||||
secure=False,
|
||||
expires=None,
|
||||
discard=True,
|
||||
comment=None,
|
||||
comment_url=None,
|
||||
rest={'HttpOnly': None},
|
||||
rfc2109=False,)
|
||||
|
||||
badargs = set(kwargs) - set(result)
|
||||
if badargs:
|
||||
err = 'create_cookie() got unexpected keyword arguments: %s'
|
||||
raise TypeError(err % list(badargs))
|
||||
|
||||
result.update(kwargs)
|
||||
result['port_specified'] = bool(result['port'])
|
||||
result['domain_specified'] = bool(result['domain'])
|
||||
result['domain_initial_dot'] = result['domain'].startswith('.')
|
||||
result['path_specified'] = bool(result['path'])
|
||||
|
||||
return cookielib.Cookie(**result)
|
||||
|
||||
|
||||
def morsel_to_cookie(morsel):
|
||||
"""Convert a Morsel object into a Cookie containing the one k/v pair."""
|
||||
c = create_cookie(
|
||||
name=morsel.key,
|
||||
value=morsel.value,
|
||||
version=morsel['version'] or 0,
|
||||
port=None,
|
||||
port_specified=False,
|
||||
domain=morsel['domain'],
|
||||
domain_specified=bool(morsel['domain']),
|
||||
domain_initial_dot=morsel['domain'].startswith('.'),
|
||||
path=morsel['path'],
|
||||
path_specified=bool(morsel['path']),
|
||||
secure=bool(morsel['secure']),
|
||||
expires=morsel['max-age'] or morsel['expires'],
|
||||
discard=False,
|
||||
comment=morsel['comment'],
|
||||
comment_url=bool(morsel['comment']),
|
||||
rest={'HttpOnly': morsel['httponly']},
|
||||
rfc2109=False,)
|
||||
return c
|
||||
|
||||
|
||||
def cookiejar_from_dict(cookie_dict, cookiejar=None):
|
||||
"""Returns a CookieJar from a key/value dictionary.
|
||||
|
||||
:param cookie_dict: Dict of key/values to insert into CookieJar.
|
||||
"""
|
||||
if cookiejar is None:
|
||||
cookiejar = RequestsCookieJar()
|
||||
|
||||
if cookie_dict is not None:
|
||||
for name in cookie_dict:
|
||||
cookiejar.set_cookie(create_cookie(name, cookie_dict[name]))
|
||||
return cookiejar
|
||||
@@ -1,40 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
requests.defaults
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
This module provides the Requests configuration defaults.
|
||||
|
||||
Configurations:
|
||||
|
||||
:base_headers: Default HTTP headers.
|
||||
:verbose: Stream to write request logging to.
|
||||
:max_redirects: Maximum number of redirects allowed within a request.s
|
||||
:keep_alive: Reuse HTTP Connections?
|
||||
:max_retries: The number of times a request should be retried in the event of a connection failure.
|
||||
:danger_mode: If true, Requests will raise errors immediately.
|
||||
:safe_mode: If true, Requests will catch all errors.
|
||||
:pool_maxsize: The maximium size of an HTTP connection pool.
|
||||
:pool_connections: The number of active HTTP connection pools to use.
|
||||
"""
|
||||
|
||||
from . import __version__
|
||||
|
||||
defaults = dict()
|
||||
|
||||
|
||||
defaults['base_headers'] = {
|
||||
'User-Agent': 'python-requests/%s' % __version__,
|
||||
'Accept-Encoding': ', '.join(('identity', 'deflate', 'compress', 'gzip')),
|
||||
'Accept': '*/*'
|
||||
}
|
||||
|
||||
defaults['verbose'] = None
|
||||
defaults['max_redirects'] = 30
|
||||
defaults['pool_connections'] = 10
|
||||
defaults['pool_maxsize'] = 10
|
||||
defaults['max_retries'] = 0
|
||||
defaults['danger_mode'] = False
|
||||
defaults['safe_mode'] = False
|
||||
defaults['keep_alive'] = True
|
||||
@@ -8,24 +8,48 @@ This module contains the set of Requests' exceptions.
|
||||
|
||||
"""
|
||||
|
||||
class RequestException(Exception):
|
||||
|
||||
class RequestException(RuntimeError):
|
||||
"""There was an ambiguous exception that occurred while handling your
|
||||
request."""
|
||||
|
||||
|
||||
class HTTPError(RequestException):
|
||||
"""An HTTP error occurred."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
""" Initializes HTTPError with optional `response` object. """
|
||||
self.response = kwargs.pop('response', None)
|
||||
super(HTTPError, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class ConnectionError(RequestException):
|
||||
"""A Connection error occurred."""
|
||||
|
||||
|
||||
class SSLError(ConnectionError):
|
||||
"""An SSL error occurred."""
|
||||
|
||||
|
||||
class Timeout(RequestException):
|
||||
"""The request timed out."""
|
||||
|
||||
|
||||
class URLRequired(RequestException):
|
||||
"""A valid URL is required to make a request."""
|
||||
|
||||
|
||||
class TooManyRedirects(RequestException):
|
||||
"""Too many redirects."""
|
||||
|
||||
|
||||
class MissingSchema(RequestException, ValueError):
|
||||
"""The URL schema (e.g. http or https) is missing."""
|
||||
|
||||
|
||||
class InvalidSchema(RequestException, ValueError):
|
||||
"""See defaults.py for valid schemas."""
|
||||
|
||||
|
||||
class InvalidURL(RequestException, ValueError):
|
||||
""" The URL provided was somehow invalid. """
|
||||
|
||||
@@ -8,27 +8,25 @@ This module provides the capabilities for the Requests hooks system.
|
||||
|
||||
Available hooks:
|
||||
|
||||
``args``:
|
||||
A dictionary of the arguments being sent to Request().
|
||||
|
||||
``pre_request``:
|
||||
The Request object, directly before being sent.
|
||||
|
||||
``post_request``:
|
||||
The Request object, directly after being sent.
|
||||
|
||||
``response``:
|
||||
The response generated from a Request.
|
||||
|
||||
"""
|
||||
|
||||
import traceback
|
||||
|
||||
HOOKS = ['response']
|
||||
|
||||
|
||||
HOOKS = ('args', 'pre_request', 'post_request', 'response')
|
||||
def default_hooks():
|
||||
hooks = {}
|
||||
for event in HOOKS:
|
||||
hooks[event] = []
|
||||
return hooks
|
||||
|
||||
# TODO: response is the only one
|
||||
|
||||
|
||||
def dispatch_hook(key, hooks, hook_data):
|
||||
def dispatch_hook(key, hooks, hook_data, **kwargs):
|
||||
"""Dispatches a hook dictionary on a given piece of data."""
|
||||
|
||||
hooks = hooks or dict()
|
||||
@@ -40,9 +38,8 @@ def dispatch_hook(key, hooks, hook_data):
|
||||
hooks = [hooks]
|
||||
|
||||
for hook in hooks:
|
||||
try:
|
||||
hook_data = hook(hook_data) or hook_data
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
_hook_data = hook(hook_data, **kwargs)
|
||||
if _hook_data is not None:
|
||||
hook_data = _hook_data
|
||||
|
||||
return hook_data
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,97 +8,219 @@ This module provides a Session object to manage and persist settings across
|
||||
requests (cookies, auth, proxies).
|
||||
|
||||
"""
|
||||
import os
|
||||
from collections import Mapping
|
||||
from datetime import datetime
|
||||
|
||||
from .defaults import defaults
|
||||
from .models import Request
|
||||
from .hooks import dispatch_hook
|
||||
from .utils import header_expand
|
||||
from .packages.urllib3.poolmanager import PoolManager
|
||||
from .compat import cookielib, OrderedDict, urljoin, urlparse
|
||||
from .cookies import cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar
|
||||
from .models import Request, PreparedRequest
|
||||
from .hooks import default_hooks, dispatch_hook
|
||||
from .utils import to_key_val_list, default_headers
|
||||
from .exceptions import TooManyRedirects, InvalidSchema
|
||||
from .structures import CaseInsensitiveDict
|
||||
|
||||
from .adapters import HTTPAdapter
|
||||
|
||||
from .utils import requote_uri, get_environ_proxies, get_netrc_auth
|
||||
|
||||
from .status_codes import codes
|
||||
REDIRECT_STATI = (
|
||||
codes.moved, # 301
|
||||
codes.found, # 302
|
||||
codes.other, # 303
|
||||
codes.temporary_moved, # 307
|
||||
)
|
||||
DEFAULT_REDIRECT_LIMIT = 30
|
||||
|
||||
|
||||
def merge_kwargs(local_kwarg, default_kwarg):
|
||||
"""Merges kwarg dictionaries.
|
||||
|
||||
If a local key in the dictionary is set to None, it will be removed.
|
||||
def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
|
||||
"""
|
||||
Determines appropriate setting for a given request, taking into account the
|
||||
explicit setting on that request, and the setting in the session. If a
|
||||
setting is a dictionary, they will be merged together using `dict_class`
|
||||
"""
|
||||
|
||||
if default_kwarg is None:
|
||||
return local_kwarg
|
||||
if session_setting is None:
|
||||
return request_setting
|
||||
|
||||
if isinstance(local_kwarg, str):
|
||||
return local_kwarg
|
||||
if request_setting is None:
|
||||
return session_setting
|
||||
|
||||
if local_kwarg is None:
|
||||
return default_kwarg
|
||||
# Bypass if not a dictionary (e.g. verify)
|
||||
if not (
|
||||
isinstance(session_setting, Mapping) and
|
||||
isinstance(request_setting, Mapping)
|
||||
):
|
||||
return request_setting
|
||||
|
||||
# Bypass if not a dictionary (e.g. timeout)
|
||||
if not hasattr(default_kwarg, 'items'):
|
||||
return local_kwarg
|
||||
|
||||
# Update new values.
|
||||
kwargs = default_kwarg.copy()
|
||||
kwargs.update(local_kwarg)
|
||||
merged_setting = dict_class(to_key_val_list(session_setting))
|
||||
merged_setting.update(to_key_val_list(request_setting))
|
||||
|
||||
# Remove keys that are set to None.
|
||||
for (k,v) in list(local_kwarg.items()):
|
||||
for (k, v) in request_setting.items():
|
||||
if v is None:
|
||||
del kwargs[k]
|
||||
del merged_setting[k]
|
||||
|
||||
return kwargs
|
||||
return merged_setting
|
||||
|
||||
|
||||
class Session(object):
|
||||
"""A Requests session."""
|
||||
class SessionRedirectMixin(object):
|
||||
def resolve_redirects(self, resp, req, stream=False, timeout=None,
|
||||
verify=True, cert=None, proxies=None):
|
||||
"""Receives a Response. Returns a generator of Responses."""
|
||||
|
||||
i = 0
|
||||
prepared_request = PreparedRequest()
|
||||
prepared_request.body = req.body
|
||||
prepared_request.headers = req.headers.copy()
|
||||
prepared_request.hooks = req.hooks
|
||||
prepared_request.method = req.method
|
||||
prepared_request.url = req.url
|
||||
|
||||
# ((resp.status_code is codes.see_other))
|
||||
while (('location' in resp.headers and resp.status_code in REDIRECT_STATI)):
|
||||
|
||||
resp.content # Consume socket so it can be released
|
||||
|
||||
if i >= self.max_redirects:
|
||||
raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects)
|
||||
|
||||
# Release the connection back into the pool.
|
||||
resp.close()
|
||||
|
||||
url = resp.headers['location']
|
||||
method = prepared_request.method
|
||||
|
||||
# Handle redirection without scheme (see: RFC 1808 Section 4)
|
||||
if url.startswith('//'):
|
||||
parsed_rurl = urlparse(resp.url)
|
||||
url = '%s:%s' % (parsed_rurl.scheme, url)
|
||||
|
||||
# Facilitate non-RFC2616-compliant 'location' headers
|
||||
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
|
||||
# Compliant with RFC3986, we percent encode the url.
|
||||
if not urlparse(url).netloc:
|
||||
url = urljoin(resp.url, requote_uri(url))
|
||||
else:
|
||||
url = requote_uri(url)
|
||||
|
||||
prepared_request.url = url
|
||||
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
|
||||
if (resp.status_code == codes.see_other and
|
||||
prepared_request.method != 'HEAD'):
|
||||
method = 'GET'
|
||||
|
||||
# Do what the browsers do, despite standards...
|
||||
if (resp.status_code in (codes.moved, codes.found) and
|
||||
prepared_request.method not in ('GET', 'HEAD')):
|
||||
method = 'GET'
|
||||
|
||||
prepared_request.method = method
|
||||
|
||||
# https://github.com/kennethreitz/requests/issues/1084
|
||||
if resp.status_code not in (codes.temporary, codes.resume):
|
||||
if 'Content-Length' in prepared_request.headers:
|
||||
del prepared_request.headers['Content-Length']
|
||||
|
||||
prepared_request.body = None
|
||||
|
||||
headers = prepared_request.headers
|
||||
try:
|
||||
del headers['Cookie']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
prepared_request.prepare_cookies(self.cookies)
|
||||
|
||||
resp = self.send(
|
||||
prepared_request,
|
||||
stream=stream,
|
||||
timeout=timeout,
|
||||
verify=verify,
|
||||
cert=cert,
|
||||
proxies=proxies,
|
||||
allow_redirects=False,
|
||||
)
|
||||
|
||||
extract_cookies_to_jar(self.cookies, prepared_request, resp.raw)
|
||||
|
||||
i += 1
|
||||
yield resp
|
||||
|
||||
|
||||
class Session(SessionRedirectMixin):
|
||||
"""A Requests session.
|
||||
|
||||
Provides cookie persistience, connection-pooling, and configuration.
|
||||
|
||||
Basic Usage::
|
||||
|
||||
>>> import requests
|
||||
>>> s = requests.Session()
|
||||
>>> s.get('http://httpbin.org/get')
|
||||
200
|
||||
"""
|
||||
|
||||
__attrs__ = [
|
||||
'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
|
||||
'params', 'config']
|
||||
'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream',
|
||||
'trust_env', 'max_redirects']
|
||||
|
||||
def __init__(self):
|
||||
|
||||
def __init__(self,
|
||||
headers=None,
|
||||
cookies=None,
|
||||
auth=None,
|
||||
timeout=None,
|
||||
proxies=None,
|
||||
hooks=None,
|
||||
params=None,
|
||||
config=None,
|
||||
verify=True):
|
||||
#: A case-insensitive dictionary of headers to be sent on each
|
||||
#: :class:`Request <Request>` sent from this
|
||||
#: :class:`Session <Session>`.
|
||||
self.headers = default_headers()
|
||||
|
||||
self.headers = headers or {}
|
||||
self.cookies = cookies or {}
|
||||
self.auth = auth
|
||||
self.timeout = timeout
|
||||
self.proxies = proxies or {}
|
||||
self.hooks = hooks or {}
|
||||
self.params = params or {}
|
||||
self.config = config or {}
|
||||
self.verify = verify
|
||||
#: Default Authentication tuple or object to attach to
|
||||
#: :class:`Request <Request>`.
|
||||
self.auth = None
|
||||
|
||||
for (k, v) in list(defaults.items()):
|
||||
self.config.setdefault(k, v)
|
||||
#: Dictionary mapping protocol to the URL of the proxy (e.g.
|
||||
#: {'http': 'foo.bar:3128'}) to be used on each
|
||||
#: :class:`Request <Request>`.
|
||||
self.proxies = {}
|
||||
|
||||
self.poolmanager = PoolManager(
|
||||
num_pools=self.config.get('pool_connections'),
|
||||
maxsize=self.config.get('pool_maxsize')
|
||||
)
|
||||
#: Event-handling hooks.
|
||||
self.hooks = default_hooks()
|
||||
|
||||
#: Dictionary of querystring data to attach to each
|
||||
#: :class:`Request <Request>`. The dictionary values may be lists for
|
||||
#: representing multivalued query parameters.
|
||||
self.params = {}
|
||||
|
||||
#: Stream response content default.
|
||||
self.stream = False
|
||||
|
||||
#: SSL Verification default.
|
||||
self.verify = True
|
||||
|
||||
#: SSL certificate default.
|
||||
self.cert = None
|
||||
|
||||
#: Maximum number of redirects allowed. If the request exceeds this
|
||||
#: limit, a :class:`TooManyRedirects` exception is raised.
|
||||
self.max_redirects = DEFAULT_REDIRECT_LIMIT
|
||||
|
||||
#: Should we trust the environment?
|
||||
self.trust_env = True
|
||||
|
||||
# Set up a CookieJar to be used by default
|
||||
self.cookies = {}
|
||||
self.cookies = cookiejar_from_dict({})
|
||||
|
||||
# Add passed cookies in.
|
||||
if cookies is not None:
|
||||
self.cookies.update(cookies)
|
||||
|
||||
def __repr__(self):
|
||||
return '<requests-client at 0x%x>' % (id(self))
|
||||
# Default connection adapters.
|
||||
self.adapters = OrderedDict()
|
||||
self.mount('https://', HTTPAdapter())
|
||||
self.mount('http://', HTTPAdapter())
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
self.close()
|
||||
|
||||
def request(self, method, url,
|
||||
params=None,
|
||||
@@ -108,181 +230,272 @@ class Session(object):
|
||||
files=None,
|
||||
auth=None,
|
||||
timeout=None,
|
||||
allow_redirects=False,
|
||||
allow_redirects=True,
|
||||
proxies=None,
|
||||
hooks=None,
|
||||
return_response=True,
|
||||
config=None,
|
||||
prefetch=False,
|
||||
verify=None):
|
||||
|
||||
"""Constructs and sends a :class:`Request <Request>`.
|
||||
stream=None,
|
||||
verify=None,
|
||||
cert=None):
|
||||
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
|
||||
Returns :class:`Response <Response>` object.
|
||||
|
||||
:param method: method for the new :class:`Request` object.
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
|
||||
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the request.
|
||||
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param return_response: (optional) If False, an un-sent Request object will returned.
|
||||
:param config: (optional) A configuration dictionary.
|
||||
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
|
||||
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
|
||||
:param params: (optional) Dictionary or bytes to be sent in the query
|
||||
string for the :class:`Request`.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the
|
||||
:class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the
|
||||
:class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the
|
||||
:class:`Request`.
|
||||
:param files: (optional) Dictionary of 'filename': file-like-objects
|
||||
for multipart encoding upload.
|
||||
:param auth: (optional) Auth tuple or callable to enable
|
||||
Basic/Digest/Custom HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the
|
||||
request.
|
||||
:param allow_redirects: (optional) Boolean. Set to True by default.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of
|
||||
the proxy.
|
||||
:param stream: (optional) whether to immediately download the response
|
||||
content. Defaults to ``False``.
|
||||
:param verify: (optional) if ``True``, the SSL cert will be verified.
|
||||
A CA_BUNDLE path can also be provided.
|
||||
:param cert: (optional) if String, path to ssl client cert file (.pem).
|
||||
If Tuple, ('cert', 'key') pair.
|
||||
"""
|
||||
|
||||
method = str(method).upper()
|
||||
cookies = cookies or {}
|
||||
proxies = proxies or {}
|
||||
|
||||
# Default empty dicts for dict params.
|
||||
cookies = {} if cookies is None else cookies
|
||||
data = {} if data is None else data
|
||||
files = {} if files is None else files
|
||||
headers = {} if headers is None else headers
|
||||
params = {} if params is None else params
|
||||
hooks = {} if hooks is None else hooks
|
||||
# Bootstrap CookieJar.
|
||||
if not isinstance(cookies, cookielib.CookieJar):
|
||||
cookies = cookiejar_from_dict(cookies)
|
||||
|
||||
if verify is None:
|
||||
verify = self.verify
|
||||
# Merge with session cookies
|
||||
merged_cookies = RequestsCookieJar()
|
||||
merged_cookies.update(self.cookies)
|
||||
merged_cookies.update(cookies)
|
||||
cookies = merged_cookies
|
||||
|
||||
# use session's hooks as defaults
|
||||
for key, cb in list(self.hooks.items()):
|
||||
hooks.setdefault(key, cb)
|
||||
# Gather clues from the surrounding environment.
|
||||
if self.trust_env:
|
||||
# Set environment's proxies.
|
||||
env_proxies = get_environ_proxies(url) or {}
|
||||
for (k, v) in env_proxies.items():
|
||||
proxies.setdefault(k, v)
|
||||
|
||||
# Expand header values.
|
||||
if headers:
|
||||
for k, v in list(headers.items()) or {}:
|
||||
headers[k] = header_expand(v)
|
||||
# Set environment's basic authentication.
|
||||
if not auth:
|
||||
auth = get_netrc_auth(url)
|
||||
|
||||
args = dict(
|
||||
method=method,
|
||||
url=url,
|
||||
data=data,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=cookies,
|
||||
files=files,
|
||||
auth=auth,
|
||||
hooks=hooks,
|
||||
timeout=timeout,
|
||||
allow_redirects=allow_redirects,
|
||||
proxies=proxies,
|
||||
config=config,
|
||||
verify=verify,
|
||||
_poolmanager=self.poolmanager
|
||||
)
|
||||
# Look for configuration.
|
||||
if not verify and verify is not False:
|
||||
verify = os.environ.get('REQUESTS_CA_BUNDLE')
|
||||
|
||||
# Merge local kwargs with session kwargs.
|
||||
for attr in self.__attrs__:
|
||||
session_val = getattr(self, attr, None)
|
||||
local_val = args.get(attr)
|
||||
# Curl compatibility.
|
||||
if not verify and verify is not False:
|
||||
verify = os.environ.get('CURL_CA_BUNDLE')
|
||||
|
||||
args[attr] = merge_kwargs(local_val, session_val)
|
||||
# Merge all the kwargs.
|
||||
params = merge_setting(params, self.params)
|
||||
headers = merge_setting(headers, self.headers, dict_class=CaseInsensitiveDict)
|
||||
auth = merge_setting(auth, self.auth)
|
||||
proxies = merge_setting(proxies, self.proxies)
|
||||
hooks = merge_setting(hooks, self.hooks)
|
||||
stream = merge_setting(stream, self.stream)
|
||||
verify = merge_setting(verify, self.verify)
|
||||
cert = merge_setting(cert, self.cert)
|
||||
|
||||
# Arguments manipulation hook.
|
||||
args = dispatch_hook('args', args['hooks'], args)
|
||||
# Create the Request.
|
||||
req = Request()
|
||||
req.method = method.upper()
|
||||
req.url = url
|
||||
req.headers = headers
|
||||
req.files = files
|
||||
req.data = data
|
||||
req.params = params
|
||||
req.auth = auth
|
||||
req.cookies = cookies
|
||||
req.hooks = hooks
|
||||
|
||||
# Create the (empty) response.
|
||||
r = Request(**args)
|
||||
# Prepare the Request.
|
||||
prep = req.prepare()
|
||||
|
||||
# Give the response some context.
|
||||
r.session = self
|
||||
|
||||
# Don't send if asked nicely.
|
||||
if not return_response:
|
||||
return r
|
||||
|
||||
# Send the HTTP Request.
|
||||
r.send(prefetch=prefetch)
|
||||
|
||||
# Send any cookies back up the to the session.
|
||||
self.cookies.update(r.response.cookies)
|
||||
|
||||
# Return the response.
|
||||
return r.response
|
||||
# Send the request.
|
||||
send_kwargs = {
|
||||
'stream': stream,
|
||||
'timeout': timeout,
|
||||
'verify': verify,
|
||||
'cert': cert,
|
||||
'proxies': proxies,
|
||||
'allow_redirects': allow_redirects,
|
||||
}
|
||||
resp = self.send(prep, **send_kwargs)
|
||||
|
||||
return resp
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
"""Sends a GET request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return self.request('get', url, **kwargs)
|
||||
|
||||
return self.request('GET', url, **kwargs)
|
||||
|
||||
def options(self, url, **kwargs):
|
||||
"""Sends a OPTIONS request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return self.request('options', url, **kwargs)
|
||||
|
||||
return self.request('OPTIONS', url, **kwargs)
|
||||
|
||||
def head(self, url, **kwargs):
|
||||
"""Sends a HEAD request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return self.request('head', url, **kwargs)
|
||||
|
||||
kwargs.setdefault('allow_redirects', False)
|
||||
return self.request('HEAD', url, **kwargs)
|
||||
|
||||
def post(self, url, data=None, **kwargs):
|
||||
"""Sends a POST request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return self.request('post', url, data=data, **kwargs)
|
||||
|
||||
return self.request('POST', url, data=data, **kwargs)
|
||||
|
||||
def put(self, url, data=None, **kwargs):
|
||||
"""Sends a PUT request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return self.request('put', url, data=data, **kwargs)
|
||||
|
||||
return self.request('PUT', url, data=data, **kwargs)
|
||||
|
||||
def patch(self, url, data=None, **kwargs):
|
||||
"""Sends a PATCH request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return self.request('patch', url, data=data, **kwargs)
|
||||
|
||||
return self.request('PATCH', url, data=data, **kwargs)
|
||||
|
||||
def delete(self, url, **kwargs):
|
||||
"""Sends a DELETE request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
:param \*\*kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return self.request('delete', url, **kwargs)
|
||||
return self.request('DELETE', url, **kwargs)
|
||||
|
||||
def send(self, request, **kwargs):
|
||||
"""Send a given PreparedRequest."""
|
||||
# Set defaults that the hooks can utilize to ensure they always have
|
||||
# the correct parameters to reproduce the previous request.
|
||||
kwargs.setdefault('stream', self.stream)
|
||||
kwargs.setdefault('verify', self.verify)
|
||||
kwargs.setdefault('cert', self.cert)
|
||||
kwargs.setdefault('proxies', self.proxies)
|
||||
|
||||
# It's possible that users might accidentally send a Request object.
|
||||
# Guard against that specific failure case.
|
||||
if getattr(request, 'prepare', None):
|
||||
raise ValueError('You can only send PreparedRequests.')
|
||||
|
||||
# Set up variables needed for resolve_redirects and dispatching of
|
||||
# hooks
|
||||
allow_redirects = kwargs.pop('allow_redirects', True)
|
||||
stream = kwargs.get('stream')
|
||||
timeout = kwargs.get('timeout')
|
||||
verify = kwargs.get('verify')
|
||||
cert = kwargs.get('cert')
|
||||
proxies = kwargs.get('proxies')
|
||||
hooks = request.hooks
|
||||
|
||||
# Get the appropriate adapter to use
|
||||
adapter = self.get_adapter(url=request.url)
|
||||
|
||||
# Start time (approximately) of the request
|
||||
start = datetime.utcnow()
|
||||
# Send the request
|
||||
r = adapter.send(request, **kwargs)
|
||||
# Total elapsed time of the request (approximately)
|
||||
r.elapsed = datetime.utcnow() - start
|
||||
|
||||
# Response manipulation hooks
|
||||
r = dispatch_hook('response', hooks, r, **kwargs)
|
||||
|
||||
# Persist cookies
|
||||
extract_cookies_to_jar(self.cookies, request, r.raw)
|
||||
|
||||
# Redirect resolving generator.
|
||||
gen = self.resolve_redirects(r, request, stream=stream,
|
||||
timeout=timeout, verify=verify, cert=cert,
|
||||
proxies=proxies)
|
||||
|
||||
# Resolve redirects if allowed.
|
||||
history = [resp for resp in gen] if allow_redirects else []
|
||||
|
||||
# Shuffle things around if there's history.
|
||||
if history:
|
||||
# Insert the first (original) request at the start
|
||||
history.insert(0, r)
|
||||
# Get the last request made
|
||||
r = history.pop()
|
||||
r.history = tuple(history)
|
||||
|
||||
return r
|
||||
|
||||
def get_adapter(self, url):
|
||||
"""Returns the appropriate connnection adapter for the given URL."""
|
||||
for (prefix, adapter) in self.adapters.items():
|
||||
|
||||
if url.startswith(prefix):
|
||||
return adapter
|
||||
|
||||
# Nothing matches :-/
|
||||
raise InvalidSchema("No connection adapters were found for '%s'" % url)
|
||||
|
||||
def close(self):
|
||||
"""Closes all adapters and as such the session"""
|
||||
for _, v in self.adapters.items():
|
||||
v.close()
|
||||
|
||||
def mount(self, prefix, adapter):
|
||||
"""Registers a connection adapter to a prefix.
|
||||
|
||||
Adapters are sorted in descending order by key length."""
|
||||
self.adapters[prefix] = adapter
|
||||
keys_to_move = [k for k in self.adapters if len(k) < len(prefix)]
|
||||
for key in keys_to_move:
|
||||
self.adapters[key] = self.adapters.pop(key)
|
||||
|
||||
def __getstate__(self):
|
||||
return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
|
||||
|
||||
def __setstate__(self, state):
|
||||
for attr, value in state.items():
|
||||
setattr(self, attr, value)
|
||||
|
||||
|
||||
def session(**kwargs):
|
||||
def session():
|
||||
"""Returns a :class:`Session` for context-management."""
|
||||
|
||||
return Session(**kwargs)
|
||||
return Session()
|
||||
|
||||
@@ -10,7 +10,7 @@ _codes = {
|
||||
102: ('processing',),
|
||||
103: ('checkpoint',),
|
||||
122: ('uri_too_long', 'request_uri_too_long'),
|
||||
200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/'),
|
||||
200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'),
|
||||
201: ('created',),
|
||||
202: ('accepted',),
|
||||
203: ('non_authoritative_info', 'non_authoritative_information'),
|
||||
@@ -62,10 +62,11 @@ _codes = {
|
||||
444: ('no_response', 'none'),
|
||||
449: ('retry_with', 'retry'),
|
||||
450: ('blocked_by_windows_parental_controls', 'parental_controls'),
|
||||
451: ('unavailable_for_legal_reasons', 'legal_reasons'),
|
||||
499: ('client_closed_request',),
|
||||
|
||||
# Server Error.
|
||||
500: ('internal_server_error', 'server_error', '/o\\'),
|
||||
500: ('internal_server_error', 'server_error', '/o\\', '✗'),
|
||||
501: ('not_implemented',),
|
||||
502: ('bad_gateway',),
|
||||
503: ('service_unavailable', 'unavailable'),
|
||||
@@ -83,4 +84,4 @@ for (code, titles) in list(_codes.items()):
|
||||
for title in titles:
|
||||
setattr(codes, title, code)
|
||||
if not title.startswith('\\'):
|
||||
setattr(codes, title.upper(), code)
|
||||
setattr(codes, title.upper(), code)
|
||||
|
||||
@@ -8,44 +8,106 @@ Data structures that power Requests.
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
import collections
|
||||
from itertools import islice
|
||||
|
||||
class CaseInsensitiveDict(dict):
|
||||
"""Case-insensitive Dictionary
|
||||
|
||||
class IteratorProxy(object):
|
||||
"""docstring for IteratorProxy"""
|
||||
def __init__(self, i):
|
||||
self.i = i
|
||||
# self.i = chain.from_iterable(i)
|
||||
|
||||
def __iter__(self):
|
||||
return self.i
|
||||
|
||||
def __len__(self):
|
||||
if hasattr(self.i, '__len__'):
|
||||
return len(self.i)
|
||||
if hasattr(self.i, 'len'):
|
||||
return self.i.len
|
||||
if hasattr(self.i, 'fileno'):
|
||||
return os.fstat(self.i.fileno()).st_size
|
||||
|
||||
def read(self, n):
|
||||
return "".join(islice(self.i, None, n))
|
||||
|
||||
|
||||
class CaseInsensitiveDict(collections.MutableMapping):
|
||||
"""
|
||||
A case-insensitive ``dict``-like object.
|
||||
|
||||
Implements all methods and operations of
|
||||
``collections.MutableMapping`` as well as dict's ``copy``. Also
|
||||
provides ``lower_items``.
|
||||
|
||||
All keys are expected to be strings. The structure remembers the
|
||||
case of the last key to be set, and ``iter(instance)``,
|
||||
``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
|
||||
will contain case-sensitive keys. However, querying and contains
|
||||
testing is case insensitive:
|
||||
|
||||
cid = CaseInsensitiveDict()
|
||||
cid['Accept'] = 'application/json'
|
||||
cid['aCCEPT'] == 'application/json' # True
|
||||
list(cid) == ['Accept'] # True
|
||||
|
||||
For example, ``headers['content-encoding']`` will return the
|
||||
value of a ``'Content-Encoding'`` response header."""
|
||||
value of a ``'Content-Encoding'`` response header, regardless
|
||||
of how the header name was originally stored.
|
||||
|
||||
@property
|
||||
def lower_keys(self):
|
||||
if not hasattr(self, '_lower_keys') or not self._lower_keys:
|
||||
self._lower_keys = dict((k.lower(), k) for k in list(self.keys()))
|
||||
return self._lower_keys
|
||||
If the constructor, ``.update``, or equality comparison
|
||||
operations are given keys that have equal ``.lower()``s, the
|
||||
behavior is undefined.
|
||||
|
||||
def _clear_lower_keys(self):
|
||||
if hasattr(self, '_lower_keys'):
|
||||
self._lower_keys.clear()
|
||||
"""
|
||||
def __init__(self, data=None, **kwargs):
|
||||
self._store = dict()
|
||||
if data is None:
|
||||
data = {}
|
||||
self.update(data, **kwargs)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
dict.__setitem__(self, key, value)
|
||||
self._clear_lower_keys()
|
||||
|
||||
def __delitem__(self, key):
|
||||
dict.__delitem__(self, key)
|
||||
self._lower_keys.clear()
|
||||
|
||||
def __contains__(self, key):
|
||||
return key.lower() in self.lower_keys
|
||||
# Use the lowercased key for lookups, but store the actual
|
||||
# key alongside the value.
|
||||
self._store[key.lower()] = (key, value)
|
||||
|
||||
def __getitem__(self, key):
|
||||
# We allow fall-through here, so values default to None
|
||||
if key in self:
|
||||
return dict.__getitem__(self, self.lower_keys[key.lower()])
|
||||
return self._store[key.lower()][1]
|
||||
|
||||
def get(self, key, default=None):
|
||||
if key in self:
|
||||
return self[key]
|
||||
def __delitem__(self, key):
|
||||
del self._store[key.lower()]
|
||||
|
||||
def __iter__(self):
|
||||
return (casedkey for casedkey, mappedvalue in self._store.values())
|
||||
|
||||
def __len__(self):
|
||||
return len(self._store)
|
||||
|
||||
def lower_items(self):
|
||||
"""Like iteritems(), but with all lowercase keys."""
|
||||
return (
|
||||
(lowerkey, keyval[1])
|
||||
for (lowerkey, keyval)
|
||||
in self._store.items()
|
||||
)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, collections.Mapping):
|
||||
other = CaseInsensitiveDict(other)
|
||||
else:
|
||||
return default
|
||||
return NotImplemented
|
||||
# Compare insensitively
|
||||
return dict(self.lower_items()) == dict(other.lower_items())
|
||||
|
||||
# Copy is required
|
||||
def copy(self):
|
||||
return CaseInsensitiveDict(self._store.values())
|
||||
|
||||
def __repr__(self):
|
||||
return '%s(%r)' % (self.__class__.__name__, dict(self.items()))
|
||||
|
||||
|
||||
class LookupDict(dict):
|
||||
"""Dictionary lookup object."""
|
||||
|
||||
@@ -11,34 +11,135 @@ that are also useful for external consumption.
|
||||
|
||||
import cgi
|
||||
import codecs
|
||||
import collections
|
||||
import os
|
||||
import random
|
||||
import platform
|
||||
import re
|
||||
import zlib
|
||||
import sys
|
||||
from netrc import netrc, NetrcParseError
|
||||
|
||||
from . import __version__
|
||||
from . import certs
|
||||
from .compat import parse_http_list as _parse_list_header
|
||||
from .compat import quote, unquote, cookielib, SimpleCookie, is_py2
|
||||
from .compat import basestring
|
||||
from .compat import quote, urlparse, bytes, str, OrderedDict, urlunparse
|
||||
from .cookies import RequestsCookieJar, cookiejar_from_dict
|
||||
from .structures import CaseInsensitiveDict
|
||||
|
||||
_hush_pyflakes = (RequestsCookieJar,)
|
||||
|
||||
NETRC_FILES = ('.netrc', '_netrc')
|
||||
|
||||
DEFAULT_CA_BUNDLE_PATH = certs.where()
|
||||
|
||||
|
||||
def dict_from_string(s):
|
||||
"""Returns a MultiDict with Cookies."""
|
||||
def dict_to_sequence(d):
|
||||
"""Returns an internal sequence dictionary update."""
|
||||
|
||||
cookies = dict()
|
||||
if hasattr(d, 'items'):
|
||||
d = d.items()
|
||||
|
||||
c = SimpleCookie()
|
||||
c.load(s)
|
||||
return d
|
||||
|
||||
for k,v in list(c.items()):
|
||||
cookies.update({k: v.value})
|
||||
|
||||
return cookies
|
||||
def super_len(o):
|
||||
if hasattr(o, '__len__'):
|
||||
return len(o)
|
||||
if hasattr(o, 'len'):
|
||||
return o.len
|
||||
if hasattr(o, 'fileno'):
|
||||
return os.fstat(o.fileno()).st_size
|
||||
|
||||
|
||||
def get_netrc_auth(url):
|
||||
"""Returns the Requests tuple auth for a given url from netrc."""
|
||||
|
||||
try:
|
||||
locations = (os.path.expanduser('~/{0}'.format(f)) for f in NETRC_FILES)
|
||||
netrc_path = None
|
||||
|
||||
for loc in locations:
|
||||
if os.path.exists(loc) and not netrc_path:
|
||||
netrc_path = loc
|
||||
|
||||
# Abort early if there isn't one.
|
||||
if netrc_path is None:
|
||||
return netrc_path
|
||||
|
||||
ri = urlparse(url)
|
||||
|
||||
# Strip port numbers from netloc
|
||||
host = ri.netloc.split(':')[0]
|
||||
|
||||
try:
|
||||
_netrc = netrc(netrc_path).authenticators(host)
|
||||
if _netrc:
|
||||
# Return with login / password
|
||||
login_i = (0 if _netrc[0] else 1)
|
||||
return (_netrc[login_i], _netrc[2])
|
||||
except (NetrcParseError, IOError):
|
||||
# If there was a parsing error or a permissions issue reading the file,
|
||||
# we'll just skip netrc auth
|
||||
pass
|
||||
|
||||
# AppEngine hackiness.
|
||||
except (ImportError, AttributeError):
|
||||
pass
|
||||
|
||||
|
||||
def guess_filename(obj):
|
||||
"""Tries to guess the filename of the given object."""
|
||||
name = getattr(obj, 'name', None)
|
||||
if name and name[0] != '<' and name[-1] != '>':
|
||||
return name
|
||||
return os.path.basename(name)
|
||||
|
||||
|
||||
def from_key_val_list(value):
|
||||
"""Take an object and test to see if it can be represented as a
|
||||
dictionary. Unless it can not be represented as such, return an
|
||||
OrderedDict, e.g.,
|
||||
|
||||
::
|
||||
|
||||
>>> from_key_val_list([('key', 'val')])
|
||||
OrderedDict([('key', 'val')])
|
||||
>>> from_key_val_list('string')
|
||||
ValueError: need more than 1 value to unpack
|
||||
>>> from_key_val_list({'key': 'val'})
|
||||
OrderedDict([('key', 'val')])
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
if isinstance(value, (str, bytes, bool, int)):
|
||||
raise ValueError('cannot encode objects that are not 2-tuples')
|
||||
|
||||
return OrderedDict(value)
|
||||
|
||||
|
||||
def to_key_val_list(value):
|
||||
"""Take an object and test to see if it can be represented as a
|
||||
dictionary. If it can be, return a list of tuples, e.g.,
|
||||
|
||||
::
|
||||
|
||||
>>> to_key_val_list([('key', 'val')])
|
||||
[('key', 'val')]
|
||||
>>> to_key_val_list({'key': 'val'})
|
||||
[('key', 'val')]
|
||||
>>> to_key_val_list('string')
|
||||
ValueError: cannot encode objects that are not 2-tuples.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
if isinstance(value, (str, bytes, bool, int)):
|
||||
raise ValueError('cannot encode objects that are not 2-tuples')
|
||||
|
||||
if isinstance(value, collections.Mapping):
|
||||
value = value.items()
|
||||
|
||||
return list(value)
|
||||
|
||||
|
||||
# From mitsuhiko/werkzeug (used with permission).
|
||||
def parse_list_header(value):
|
||||
@@ -130,62 +231,6 @@ def unquote_header_value(value, is_filename=False):
|
||||
return value
|
||||
|
||||
|
||||
def header_expand(headers):
|
||||
"""Returns an HTTP Header value string from a dictionary.
|
||||
|
||||
Example expansion::
|
||||
|
||||
{'text/x-dvi': {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}, 'text/x-c': {}}
|
||||
# Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
|
||||
|
||||
(('text/x-dvi', {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}), ('text/x-c', {}))
|
||||
# Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
|
||||
"""
|
||||
|
||||
collector = []
|
||||
|
||||
if isinstance(headers, dict):
|
||||
headers = list(headers.items())
|
||||
|
||||
elif isinstance(headers, basestring):
|
||||
return headers
|
||||
|
||||
for i, (value, params) in enumerate(headers):
|
||||
|
||||
_params = []
|
||||
|
||||
for (p_k, p_v) in list(params.items()):
|
||||
|
||||
_params.append('%s=%s' % (p_k, p_v))
|
||||
|
||||
collector.append(value)
|
||||
collector.append('; ')
|
||||
|
||||
if len(params):
|
||||
|
||||
collector.append('; '.join(_params))
|
||||
|
||||
if not len(headers) == i+1:
|
||||
collector.append(', ')
|
||||
|
||||
|
||||
# Remove trailing separators.
|
||||
if collector[-1] in (', ', '; '):
|
||||
del collector[-1]
|
||||
|
||||
return ''.join(collector)
|
||||
|
||||
|
||||
|
||||
def randombytes(n):
|
||||
"""Return n random bytes."""
|
||||
if is_py2:
|
||||
L = [chr(random.randrange(0, 256)) for i in range(n)]
|
||||
else:
|
||||
L = [chr(random.randrange(0, 256)).encode('utf-8') for i in range(n)]
|
||||
return b"".join(L)
|
||||
|
||||
|
||||
def dict_from_cookiejar(cj):
|
||||
"""Returns a key/value dictionary from a CookieJar.
|
||||
|
||||
@@ -194,33 +239,12 @@ def dict_from_cookiejar(cj):
|
||||
|
||||
cookie_dict = {}
|
||||
|
||||
for _, cookies in list(cj._cookies.items()):
|
||||
for _, cookies in list(cookies.items()):
|
||||
for cookie in list(cookies.values()):
|
||||
# print cookie
|
||||
cookie_dict[cookie.name] = cookie.value
|
||||
for cookie in cj:
|
||||
cookie_dict[cookie.name] = cookie.value
|
||||
|
||||
return cookie_dict
|
||||
|
||||
|
||||
def cookiejar_from_dict(cookie_dict):
|
||||
"""Returns a CookieJar from a key/value dictionary.
|
||||
|
||||
:param cookie_dict: Dict of key/values to insert into CookieJar.
|
||||
"""
|
||||
|
||||
# return cookiejar if one was passed in
|
||||
if isinstance(cookie_dict, cookielib.CookieJar):
|
||||
return cookie_dict
|
||||
|
||||
# create cookiejar
|
||||
cj = cookielib.CookieJar()
|
||||
|
||||
cj = add_dict_to_cookiejar(cj, cookie_dict)
|
||||
|
||||
return cj
|
||||
|
||||
|
||||
def add_dict_to_cookiejar(cj, cookie_dict):
|
||||
"""Returns a CookieJar from a key/value dictionary.
|
||||
|
||||
@@ -228,31 +252,8 @@ def add_dict_to_cookiejar(cj, cookie_dict):
|
||||
:param cookie_dict: Dict of key/values to insert into CookieJar.
|
||||
"""
|
||||
|
||||
for k, v in list(cookie_dict.items()):
|
||||
|
||||
cookie = cookielib.Cookie(
|
||||
version=0,
|
||||
name=k,
|
||||
value=v,
|
||||
port=None,
|
||||
port_specified=False,
|
||||
domain='',
|
||||
domain_specified=False,
|
||||
domain_initial_dot=False,
|
||||
path='/',
|
||||
path_specified=True,
|
||||
secure=False,
|
||||
expires=None,
|
||||
discard=True,
|
||||
comment=None,
|
||||
comment_url=None,
|
||||
rest={'HttpOnly': None},
|
||||
rfc2109=False
|
||||
)
|
||||
|
||||
# add cookie to cookiejar
|
||||
cj.set_cookie(cookie)
|
||||
|
||||
cj2 = cookiejar_from_dict(cookie_dict)
|
||||
cj.update(cj2)
|
||||
return cj
|
||||
|
||||
|
||||
@@ -287,23 +288,6 @@ def get_encoding_from_headers(headers):
|
||||
return 'ISO-8859-1'
|
||||
|
||||
|
||||
def unicode_from_html(content):
|
||||
"""Attempts to decode an HTML string into unicode.
|
||||
If unsuccessful, the original content is returned.
|
||||
"""
|
||||
|
||||
encodings = get_encodings_from_content(content)
|
||||
|
||||
for encoding in encodings:
|
||||
|
||||
try:
|
||||
return str(content, encoding)
|
||||
except (UnicodeError, TypeError):
|
||||
pass
|
||||
|
||||
return content
|
||||
|
||||
|
||||
def stream_decode_response_unicode(iterator, r):
|
||||
"""Stream decodes a iterator."""
|
||||
|
||||
@@ -322,6 +306,14 @@ def stream_decode_response_unicode(iterator, r):
|
||||
yield rv
|
||||
|
||||
|
||||
def iter_slices(string, slice_length):
|
||||
"""Iterate over slices of a string."""
|
||||
pos = 0
|
||||
while pos < len(string):
|
||||
yield string[pos:pos + slice_length]
|
||||
pos += slice_length
|
||||
|
||||
|
||||
def get_unicode_from_response(r):
|
||||
"""Returns the requested content back in unicode.
|
||||
|
||||
@@ -355,54 +347,204 @@ def get_unicode_from_response(r):
|
||||
return r.content
|
||||
|
||||
|
||||
def decode_gzip(content):
|
||||
"""Return gzip-decoded string.
|
||||
# The unreserved URI characters (RFC 3986)
|
||||
UNRESERVED_SET = frozenset(
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
+ "0123456789-._~")
|
||||
|
||||
:param content: bytestring to gzip-decode.
|
||||
|
||||
def unquote_unreserved(uri):
|
||||
"""Un-escape any percent-escape sequences in a URI that are unreserved
|
||||
characters. This leaves all reserved, illegal and non-ASCII bytes encoded.
|
||||
"""
|
||||
|
||||
return zlib.decompress(content, 16 + zlib.MAX_WBITS)
|
||||
parts = uri.split('%')
|
||||
for i in range(1, len(parts)):
|
||||
h = parts[i][0:2]
|
||||
if len(h) == 2 and h.isalnum():
|
||||
c = chr(int(h, 16))
|
||||
if c in UNRESERVED_SET:
|
||||
parts[i] = c + parts[i][2:]
|
||||
else:
|
||||
parts[i] = '%' + parts[i]
|
||||
else:
|
||||
parts[i] = '%' + parts[i]
|
||||
return ''.join(parts)
|
||||
|
||||
|
||||
def stream_decompress(iterator, mode='gzip'):
|
||||
"""
|
||||
Stream decodes an iterator over compressed data
|
||||
def requote_uri(uri):
|
||||
"""Re-quote the given URI.
|
||||
|
||||
:param iterator: An iterator over compressed data
|
||||
:param mode: 'gzip' or 'deflate'
|
||||
:return: An iterator over decompressed data
|
||||
"""
|
||||
|
||||
if mode not in ['gzip', 'deflate']:
|
||||
raise ValueError('stream_decompress mode must be gzip or deflate')
|
||||
|
||||
zlib_mode = 16 + zlib.MAX_WBITS if mode == 'gzip' else -zlib.MAX_WBITS
|
||||
dec = zlib.decompressobj(zlib_mode)
|
||||
try:
|
||||
for chunk in iterator:
|
||||
rv = dec.decompress(chunk)
|
||||
if rv:
|
||||
yield rv
|
||||
except zlib.error:
|
||||
# If there was an error decompressing, just return the raw chunk
|
||||
yield chunk
|
||||
# Continue to return the rest of the raw data
|
||||
for chunk in iterator:
|
||||
yield chunk
|
||||
else:
|
||||
# Make sure everything has been returned from the decompression object
|
||||
buf = dec.decompress('')
|
||||
rv = buf + dec.flush()
|
||||
if rv:
|
||||
yield rv
|
||||
|
||||
|
||||
def requote_path(path):
|
||||
"""Re-quote the given URL path component.
|
||||
|
||||
This function passes the given path through an unquote/quote cycle to
|
||||
This function passes the given URI through an unquote/quote cycle to
|
||||
ensure that it is fully and consistently quoted.
|
||||
"""
|
||||
parts = path.split(b"/")
|
||||
parts = (quote(unquote(part), safe=b"") for part in parts)
|
||||
return b"/".join(parts)
|
||||
# Unquote only the unreserved characters
|
||||
# Then quote only illegal characters (do not quote reserved, unreserved,
|
||||
# or '%')
|
||||
return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~")
|
||||
|
||||
|
||||
def get_environ_proxies(url):
|
||||
"""Return a dict of environment proxies."""
|
||||
|
||||
proxy_keys = [
|
||||
'all',
|
||||
'http',
|
||||
'https',
|
||||
'ftp',
|
||||
'socks'
|
||||
]
|
||||
|
||||
get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
|
||||
|
||||
# First check whether no_proxy is defined. If it is, check that the URL
|
||||
# we're getting isn't in the no_proxy list.
|
||||
no_proxy = get_proxy('no_proxy')
|
||||
|
||||
if no_proxy:
|
||||
# We need to check whether we match here. We need to see if we match
|
||||
# the end of the netloc, both with and without the port.
|
||||
no_proxy = no_proxy.split(',')
|
||||
netloc = urlparse(url).netloc
|
||||
|
||||
for host in no_proxy:
|
||||
if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
|
||||
# The URL does match something in no_proxy, so we don't want
|
||||
# to apply the proxies on this URL.
|
||||
return {}
|
||||
|
||||
# If we get here, we either didn't have no_proxy set or we're not going
|
||||
# anywhere that no_proxy applies to.
|
||||
proxies = [(key, get_proxy(key + '_proxy')) for key in proxy_keys]
|
||||
return dict([(key, val) for (key, val) in proxies if val])
|
||||
|
||||
|
||||
def default_user_agent():
|
||||
"""Return a string representing the default user agent."""
|
||||
_implementation = platform.python_implementation()
|
||||
|
||||
if _implementation == 'CPython':
|
||||
_implementation_version = platform.python_version()
|
||||
elif _implementation == 'PyPy':
|
||||
_implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major,
|
||||
sys.pypy_version_info.minor,
|
||||
sys.pypy_version_info.micro)
|
||||
if sys.pypy_version_info.releaselevel != 'final':
|
||||
_implementation_version = ''.join([_implementation_version, sys.pypy_version_info.releaselevel])
|
||||
elif _implementation == 'Jython':
|
||||
_implementation_version = platform.python_version() # Complete Guess
|
||||
elif _implementation == 'IronPython':
|
||||
_implementation_version = platform.python_version() # Complete Guess
|
||||
else:
|
||||
_implementation_version = 'Unknown'
|
||||
|
||||
try:
|
||||
p_system = platform.system()
|
||||
p_release = platform.release()
|
||||
except IOError:
|
||||
p_system = 'Unknown'
|
||||
p_release = 'Unknown'
|
||||
|
||||
return " ".join(['python-requests/%s' % __version__,
|
||||
'%s/%s' % (_implementation, _implementation_version),
|
||||
'%s/%s' % (p_system, p_release)])
|
||||
|
||||
|
||||
def default_headers():
|
||||
return CaseInsensitiveDict({
|
||||
'User-Agent': default_user_agent(),
|
||||
'Accept-Encoding': ', '.join(('gzip', 'deflate', 'compress')),
|
||||
'Accept': '*/*'
|
||||
})
|
||||
|
||||
|
||||
def parse_header_links(value):
|
||||
"""Return a dict of parsed link headers proxies.
|
||||
|
||||
i.e. Link: <http:/.../front.jpeg>; rel=front; type="image/jpeg",<http://.../back.jpeg>; rel=back;type="image/jpeg"
|
||||
|
||||
"""
|
||||
|
||||
links = []
|
||||
|
||||
replace_chars = " '\""
|
||||
|
||||
for val in value.split(","):
|
||||
try:
|
||||
url, params = val.split(";", 1)
|
||||
except ValueError:
|
||||
url, params = val, ''
|
||||
|
||||
link = {}
|
||||
|
||||
link["url"] = url.strip("<> '\"")
|
||||
|
||||
for param in params.split(";"):
|
||||
try:
|
||||
key, value = param.split("=")
|
||||
except ValueError:
|
||||
break
|
||||
|
||||
link[key.strip(replace_chars)] = value.strip(replace_chars)
|
||||
|
||||
links.append(link)
|
||||
|
||||
return links
|
||||
|
||||
|
||||
# Null bytes; no need to recreate these on each call to guess_json_utf
|
||||
_null = '\x00'.encode('ascii') # encoding to ASCII for Python 3
|
||||
_null2 = _null * 2
|
||||
_null3 = _null * 3
|
||||
|
||||
|
||||
def guess_json_utf(data):
|
||||
# JSON always starts with two ASCII characters, so detection is as
|
||||
# easy as counting the nulls and from their location and count
|
||||
# determine the encoding. Also detect a BOM, if present.
|
||||
sample = data[:4]
|
||||
if sample in (codecs.BOM_UTF32_LE, codecs.BOM32_BE):
|
||||
return 'utf-32' # BOM included
|
||||
if sample[:3] == codecs.BOM_UTF8:
|
||||
return 'utf-8-sig' # BOM included, MS style (discouraged)
|
||||
if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):
|
||||
return 'utf-16' # BOM included
|
||||
nullcount = sample.count(_null)
|
||||
if nullcount == 0:
|
||||
return 'utf-8'
|
||||
if nullcount == 2:
|
||||
if sample[::2] == _null2: # 1st and 3rd are null
|
||||
return 'utf-16-be'
|
||||
if sample[1::2] == _null2: # 2nd and 4th are null
|
||||
return 'utf-16-le'
|
||||
# Did not detect 2 valid UTF-16 ascii-range characters
|
||||
if nullcount == 3:
|
||||
if sample[:3] == _null3:
|
||||
return 'utf-32-be'
|
||||
if sample[1:] == _null3:
|
||||
return 'utf-32-le'
|
||||
# Did not detect a valid UTF-32 ascii-range character
|
||||
return None
|
||||
|
||||
|
||||
def prepend_scheme_if_needed(url, new_scheme):
|
||||
'''Given a URL that may or may not have a scheme, prepend the given scheme.
|
||||
Does not replace a present scheme with the one provided as an argument.'''
|
||||
scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme)
|
||||
|
||||
# urlparse is a finicky beast, and sometimes decides that there isn't a
|
||||
# netloc present. Assume that it's being over-cautious, and switch netloc
|
||||
# and path if urlparse decided there was no netloc.
|
||||
if not netloc:
|
||||
netloc, path = path, netloc
|
||||
|
||||
return urlunparse((scheme, netloc, path, params, query, fragment))
|
||||
|
||||
|
||||
def get_auth_from_url(url):
|
||||
"""Given a url with authentication components, extract them into a tuple of
|
||||
username,password."""
|
||||
if url:
|
||||
parsed = urlparse(url)
|
||||
return (parsed.username, parsed.password)
|
||||
else:
|
||||
return ('', '')
|
||||
|
||||
Reference in New Issue
Block a user