feat(ST2.UtilPackages): bump up all packages
- Refresh PackageCache with latest versions of everything
This commit is contained in:
Iristyle
@@ -0,0 +1,257 @@
|
||||
import re
|
||||
|
||||
from ..versions import version_sort, version_filter
|
||||
from .json_api_client import JSONApiClient
|
||||
|
||||
|
||||
# A predefined list of readme filenames to look for
|
||||
_readme_filenames = [
|
||||
'readme',
|
||||
'readme.txt',
|
||||
'readme.md',
|
||||
'readme.mkd',
|
||||
'readme.mdown',
|
||||
'readme.markdown',
|
||||
'readme.textile',
|
||||
'readme.creole',
|
||||
'readme.rst'
|
||||
]
|
||||
|
||||
|
||||
class BitBucketClient(JSONApiClient):
|
||||
|
||||
def download_info(self, url):
|
||||
"""
|
||||
Retrieve information about downloading a package
|
||||
|
||||
:param url:
|
||||
The URL of the repository, in one of the forms:
|
||||
https://bitbucket.org/{user}/{repo}
|
||||
https://bitbucket.org/{user}/{repo}/src/{branch}
|
||||
https://bitbucket.org/{user}/{repo}/#tags
|
||||
If the last option, grabs the info from the newest
|
||||
tag that is a valid semver version.
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
None if no match, False if no commit, or a dict with the following keys:
|
||||
`version` - the version number of the download
|
||||
`url` - the download URL of a zip file of the package
|
||||
`date` - the ISO-8601 timestamp string when the version was published
|
||||
"""
|
||||
|
||||
commit_info = self._commit_info(url)
|
||||
if not commit_info:
|
||||
return commit_info
|
||||
|
||||
return {
|
||||
'version': commit_info['version'],
|
||||
'url': 'https://bitbucket.org/%s/get/%s.zip' % (commit_info['user_repo'], commit_info['commit']),
|
||||
'date': commit_info['timestamp']
|
||||
}
|
||||
|
||||
def repo_info(self, url):
|
||||
"""
|
||||
Retrieve general information about a repository
|
||||
|
||||
:param url:
|
||||
The URL to the repository, in one of the forms:
|
||||
https://bitbucket.org/{user}/{repo}
|
||||
https://bitbucket.org/{user}/{repo}/src/{branch}
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
None if no match, or a dict with the following keys:
|
||||
`name`
|
||||
`description`
|
||||
`homepage` - URL of the homepage
|
||||
`author`
|
||||
`readme` - URL of the readme
|
||||
`issues` - URL of bug tracker
|
||||
`donate` - URL of a donate page
|
||||
"""
|
||||
|
||||
user_repo, branch = self._user_repo_branch(url)
|
||||
if not user_repo:
|
||||
return user_repo
|
||||
|
||||
api_url = self._make_api_url(user_repo)
|
||||
|
||||
info = self.fetch_json(api_url)
|
||||
|
||||
issues_url = u'https://bitbucket.org/%s/issues' % user_repo
|
||||
|
||||
return {
|
||||
'name': info['name'],
|
||||
'description': info['description'] or 'No description provided',
|
||||
'homepage': info['website'] or url,
|
||||
'author': info['owner'],
|
||||
'donate': u'https://www.gittip.com/on/bitbucket/%s/' % info['owner'],
|
||||
'readme': self._readme_url(user_repo, branch),
|
||||
'issues': issues_url if info['has_issues'] else None
|
||||
}
|
||||
|
||||
def _commit_info(self, url):
|
||||
"""
|
||||
Fetches info about the latest commit to a repository
|
||||
|
||||
:param url:
|
||||
The URL to the repository, in one of the forms:
|
||||
https://bitbucket.org/{user}/{repo}
|
||||
https://bitbucket.org/{user}/{repo}/src/{branch}
|
||||
https://bitbucket.org/{user}/{repo}/#tags
|
||||
If the last option, grabs the info from the newest
|
||||
tag that is a valid semver version.
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
None if no match, False if no commit, or a dict with the following keys:
|
||||
`user_repo` - the user/repo name
|
||||
`timestamp` - the ISO-8601 UTC timestamp string
|
||||
`commit` - the branch or tag name
|
||||
`version` - the extracted version number
|
||||
"""
|
||||
|
||||
tags_match = re.match('https?://bitbucket.org/([^/]+/[^#/]+)/?#tags$', url)
|
||||
|
||||
version = None
|
||||
|
||||
if tags_match:
|
||||
user_repo = tags_match.group(1)
|
||||
tags_url = self._make_api_url(user_repo, '/tags')
|
||||
tags_list = self.fetch_json(tags_url)
|
||||
tags = version_filter(tags_list.keys(), self.settings.get('install_prereleases'))
|
||||
tags = version_sort(tags, reverse=True)
|
||||
if not tags:
|
||||
return False
|
||||
commit = tags[0]
|
||||
version = re.sub('^v', '', commit)
|
||||
|
||||
else:
|
||||
user_repo, commit = self._user_repo_branch(url)
|
||||
if not user_repo:
|
||||
return user_repo
|
||||
|
||||
changeset_url = self._make_api_url(user_repo, '/changesets/%s' % commit)
|
||||
commit_info = self.fetch_json(changeset_url)
|
||||
|
||||
commit_date = commit_info['timestamp'][0:19]
|
||||
|
||||
if not version:
|
||||
version = re.sub('[\-: ]', '.', commit_date)
|
||||
|
||||
return {
|
||||
'user_repo': user_repo,
|
||||
'timestamp': commit_date,
|
||||
'commit': commit,
|
||||
'version': version
|
||||
}
|
||||
|
||||
def _main_branch_name(self, user_repo):
|
||||
"""
|
||||
Fetch the name of the default branch
|
||||
|
||||
:param user_repo:
|
||||
The user/repo name to get the main branch for
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
The name of the main branch - `master` or `default`
|
||||
"""
|
||||
|
||||
main_branch_url = self._make_api_url(user_repo, '/main-branch')
|
||||
main_branch_info = self.fetch_json(main_branch_url, True)
|
||||
return main_branch_info['name']
|
||||
|
||||
def _make_api_url(self, user_repo, suffix=''):
|
||||
"""
|
||||
Generate a URL for the BitBucket API
|
||||
|
||||
:param user_repo:
|
||||
The user/repo of the repository
|
||||
|
||||
:param suffix:
|
||||
The extra API path info to add to the URL
|
||||
|
||||
:return:
|
||||
The API URL
|
||||
"""
|
||||
|
||||
return 'https://api.bitbucket.org/1.0/repositories/%s%s' % (user_repo, suffix)
|
||||
|
||||
def _readme_url(self, user_repo, branch, prefer_cached=False):
|
||||
"""
|
||||
Parse the root directory listing for the repo and return the URL
|
||||
to any file that looks like a readme
|
||||
|
||||
:param user_repo:
|
||||
The user/repo string
|
||||
|
||||
:param branch:
|
||||
The branch to fetch the readme from
|
||||
|
||||
:param prefer_cached:
|
||||
If a cached directory listing should be used instead of a new HTTP request
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
The URL to the readme file, or None
|
||||
"""
|
||||
|
||||
listing_url = self._make_api_url(user_repo, '/src/%s/' % branch)
|
||||
root_dir_info = self.fetch_json(listing_url, prefer_cached)
|
||||
|
||||
for entry in root_dir_info['files']:
|
||||
if entry['path'].lower() in _readme_filenames:
|
||||
return 'https://bitbucket.org/%s/raw/%s/%s' % (user_repo,
|
||||
branch, entry['path'])
|
||||
|
||||
return None
|
||||
|
||||
def _user_repo_branch(self, url):
|
||||
"""
|
||||
Extract the username/repo and branch name from the URL
|
||||
|
||||
:param url:
|
||||
The URL to extract the info from, in one of the forms:
|
||||
https://bitbucket.org/{user}/{repo}
|
||||
https://bitbucket.org/{user}/{repo}/src/{branch}
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
A tuple of (user/repo, branch name) or (None, None) if not matching
|
||||
"""
|
||||
|
||||
repo_match = re.match('https?://bitbucket.org/([^/]+/[^/]+)/?$', url)
|
||||
branch_match = re.match('https?://bitbucket.org/([^/]+/[^/]+)/src/([^/]+)/?$', url)
|
||||
|
||||
if repo_match:
|
||||
user_repo = repo_match.group(1)
|
||||
branch = self._main_branch_name(user_repo)
|
||||
|
||||
elif branch_match:
|
||||
user_repo = branch_match.group(1)
|
||||
branch = branch_match.group(2)
|
||||
|
||||
else:
|
||||
return (None, None)
|
||||
|
||||
return (user_repo, branch)
|
||||
@@ -0,0 +1,5 @@
|
||||
class ClientException(Exception):
|
||||
"""If a client could not fetch information"""
|
||||
|
||||
def __str__(self):
|
||||
return self.args[0]
|
||||
@@ -0,0 +1,292 @@
|
||||
import re
|
||||
|
||||
try:
|
||||
# Python 3
|
||||
from urllib.parse import urlencode, quote
|
||||
except (ImportError):
|
||||
# Python 2
|
||||
from urllib import urlencode, quote
|
||||
|
||||
from ..versions import version_sort, version_filter
|
||||
from .json_api_client import JSONApiClient
|
||||
from ..downloaders.downloader_exception import DownloaderException
|
||||
|
||||
|
||||
class GitHubClient(JSONApiClient):
|
||||
|
||||
def download_info(self, url):
|
||||
"""
|
||||
Retrieve information about downloading a package
|
||||
|
||||
:param url:
|
||||
The URL of the repository, in one of the forms:
|
||||
https://github.com/{user}/{repo}
|
||||
https://github.com/{user}/{repo}/tree/{branch}
|
||||
https://github.com/{user}/{repo}/tags
|
||||
If the last option, grabs the info from the newest
|
||||
tag that is a valid semver version.
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
None if no match, False if no commit, or a dict with the following keys:
|
||||
`version` - the version number of the download
|
||||
`url` - the download URL of a zip file of the package
|
||||
`date` - the ISO-8601 timestamp string when the version was published
|
||||
"""
|
||||
|
||||
commit_info = self._commit_info(url)
|
||||
if not commit_info:
|
||||
return commit_info
|
||||
|
||||
return {
|
||||
'version': commit_info['version'],
|
||||
# We specifically use codeload.github.com here because the download
|
||||
# URLs all redirect there, and some of the downloaders don't follow
|
||||
# HTTP redirect headers
|
||||
'url': 'https://codeload.github.com/%s/zip/%s' % (commit_info['user_repo'], quote(commit_info['commit'])),
|
||||
'date': commit_info['timestamp']
|
||||
}
|
||||
|
||||
def repo_info(self, url):
|
||||
"""
|
||||
Retrieve general information about a repository
|
||||
|
||||
:param url:
|
||||
The URL to the repository, in one of the forms:
|
||||
https://github.com/{user}/{repo}
|
||||
https://github.com/{user}/{repo}/tree/{branch}
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
None if no match, or a dict with the following keys:
|
||||
`name`
|
||||
`description`
|
||||
`homepage` - URL of the homepage
|
||||
`author`
|
||||
`readme` - URL of the readme
|
||||
`issues` - URL of bug tracker
|
||||
`donate` - URL of a donate page
|
||||
"""
|
||||
|
||||
user_repo, branch = self._user_repo_branch(url)
|
||||
if not user_repo:
|
||||
return user_repo
|
||||
|
||||
api_url = self._make_api_url(user_repo)
|
||||
|
||||
info = self.fetch_json(api_url)
|
||||
|
||||
output = self._extract_repo_info(info)
|
||||
output['readme'] = None
|
||||
|
||||
readme_info = self._readme_info(user_repo, branch)
|
||||
if not readme_info:
|
||||
return output
|
||||
|
||||
output['readme'] = 'https://raw.github.com/%s/%s/%s' % (user_repo,
|
||||
branch, readme_info['path'])
|
||||
return output
|
||||
|
||||
def user_info(self, url):
|
||||
"""
|
||||
Retrieve general information about all repositories that are
|
||||
part of a user/organization.
|
||||
|
||||
:param url:
|
||||
The URL to the user/organization, in the following form:
|
||||
https://github.com/{user}
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
None if no match, or am list of dicts with the following keys:
|
||||
`name`
|
||||
`description`
|
||||
`homepage` - URL of the homepage
|
||||
`author`
|
||||
`readme` - URL of the readme
|
||||
`issues` - URL of bug tracker
|
||||
`donate` - URL of a donate page
|
||||
"""
|
||||
|
||||
user_match = re.match('https?://github.com/([^/]+)/?$', url)
|
||||
if user_match == None:
|
||||
return None
|
||||
|
||||
user = user_match.group(1)
|
||||
api_url = self._make_api_url(user)
|
||||
|
||||
repos_info = self.fetch_json(api_url)
|
||||
|
||||
output = []
|
||||
for info in repos_info:
|
||||
output.append(self._extract_repo_info(info))
|
||||
return output
|
||||
|
||||
def _commit_info(self, url):
|
||||
"""
|
||||
Fetches info about the latest commit to a repository
|
||||
|
||||
:param url:
|
||||
The URL to the repository, in one of the forms:
|
||||
https://github.com/{user}/{repo}
|
||||
https://github.com/{user}/{repo}/tree/{branch}
|
||||
https://github.com/{user}/{repo}/tags
|
||||
If the last option, grabs the info from the newest
|
||||
tag that is a valid semver version.
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
None if no match, False is no commit, or a dict with the following keys:
|
||||
`user_repo` - the user/repo name
|
||||
`timestamp` - the ISO-8601 UTC timestamp string
|
||||
`commit` - the branch or tag name
|
||||
`version` - the extracted version number
|
||||
"""
|
||||
|
||||
tags_match = re.match('https?://github.com/([^/]+/[^/]+)/tags/?$', url)
|
||||
|
||||
version = None
|
||||
|
||||
if tags_match:
|
||||
user_repo = tags_match.group(1)
|
||||
tags_url = self._make_api_url(user_repo, '/tags')
|
||||
tags_list = self.fetch_json(tags_url)
|
||||
tags = [tag['name'] for tag in tags_list]
|
||||
tags = version_filter(tags, self.settings.get('install_prereleases'))
|
||||
tags = version_sort(tags, reverse=True)
|
||||
if not tags:
|
||||
return False
|
||||
commit = tags[0]
|
||||
version = re.sub('^v', '', commit)
|
||||
|
||||
else:
|
||||
user_repo, commit = self._user_repo_branch(url)
|
||||
if not user_repo:
|
||||
return user_repo
|
||||
|
||||
query_string = urlencode({'sha': commit, 'per_page': 1})
|
||||
commit_url = self._make_api_url(user_repo, '/commits?%s' % query_string)
|
||||
commit_info = self.fetch_json(commit_url)
|
||||
|
||||
commit_date = commit_info[0]['commit']['committer']['date'][0:19].replace('T', ' ')
|
||||
|
||||
if not version:
|
||||
version = re.sub('[\-: ]', '.', commit_date)
|
||||
|
||||
return {
|
||||
'user_repo': user_repo,
|
||||
'timestamp': commit_date,
|
||||
'commit': commit,
|
||||
'version': version
|
||||
}
|
||||
|
||||
def _extract_repo_info(self, result):
|
||||
"""
|
||||
Extracts information about a repository from the API result
|
||||
|
||||
:param result:
|
||||
A dict representing the data returned from the GitHub API
|
||||
|
||||
:return:
|
||||
A dict with the following keys:
|
||||
`name`
|
||||
`description`
|
||||
`homepage` - URL of the homepage
|
||||
`author`
|
||||
`issues` - URL of bug tracker
|
||||
`donate` - URL of a donate page
|
||||
"""
|
||||
|
||||
issues_url = u'https://github.com/%s/%s/issues' % (result['owner']['login'], result['name'])
|
||||
|
||||
return {
|
||||
'name': result['name'],
|
||||
'description': result['description'] or 'No description provided',
|
||||
'homepage': result['homepage'] or result['html_url'],
|
||||
'author': result['owner']['login'],
|
||||
'issues': issues_url if result['has_issues'] else None,
|
||||
'donate': u'https://www.gittip.com/on/github/%s/' % result['owner']['login']
|
||||
}
|
||||
|
||||
def _make_api_url(self, user_repo, suffix=''):
|
||||
"""
|
||||
Generate a URL for the BitBucket API
|
||||
|
||||
:param user_repo:
|
||||
The user/repo of the repository
|
||||
|
||||
:param suffix:
|
||||
The extra API path info to add to the URL
|
||||
|
||||
:return:
|
||||
The API URL
|
||||
"""
|
||||
|
||||
return 'https://api.github.com/repos/%s%s' % (user_repo, suffix)
|
||||
|
||||
def _readme_info(self, user_repo, branch, prefer_cached=False):
|
||||
"""
|
||||
Fetches the raw GitHub API information about a readme
|
||||
|
||||
:param user_repo:
|
||||
The user/repo of the repository
|
||||
|
||||
:param branch:
|
||||
The branch to pull the readme from
|
||||
|
||||
:param prefer_cached:
|
||||
If a cached version of the info should be returned instead of making a new HTTP request
|
||||
|
||||
:raises:
|
||||
DownloaderException: when there is an error downloading
|
||||
ClientException: when there is an error parsing the response
|
||||
|
||||
:return:
|
||||
A dict containing all of the info from the GitHub API, or None if no readme exists
|
||||
"""
|
||||
|
||||
query_string = urlencode({'ref': branch})
|
||||
readme_url = self._make_api_url(user_repo, '/readme?%s' % query_string)
|
||||
try:
|
||||
return self.fetch_json(readme_url, prefer_cached)
|
||||
except (DownloaderException) as e:
|
||||
if str(e).find('HTTP error 404') != -1:
|
||||
return None
|
||||
raise
|
||||
|
||||
def _user_repo_branch(self, url):
|
||||
"""
|
||||
Extract the username/repo and branch name from the URL
|
||||
|
||||
:param url:
|
||||
The URL to extract the info from, in one of the forms:
|
||||
https://github.com/{user}/{repo}
|
||||
https://github.com/{user}/{repo}/tree/{branch}
|
||||
|
||||
:return:
|
||||
A tuple of (user/repo, branch name) or (None, None) if no match
|
||||
"""
|
||||
|
||||
branch = 'master'
|
||||
branch_match = re.match('https?://github.com/[^/]+/[^/]+/tree/([^/]+)/?$', url)
|
||||
if branch_match != None:
|
||||
branch = branch_match.group(1)
|
||||
|
||||
repo_match = re.match('https?://github.com/([^/]+/[^/]+)($|/.*$)', url)
|
||||
if repo_match == None:
|
||||
return (None, None)
|
||||
|
||||
user_repo = repo_match.group(1)
|
||||
return (user_repo, branch)
|
||||
@@ -0,0 +1,64 @@
|
||||
import json
|
||||
|
||||
try:
|
||||
# Python 3
|
||||
from urllib.parse import urlencode, urlparse
|
||||
except (ImportError):
|
||||
# Python 2
|
||||
from urllib import urlencode
|
||||
from urlparse import urlparse
|
||||
|
||||
from .client_exception import ClientException
|
||||
from ..download_manager import downloader
|
||||
|
||||
|
||||
class JSONApiClient():
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
|
||||
def fetch(self, url, prefer_cached=False):
|
||||
"""
|
||||
Retrieves the contents of a URL
|
||||
|
||||
:param url:
|
||||
The URL to download the content from
|
||||
|
||||
:param prefer_cached:
|
||||
If a cached copy of the content is preferred
|
||||
|
||||
:return: The bytes/string
|
||||
"""
|
||||
|
||||
# If there are extra params for the domain name, add them
|
||||
extra_params = self.settings.get('query_string_params')
|
||||
domain_name = urlparse(url).netloc
|
||||
if extra_params and domain_name in extra_params:
|
||||
params = urlencode(extra_params[domain_name])
|
||||
joiner = '?%s' if url.find('?') == -1 else '&%s'
|
||||
url += joiner % params
|
||||
|
||||
with downloader(url, self.settings) as manager:
|
||||
content = manager.fetch(url, 'Error downloading repository.',
|
||||
prefer_cached)
|
||||
return content
|
||||
|
||||
def fetch_json(self, url, prefer_cached=False):
|
||||
"""
|
||||
Retrieves and parses the JSON from a URL
|
||||
|
||||
:param url:
|
||||
The URL to download the JSON from
|
||||
|
||||
:param prefer_cached:
|
||||
If a cached copy of the JSON is preferred
|
||||
|
||||
:return: A dict or list from the JSON
|
||||
"""
|
||||
|
||||
repository_json = self.fetch(url, prefer_cached)
|
||||
|
||||
try:
|
||||
return json.loads(repository_json.decode('utf-8'))
|
||||
except (ValueError):
|
||||
error_string = u'Error parsing JSON from URL %s.' % url
|
||||
raise ClientException(error_string)
|
||||
@@ -0,0 +1,83 @@
|
||||
import re
|
||||
import os
|
||||
import base64
|
||||
|
||||
try:
|
||||
# Python 3
|
||||
from urllib.parse import urlencode
|
||||
except (ImportError):
|
||||
# Python 2
|
||||
from urllib import urlencode
|
||||
|
||||
from .json_api_client import JSONApiClient
|
||||
from ..downloaders.downloader_exception import DownloaderException
|
||||
|
||||
|
||||
# Used to map file extensions to formats
|
||||
_readme_formats = {
|
||||
'.md': 'markdown',
|
||||
'.mkd': 'markdown',
|
||||
'.mdown': 'markdown',
|
||||
'.markdown': 'markdown',
|
||||
'.textile': 'textile',
|
||||
'.creole': 'creole',
|
||||
'.rst': 'rst'
|
||||
}
|
||||
|
||||
|
||||
class ReadmeClient(JSONApiClient):
|
||||
|
||||
def readme_info(self, url):
|
||||
"""
|
||||
Retrieve the readme and info about it
|
||||
|
||||
:param url:
|
||||
The URL of the readme file
|
||||
|
||||
:raises:
|
||||
DownloaderException: if there is an error downloading the readme
|
||||
ClientException: if there is an error parsing the response
|
||||
|
||||
:return:
|
||||
A dict with the following keys:
|
||||
`filename`
|
||||
`format` - `markdown`, `textile`, `creole`, `rst` or `txt`
|
||||
`contents` - contents of the readme as str/unicode
|
||||
"""
|
||||
|
||||
contents = None
|
||||
|
||||
# Try to grab the contents of a GitHub-based readme by grabbing the cached
|
||||
# content of the readme API call
|
||||
github_match = re.match('https://raw.github.com/([^/]+/[^/]+)/([^/]+)/readme(\.(md|mkd|mdown|markdown|textile|creole|rst|txt))?$', url, re.I)
|
||||
if github_match:
|
||||
user_repo = github_match.group(1)
|
||||
branch = github_match.group(2)
|
||||
|
||||
query_string = urlencode({'ref': branch})
|
||||
readme_json_url = 'https://api.github.com/repos/%s/readme?%s' % (user_repo, query_string)
|
||||
try:
|
||||
info = self.fetch_json(readme_json_url, prefer_cached=True)
|
||||
contents = base64.b64decode(info['content'])
|
||||
except (ValueError) as e:
|
||||
pass
|
||||
|
||||
if not contents:
|
||||
contents = self.fetch(url)
|
||||
|
||||
basename, ext = os.path.splitext(url)
|
||||
format = 'txt'
|
||||
ext = ext.lower()
|
||||
if ext in _readme_formats:
|
||||
format = _readme_formats[ext]
|
||||
|
||||
try:
|
||||
contents = contents.decode('utf-8')
|
||||
except (UnicodeDecodeError) as e:
|
||||
contents = contents.decode('cp1252', errors='replace')
|
||||
|
||||
return {
|
||||
'filename': os.path.basename(url),
|
||||
'format': format,
|
||||
'contents': contents
|
||||
}
|
||||
Reference in New Issue
Block a user