feat(SublimeText2.WebPackages): cache packages
This commit is contained in:
Iristyle
@@ -0,0 +1,591 @@
|
||||
# coding=utf-8
|
||||
import os
|
||||
import os.path
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import subprocess
|
||||
import tempfile
|
||||
import collections
|
||||
import platform
|
||||
import semver
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
is_python3 = sys.version_info[0] > 2
|
||||
|
||||
if is_python3:
|
||||
import urllib.request as url_req
|
||||
import urllib.error as url_err
|
||||
import urllib.parse as url_parse
|
||||
else:
|
||||
import urllib
|
||||
import urllib2
|
||||
url_req = urllib2
|
||||
url_err = urllib2
|
||||
url_parse = urllib2
|
||||
|
||||
CHECK_INTERVAL = 60 * 60 * 24
|
||||
|
||||
# PACKAGES_URL = 'https://api.github.com/repos/emmetio/pyv8-binaries/downloads'
|
||||
PACKAGES_URL = 'https://api.github.com/repos/emmetio/pyv8-binaries/contents'
|
||||
|
||||
def load(dest_path, delegate=None):
|
||||
"""
|
||||
Main function that attempts to load or update PyV8 binary.
|
||||
First, it loads list of available PyV8 modules and check if
|
||||
PyV8 should be downloaded or updated.
|
||||
@param dest_path: Path where PyV8 lib should be downloaded
|
||||
@param delegate: instance of LoaderDelegate that will receive
|
||||
loader progress events
|
||||
@returns: `True` if download progress was initiated
|
||||
"""
|
||||
if delegate is None:
|
||||
delegate = LoaderDelegate()
|
||||
|
||||
config = get_loader_config(dest_path)
|
||||
|
||||
if 'PyV8' in sys.modules and (config['skip_update'] or time.time() < config['last_update'] + CHECK_INTERVAL):
|
||||
# No need to load anything: user already has PyV8 binary
|
||||
# or decided to disable update process
|
||||
delegate.log('No need to update PyV8')
|
||||
return False
|
||||
|
||||
def on_complete(result, *args, **kwargs):
|
||||
if result is not None:
|
||||
# Most recent version was downloaded
|
||||
config['last_id'] = result
|
||||
if 'PyV8' not in sys.modules:
|
||||
# PyV8 is not loaded yet, we can safely unpack it
|
||||
unpack_pyv8(dest_path)
|
||||
|
||||
config['last_update'] = time.time()
|
||||
save_loader_config(dest_path, config)
|
||||
delegate.on_complete(*args, **kwargs)
|
||||
|
||||
# try to download most recent version of PyV8
|
||||
# As PyV8 for Sublime Text spreads the world, it's possible
|
||||
# that multiple distinct PyV8Loader's may start doing the same
|
||||
# job at the same time. In this case, we should check if there's
|
||||
# already a thread that load PyV8 and hook on existing thread
|
||||
# rather that creating a new one
|
||||
thread = None
|
||||
thread_exists = False
|
||||
for t in threading.enumerate():
|
||||
if hasattr(t, 'is_pyv8_thread'):
|
||||
print('PyV8: Reusing thread')
|
||||
thread = t
|
||||
thread_exists = True
|
||||
break
|
||||
|
||||
if not thread:
|
||||
print('PyV8: Creating new thread')
|
||||
thread = PyV8Loader(get_arch(), dest_path, config, delegate=delegate)
|
||||
thread.start()
|
||||
|
||||
delegate.on_start()
|
||||
|
||||
# watch on download progress
|
||||
prog = ThreadProgress(thread, delegate, thread_exists)
|
||||
prog.on('complete', on_complete if not thread_exists else delegate.on_complete)
|
||||
prog.on('error', delegate.on_error)
|
||||
|
||||
def get_arch():
|
||||
"Returns architecture name for PyV8 binary"
|
||||
suffix = is_python3 and '-p3' or ''
|
||||
p = lambda a: '%s%s' % (a, suffix)
|
||||
is_64bit = sys.maxsize > 2**32
|
||||
system_name = platform.system()
|
||||
if system_name == 'Darwin':
|
||||
if semver.match(platform.mac_ver()[0], '<10.7.0'):
|
||||
return p('mac106')
|
||||
|
||||
return p('osx')
|
||||
if system_name == 'Windows':
|
||||
return p('win64') if is_64bit else p('win32')
|
||||
if system_name == 'Linux':
|
||||
return p('linux64') if is_64bit else p('linux32')
|
||||
|
||||
def get_loader_config(path):
|
||||
config = {
|
||||
"last_id": 0,
|
||||
"last_update": 0,
|
||||
"skip_update": False
|
||||
}
|
||||
|
||||
config_path = os.path.join(path, 'config.json')
|
||||
if os.path.exists(config_path):
|
||||
with open(config_path) as fd:
|
||||
for k,v in json.load(fd).items():
|
||||
config[k] = v
|
||||
|
||||
return config
|
||||
|
||||
def save_loader_config(path, data):
|
||||
config_path = os.path.join(path, 'config.json')
|
||||
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
fp = open(config_path, 'w')
|
||||
fp.write(json.dumps(data))
|
||||
fp.close()
|
||||
|
||||
def clean_old_data():
|
||||
for f in os.listdir('.'):
|
||||
if f.lower() != 'config.json' and f.lower() != 'pack.zip':
|
||||
try:
|
||||
os.remove(f)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
def unpack_pyv8(package_dir):
|
||||
f = os.path.join(package_dir, 'pack.zip')
|
||||
if not os.path.exists(f):
|
||||
return
|
||||
|
||||
package_zip = zipfile.ZipFile(f, 'r')
|
||||
|
||||
root_level_paths = []
|
||||
last_path = None
|
||||
for path in package_zip.namelist():
|
||||
last_path = path
|
||||
if path.find('/') in [len(path) - 1, -1]:
|
||||
root_level_paths.append(path)
|
||||
if path[0] == '/' or path.find('../') != -1 or path.find('..\\') != -1:
|
||||
raise 'The PyV8 package contains files outside of the package dir and cannot be safely installed.'
|
||||
|
||||
if last_path and len(root_level_paths) == 0:
|
||||
root_level_paths.append(last_path[0:last_path.find('/') + 1])
|
||||
|
||||
prev_dir = os.getcwd()
|
||||
os.chdir(package_dir)
|
||||
|
||||
clean_old_data()
|
||||
|
||||
# Here we don't use .extractall() since it was having issues on OS X
|
||||
skip_root_dir = len(root_level_paths) == 1 and \
|
||||
root_level_paths[0].endswith('/')
|
||||
extracted_paths = []
|
||||
for path in package_zip.namelist():
|
||||
dest = path
|
||||
|
||||
if not is_python3:
|
||||
try:
|
||||
if not isinstance(dest, unicode):
|
||||
dest = unicode(dest, 'utf-8', 'strict')
|
||||
except UnicodeDecodeError:
|
||||
dest = unicode(dest, 'cp1252', 'replace')
|
||||
|
||||
if os.name == 'nt':
|
||||
regex = ':|\*|\?|"|<|>|\|'
|
||||
if re.search(regex, dest) != None:
|
||||
print ('%s: Skipping file from package named %s due to ' +
|
||||
'an invalid filename') % (__name__, path)
|
||||
continue
|
||||
|
||||
# If there was only a single directory in the package, we remove
|
||||
# that folder name from the paths as we extract entries
|
||||
if skip_root_dir:
|
||||
dest = dest[len(root_level_paths[0]):]
|
||||
|
||||
if os.name == 'nt':
|
||||
dest = dest.replace('/', '\\')
|
||||
else:
|
||||
dest = dest.replace('\\', '/')
|
||||
|
||||
dest = os.path.join(package_dir, dest)
|
||||
|
||||
def add_extracted_dirs(dir):
|
||||
while dir not in extracted_paths:
|
||||
extracted_paths.append(dir)
|
||||
dir = os.path.dirname(dir)
|
||||
if dir == package_dir:
|
||||
break
|
||||
|
||||
if path.endswith('/'):
|
||||
if not os.path.exists(dest):
|
||||
os.makedirs(dest)
|
||||
add_extracted_dirs(dest)
|
||||
else:
|
||||
dest_dir = os.path.dirname(dest)
|
||||
if not os.path.exists(dest_dir):
|
||||
os.makedirs(dest_dir)
|
||||
add_extracted_dirs(dest_dir)
|
||||
extracted_paths.append(dest)
|
||||
try:
|
||||
open(dest, 'wb').write(package_zip.read(path))
|
||||
except (IOError, UnicodeDecodeError):
|
||||
print ('%s: Skipping file from package named %s due to ' +
|
||||
'an invalid filename') % (__name__, path)
|
||||
package_zip.close()
|
||||
|
||||
os.chdir(prev_dir)
|
||||
os.remove(f)
|
||||
|
||||
class LoaderDelegate():
|
||||
"""
|
||||
Abstract class used to display PyV8 binary download progress,
|
||||
and provide some settings for downloader
|
||||
"""
|
||||
def __init__(self, settings={}):
|
||||
self.settings = settings
|
||||
|
||||
def on_start(self, *args, **kwargs):
|
||||
"Invoked when download process is initiated"
|
||||
pass
|
||||
|
||||
def on_progress(self, *args, **kwargs):
|
||||
"Invoked on download progress"
|
||||
pass
|
||||
|
||||
def on_complete(self, *args, **kwargs):
|
||||
"Invoked when download process was finished successfully"
|
||||
pass
|
||||
|
||||
def on_error(self, *args, **kwargs):
|
||||
"Invoked when error occured during download process"
|
||||
pass
|
||||
|
||||
def setting(self, name, default=None):
|
||||
"Returns specified setting name"
|
||||
return self.settings[name] if name in self.settings else default
|
||||
|
||||
def log(self, message):
|
||||
pass
|
||||
|
||||
class ThreadProgress():
|
||||
def __init__(self, thread, delegate, is_background=False):
|
||||
self.thread = thread
|
||||
self.delegate = delegate
|
||||
self.is_background = is_background
|
||||
self._callbacks = {}
|
||||
threading.Timer(0, self.run).start()
|
||||
|
||||
def run(self):
|
||||
if not self.thread.is_alive():
|
||||
if self.thread.exit_code != 0:
|
||||
return self.trigger('error', exit_code=self.thread.exit_code, progress=self)
|
||||
|
||||
return self.trigger('complete', result=self.thread.result, progress=self)
|
||||
|
||||
self.trigger('progress', progress=self)
|
||||
threading.Timer(0.1, self.run).start()
|
||||
|
||||
def on(self, event_name, callback):
|
||||
if event_name not in self._callbacks:
|
||||
self._callbacks[event_name] = []
|
||||
|
||||
if isinstance(callback, collections.Callable):
|
||||
self._callbacks[event_name].append(callback)
|
||||
|
||||
return self
|
||||
|
||||
def trigger(self, event_name, *args, **kwargs):
|
||||
if event_name in self._callbacks:
|
||||
for c in self._callbacks[event_name]:
|
||||
c(*args, **kwargs)
|
||||
|
||||
if self.delegate and hasattr(self.delegate, 'on_%s' % event_name):
|
||||
getattr(self.delegate, 'on_%s' % event_name)(*args, **kwargs)
|
||||
|
||||
return self
|
||||
|
||||
class BinaryNotFoundError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NonCleanExitError(Exception):
|
||||
def __init__(self, returncode):
|
||||
self.returncode = returncode
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.returncode)
|
||||
|
||||
|
||||
class CliDownloader():
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
|
||||
def find_binary(self, name):
|
||||
for dir in os.environ['PATH'].split(os.pathsep):
|
||||
path = os.path.join(dir, name)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
raise BinaryNotFoundError('The binary %s could not be located' % name)
|
||||
|
||||
def execute(self, args):
|
||||
proc = subprocess.Popen(args, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
|
||||
output = proc.stdout.read()
|
||||
returncode = proc.wait()
|
||||
if returncode != 0:
|
||||
error = NonCleanExitError(returncode)
|
||||
error.output = output
|
||||
raise error
|
||||
return output
|
||||
|
||||
class WgetDownloader(CliDownloader):
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.wget = self.find_binary('wget')
|
||||
|
||||
def clean_tmp_file(self):
|
||||
os.remove(self.tmp_file)
|
||||
|
||||
def download(self, url, error_message, timeout, tries):
|
||||
if not self.wget:
|
||||
return False
|
||||
|
||||
self.tmp_file = tempfile.NamedTemporaryFile().name
|
||||
command = [self.wget, '--connect-timeout=' + str(int(timeout)), '-o',
|
||||
self.tmp_file, '-O', '-', '-U', 'Emmet PyV8 Loader',
|
||||
'--no-check-certificate']
|
||||
|
||||
command.append(url)
|
||||
|
||||
if self.settings.get('http_proxy'):
|
||||
os.putenv('http_proxy', self.settings.get('http_proxy'))
|
||||
if not self.settings.get('https_proxy'):
|
||||
os.putenv('https_proxy', self.settings.get('http_proxy'))
|
||||
if self.settings.get('https_proxy'):
|
||||
os.putenv('https_proxy', self.settings.get('https_proxy'))
|
||||
|
||||
while tries > 0:
|
||||
tries -= 1
|
||||
try:
|
||||
result = self.execute(command)
|
||||
self.clean_tmp_file()
|
||||
return result
|
||||
except NonCleanExitError as e:
|
||||
error_line = ''
|
||||
with open(self.tmp_file) as f:
|
||||
for line in list(f):
|
||||
if re.search('ERROR[: ]|failed: ', line):
|
||||
error_line = line
|
||||
break
|
||||
|
||||
if e.returncode == 8:
|
||||
regex = re.compile('^.*ERROR (\d+):.*', re.S)
|
||||
if re.sub(regex, '\\1', error_line) == '503':
|
||||
# GitHub and BitBucket seem to rate limit via 503
|
||||
print('%s: Downloading %s was rate limited, trying again' % (__name__, url))
|
||||
continue
|
||||
error_string = 'HTTP error ' + re.sub('^.*? ERROR ', '',
|
||||
error_line)
|
||||
|
||||
elif e.returncode == 4:
|
||||
error_string = re.sub('^.*?failed: ', '', error_line)
|
||||
# GitHub and BitBucket seem to time out a lot
|
||||
if error_string.find('timed out') != -1:
|
||||
print('%s: Downloading %s timed out, trying again' % (__name__, url))
|
||||
continue
|
||||
|
||||
else:
|
||||
error_string = re.sub('^.*?(ERROR[: ]|failed: )', '\\1',
|
||||
error_line)
|
||||
|
||||
error_string = re.sub('\\.?\s*\n\s*$', '', error_string)
|
||||
print('%s: %s %s downloading %s.' % (__name__, error_message,
|
||||
error_string, url))
|
||||
self.clean_tmp_file()
|
||||
break
|
||||
return False
|
||||
|
||||
|
||||
class CurlDownloader(CliDownloader):
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.curl = self.find_binary('curl')
|
||||
|
||||
def download(self, url, error_message, timeout, tries):
|
||||
if not self.curl:
|
||||
return False
|
||||
command = [self.curl, '-f', '--user-agent', 'Emmet PyV8 Loader',
|
||||
'--connect-timeout', str(int(timeout)), '-sS']
|
||||
|
||||
command.append(url)
|
||||
|
||||
if self.settings.get('http_proxy'):
|
||||
os.putenv('http_proxy', self.settings.get('http_proxy'))
|
||||
if not self.settings.get('https_proxy'):
|
||||
os.putenv('HTTPS_PROXY', self.settings.get('http_proxy'))
|
||||
if self.settings.get('https_proxy'):
|
||||
os.putenv('HTTPS_PROXY', self.settings.get('https_proxy'))
|
||||
|
||||
while tries > 0:
|
||||
tries -= 1
|
||||
try:
|
||||
return self.execute(command)
|
||||
except NonCleanExitError as e:
|
||||
if e.returncode == 22:
|
||||
code = re.sub('^.*?(\d+)\s*$', '\\1', e.output)
|
||||
if code == '503':
|
||||
# GitHub and BitBucket seem to rate limit via 503
|
||||
print('%s: Downloading %s was rate limited, trying again' % (__name__, url))
|
||||
continue
|
||||
error_string = 'HTTP error ' + code
|
||||
elif e.returncode == 6:
|
||||
error_string = 'URL error host not found'
|
||||
elif e.returncode == 28:
|
||||
# GitHub and BitBucket seem to time out a lot
|
||||
print('%s: Downloading %s timed out, trying again' % (__name__, url))
|
||||
continue
|
||||
else:
|
||||
error_string = e.output.rstrip()
|
||||
|
||||
print('%s: %s %s downloading %s.' % (__name__, error_message, error_string, url))
|
||||
break
|
||||
return False
|
||||
|
||||
|
||||
class UrlLib2Downloader():
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
|
||||
def download(self, url, error_message, timeout, tries):
|
||||
http_proxy = self.settings.get('http_proxy')
|
||||
https_proxy = self.settings.get('https_proxy')
|
||||
if http_proxy or https_proxy:
|
||||
proxies = {}
|
||||
if http_proxy:
|
||||
proxies['http'] = http_proxy
|
||||
if not https_proxy:
|
||||
proxies['https'] = http_proxy
|
||||
if https_proxy:
|
||||
proxies['https'] = https_proxy
|
||||
proxy_handler = url_req.ProxyHandler(proxies)
|
||||
else:
|
||||
proxy_handler = url_req.ProxyHandler()
|
||||
handlers = [proxy_handler]
|
||||
|
||||
# secure_url_match = re.match('^https://([^/]+)', url)
|
||||
# if secure_url_match != None:
|
||||
# secure_domain = secure_url_match.group(1)
|
||||
# bundle_path = self.check_certs(secure_domain, timeout)
|
||||
# if not bundle_path:
|
||||
# return False
|
||||
# handlers.append(VerifiedHTTPSHandler(ca_certs=bundle_path))
|
||||
url_req.install_opener(url_req.build_opener(*handlers))
|
||||
|
||||
while tries > 0:
|
||||
tries -= 1
|
||||
try:
|
||||
request = url_req.Request(url, headers={"User-Agent":
|
||||
"Emmet PyV8 Loader"})
|
||||
http_file = url_req.urlopen(request, timeout=timeout)
|
||||
return http_file.read()
|
||||
|
||||
except url_err.HTTPError as e:
|
||||
# Bitbucket and Github ratelimit using 503 a decent amount
|
||||
if str(e.code) == '503':
|
||||
print('%s: Downloading %s was rate limited, trying again' % (__name__, url))
|
||||
continue
|
||||
print('%s: %s HTTP error %s downloading %s.' % (__name__, error_message, str(e.code), url))
|
||||
|
||||
except url_err.URLError as e:
|
||||
# Bitbucket and Github timeout a decent amount
|
||||
if str(e.reason) == 'The read operation timed out' or \
|
||||
str(e.reason) == 'timed out':
|
||||
print('%s: Downloading %s timed out, trying again' % (__name__, url))
|
||||
continue
|
||||
print('%s: %s URL error %s downloading %s.' % (__name__, error_message, str(e.reason), url))
|
||||
break
|
||||
return False
|
||||
|
||||
class PyV8Loader(threading.Thread):
|
||||
def __init__(self, arch, download_path, config, delegate=None):
|
||||
self.arch = arch
|
||||
self.config = config
|
||||
self.download_path = download_path
|
||||
self.exit_code = 0
|
||||
self.result = None
|
||||
self.delegate = delegate or LoaderDelegate()
|
||||
self.is_pyv8_thread = True
|
||||
|
||||
threading.Thread.__init__(self)
|
||||
self.delegate.log('Creating thread')
|
||||
|
||||
def download_url(self, url, error_message):
|
||||
# TODO add settings
|
||||
has_ssl = 'ssl' in sys.modules and hasattr(url_req, 'HTTPSHandler')
|
||||
is_ssl = re.search('^https://', url) != None
|
||||
|
||||
if (is_ssl and has_ssl) or not is_ssl:
|
||||
downloader = UrlLib2Downloader(self.delegate.settings)
|
||||
else:
|
||||
for downloader_class in [CurlDownloader, WgetDownloader]:
|
||||
try:
|
||||
downloader = downloader_class(self.delegate.settings)
|
||||
break
|
||||
except BinaryNotFoundError:
|
||||
pass
|
||||
|
||||
if not downloader:
|
||||
self.delegate.log('Unable to download PyV8 binary due to invalid downloader')
|
||||
return False
|
||||
|
||||
timeout = self.delegate.settings.get('timeout', 60)
|
||||
# timeout = 3
|
||||
return downloader.download(url.replace(' ', '%20'), error_message, timeout, 3)
|
||||
|
||||
def run(self):
|
||||
# get list of available packages first
|
||||
self.delegate.log('Loading %s' % PACKAGES_URL)
|
||||
try:
|
||||
packages = self.download_url(PACKAGES_URL, 'Unable to download packages list.')
|
||||
except Exception as e:
|
||||
self.delegate.log('Unable to download file: %s' % e)
|
||||
self.exit_code = 4
|
||||
return
|
||||
|
||||
if not packages:
|
||||
self.exit_code = 1
|
||||
return
|
||||
|
||||
if isinstance(packages, bytes):
|
||||
packages = packages.decode('utf-8')
|
||||
|
||||
files = json.loads(packages)
|
||||
|
||||
# find package for current architecture
|
||||
cur_item = None
|
||||
bundle_name = 'pyv8-%s.zip' % self.arch
|
||||
for item in files:
|
||||
if bundle_name == item['name']:
|
||||
cur_item = item
|
||||
break
|
||||
|
||||
if not cur_item:
|
||||
self.delegate.log('Unable to find binary for %s architecture' % self.arch)
|
||||
self.exit_code = 2
|
||||
return
|
||||
|
||||
if cur_item['sha'] == self.config['last_id']:
|
||||
self.delegate.log('You have the most recent PyV8 binary')
|
||||
return
|
||||
|
||||
url = 'https://raw.github.com/emmetio/pyv8-binaries/master/%s' % cur_item['name']
|
||||
self.delegate.log('Loading PyV8 binary from %s' % url)
|
||||
package = self.download_url(url, 'Unable to download package from %s' % url)
|
||||
if not package:
|
||||
self.exit_code = 3
|
||||
return
|
||||
|
||||
# we should only save downloaded package and delegate module
|
||||
# loading/unloading to main thread since improper PyV8 unload
|
||||
# may cause editor crash
|
||||
try:
|
||||
os.makedirs(self.download_path)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
fp = open(os.path.join(self.download_path, 'pack.zip'), 'wb')
|
||||
fp.write(package)
|
||||
fp.close()
|
||||
|
||||
self.result = cur_item['sha']
|
||||
# Done!
|
||||
|
||||
Reference in New Issue
Block a user