592 lines
17 KiB
Python
592 lines
17 KiB
Python
# coding=utf-8
|
|
import os
|
|
import os.path
|
|
import sys
|
|
import json
|
|
import re
|
|
import threading
|
|
import subprocess
|
|
import tempfile
|
|
import collections
|
|
import platform
|
|
import semver
|
|
import time
|
|
import zipfile
|
|
|
|
is_python3 = sys.version_info[0] > 2
|
|
|
|
if is_python3:
|
|
import urllib.request as url_req
|
|
import urllib.error as url_err
|
|
import urllib.parse as url_parse
|
|
else:
|
|
import urllib
|
|
import urllib2
|
|
url_req = urllib2
|
|
url_err = urllib2
|
|
url_parse = urllib2
|
|
|
|
CHECK_INTERVAL = 60 * 60 * 24
|
|
|
|
# PACKAGES_URL = 'https://api.github.com/repos/emmetio/pyv8-binaries/downloads'
|
|
PACKAGES_URL = 'https://api.github.com/repos/emmetio/pyv8-binaries/contents'
|
|
|
|
def load(dest_path, delegate=None):
|
|
"""
|
|
Main function that attempts to load or update PyV8 binary.
|
|
First, it loads list of available PyV8 modules and check if
|
|
PyV8 should be downloaded or updated.
|
|
@param dest_path: Path where PyV8 lib should be downloaded
|
|
@param delegate: instance of LoaderDelegate that will receive
|
|
loader progress events
|
|
@returns: `True` if download progress was initiated
|
|
"""
|
|
if delegate is None:
|
|
delegate = LoaderDelegate()
|
|
|
|
config = get_loader_config(dest_path)
|
|
|
|
if 'PyV8' in sys.modules and (config['skip_update'] or time.time() < config['last_update'] + CHECK_INTERVAL):
|
|
# No need to load anything: user already has PyV8 binary
|
|
# or decided to disable update process
|
|
delegate.log('No need to update PyV8')
|
|
return False
|
|
|
|
def on_complete(result, *args, **kwargs):
|
|
if result is not None:
|
|
# Most recent version was downloaded
|
|
config['last_id'] = result
|
|
if 'PyV8' not in sys.modules:
|
|
# PyV8 is not loaded yet, we can safely unpack it
|
|
unpack_pyv8(dest_path)
|
|
|
|
config['last_update'] = time.time()
|
|
save_loader_config(dest_path, config)
|
|
delegate.on_complete(*args, **kwargs)
|
|
|
|
# try to download most recent version of PyV8
|
|
# As PyV8 for Sublime Text spreads the world, it's possible
|
|
# that multiple distinct PyV8Loader's may start doing the same
|
|
# job at the same time. In this case, we should check if there's
|
|
# already a thread that load PyV8 and hook on existing thread
|
|
# rather that creating a new one
|
|
thread = None
|
|
thread_exists = False
|
|
for t in threading.enumerate():
|
|
if hasattr(t, 'is_pyv8_thread'):
|
|
print('PyV8: Reusing thread')
|
|
thread = t
|
|
thread_exists = True
|
|
break
|
|
|
|
if not thread:
|
|
print('PyV8: Creating new thread')
|
|
thread = PyV8Loader(get_arch(), dest_path, config, delegate=delegate)
|
|
thread.start()
|
|
|
|
delegate.on_start()
|
|
|
|
# watch on download progress
|
|
prog = ThreadProgress(thread, delegate, thread_exists)
|
|
prog.on('complete', on_complete if not thread_exists else delegate.on_complete)
|
|
prog.on('error', delegate.on_error)
|
|
|
|
def get_arch():
|
|
"Returns architecture name for PyV8 binary"
|
|
suffix = is_python3 and '-p3' or ''
|
|
p = lambda a: '%s%s' % (a, suffix)
|
|
is_64bit = sys.maxsize > 2**32
|
|
system_name = platform.system()
|
|
if system_name == 'Darwin':
|
|
if semver.match(platform.mac_ver()[0], '<10.7.0'):
|
|
return p('mac106')
|
|
|
|
return p('osx')
|
|
if system_name == 'Windows':
|
|
return p('win64') if is_64bit else p('win32')
|
|
if system_name == 'Linux':
|
|
return p('linux64') if is_64bit else p('linux32')
|
|
|
|
def get_loader_config(path):
|
|
config = {
|
|
"last_id": 0,
|
|
"last_update": 0,
|
|
"skip_update": False
|
|
}
|
|
|
|
config_path = os.path.join(path, 'config.json')
|
|
if os.path.exists(config_path):
|
|
with open(config_path) as fd:
|
|
for k,v in json.load(fd).items():
|
|
config[k] = v
|
|
|
|
return config
|
|
|
|
def save_loader_config(path, data):
|
|
config_path = os.path.join(path, 'config.json')
|
|
|
|
if not os.path.exists(path):
|
|
os.makedirs(path)
|
|
fp = open(config_path, 'w')
|
|
fp.write(json.dumps(data))
|
|
fp.close()
|
|
|
|
def clean_old_data():
|
|
for f in os.listdir('.'):
|
|
if f.lower() != 'config.json' and f.lower() != 'pack.zip':
|
|
try:
|
|
os.remove(f)
|
|
except Exception as e:
|
|
pass
|
|
|
|
def unpack_pyv8(package_dir):
|
|
f = os.path.join(package_dir, 'pack.zip')
|
|
if not os.path.exists(f):
|
|
return
|
|
|
|
package_zip = zipfile.ZipFile(f, 'r')
|
|
|
|
root_level_paths = []
|
|
last_path = None
|
|
for path in package_zip.namelist():
|
|
last_path = path
|
|
if path.find('/') in [len(path) - 1, -1]:
|
|
root_level_paths.append(path)
|
|
if path[0] == '/' or path.find('../') != -1 or path.find('..\\') != -1:
|
|
raise 'The PyV8 package contains files outside of the package dir and cannot be safely installed.'
|
|
|
|
if last_path and len(root_level_paths) == 0:
|
|
root_level_paths.append(last_path[0:last_path.find('/') + 1])
|
|
|
|
prev_dir = os.getcwd()
|
|
os.chdir(package_dir)
|
|
|
|
clean_old_data()
|
|
|
|
# Here we don't use .extractall() since it was having issues on OS X
|
|
skip_root_dir = len(root_level_paths) == 1 and \
|
|
root_level_paths[0].endswith('/')
|
|
extracted_paths = []
|
|
for path in package_zip.namelist():
|
|
dest = path
|
|
|
|
if not is_python3:
|
|
try:
|
|
if not isinstance(dest, unicode):
|
|
dest = unicode(dest, 'utf-8', 'strict')
|
|
except UnicodeDecodeError:
|
|
dest = unicode(dest, 'cp1252', 'replace')
|
|
|
|
if os.name == 'nt':
|
|
regex = ':|\*|\?|"|<|>|\|'
|
|
if re.search(regex, dest) != None:
|
|
print ('%s: Skipping file from package named %s due to ' +
|
|
'an invalid filename') % (__name__, path)
|
|
continue
|
|
|
|
# If there was only a single directory in the package, we remove
|
|
# that folder name from the paths as we extract entries
|
|
if skip_root_dir:
|
|
dest = dest[len(root_level_paths[0]):]
|
|
|
|
if os.name == 'nt':
|
|
dest = dest.replace('/', '\\')
|
|
else:
|
|
dest = dest.replace('\\', '/')
|
|
|
|
dest = os.path.join(package_dir, dest)
|
|
|
|
def add_extracted_dirs(dir):
|
|
while dir not in extracted_paths:
|
|
extracted_paths.append(dir)
|
|
dir = os.path.dirname(dir)
|
|
if dir == package_dir:
|
|
break
|
|
|
|
if path.endswith('/'):
|
|
if not os.path.exists(dest):
|
|
os.makedirs(dest)
|
|
add_extracted_dirs(dest)
|
|
else:
|
|
dest_dir = os.path.dirname(dest)
|
|
if not os.path.exists(dest_dir):
|
|
os.makedirs(dest_dir)
|
|
add_extracted_dirs(dest_dir)
|
|
extracted_paths.append(dest)
|
|
try:
|
|
open(dest, 'wb').write(package_zip.read(path))
|
|
except (IOError, UnicodeDecodeError):
|
|
print ('%s: Skipping file from package named %s due to ' +
|
|
'an invalid filename') % (__name__, path)
|
|
package_zip.close()
|
|
|
|
os.chdir(prev_dir)
|
|
os.remove(f)
|
|
|
|
class LoaderDelegate():
|
|
"""
|
|
Abstract class used to display PyV8 binary download progress,
|
|
and provide some settings for downloader
|
|
"""
|
|
def __init__(self, settings={}):
|
|
self.settings = settings
|
|
|
|
def on_start(self, *args, **kwargs):
|
|
"Invoked when download process is initiated"
|
|
pass
|
|
|
|
def on_progress(self, *args, **kwargs):
|
|
"Invoked on download progress"
|
|
pass
|
|
|
|
def on_complete(self, *args, **kwargs):
|
|
"Invoked when download process was finished successfully"
|
|
pass
|
|
|
|
def on_error(self, *args, **kwargs):
|
|
"Invoked when error occured during download process"
|
|
pass
|
|
|
|
def setting(self, name, default=None):
|
|
"Returns specified setting name"
|
|
return self.settings[name] if name in self.settings else default
|
|
|
|
def log(self, message):
|
|
pass
|
|
|
|
class ThreadProgress():
|
|
def __init__(self, thread, delegate, is_background=False):
|
|
self.thread = thread
|
|
self.delegate = delegate
|
|
self.is_background = is_background
|
|
self._callbacks = {}
|
|
threading.Timer(0, self.run).start()
|
|
|
|
def run(self):
|
|
if not self.thread.is_alive():
|
|
if self.thread.exit_code != 0:
|
|
return self.trigger('error', exit_code=self.thread.exit_code, progress=self)
|
|
|
|
return self.trigger('complete', result=self.thread.result, progress=self)
|
|
|
|
self.trigger('progress', progress=self)
|
|
threading.Timer(0.1, self.run).start()
|
|
|
|
def on(self, event_name, callback):
|
|
if event_name not in self._callbacks:
|
|
self._callbacks[event_name] = []
|
|
|
|
if isinstance(callback, collections.Callable):
|
|
self._callbacks[event_name].append(callback)
|
|
|
|
return self
|
|
|
|
def trigger(self, event_name, *args, **kwargs):
|
|
if event_name in self._callbacks:
|
|
for c in self._callbacks[event_name]:
|
|
c(*args, **kwargs)
|
|
|
|
if self.delegate and hasattr(self.delegate, 'on_%s' % event_name):
|
|
getattr(self.delegate, 'on_%s' % event_name)(*args, **kwargs)
|
|
|
|
return self
|
|
|
|
class BinaryNotFoundError(Exception):
|
|
pass
|
|
|
|
|
|
class NonCleanExitError(Exception):
|
|
def __init__(self, returncode):
|
|
self.returncode = returncode
|
|
|
|
def __str__(self):
|
|
return repr(self.returncode)
|
|
|
|
|
|
class CliDownloader():
|
|
def __init__(self, settings):
|
|
self.settings = settings
|
|
|
|
def find_binary(self, name):
|
|
for dir in os.environ['PATH'].split(os.pathsep):
|
|
path = os.path.join(dir, name)
|
|
if os.path.exists(path):
|
|
return path
|
|
|
|
raise BinaryNotFoundError('The binary %s could not be located' % name)
|
|
|
|
def execute(self, args):
|
|
proc = subprocess.Popen(args, stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
|
|
output = proc.stdout.read()
|
|
returncode = proc.wait()
|
|
if returncode != 0:
|
|
error = NonCleanExitError(returncode)
|
|
error.output = output
|
|
raise error
|
|
return output
|
|
|
|
class WgetDownloader(CliDownloader):
|
|
def __init__(self, settings):
|
|
self.settings = settings
|
|
self.wget = self.find_binary('wget')
|
|
|
|
def clean_tmp_file(self):
|
|
os.remove(self.tmp_file)
|
|
|
|
def download(self, url, error_message, timeout, tries):
|
|
if not self.wget:
|
|
return False
|
|
|
|
self.tmp_file = tempfile.NamedTemporaryFile().name
|
|
command = [self.wget, '--connect-timeout=' + str(int(timeout)), '-o',
|
|
self.tmp_file, '-O', '-', '-U', 'Emmet PyV8 Loader',
|
|
'--no-check-certificate']
|
|
|
|
command.append(url)
|
|
|
|
if self.settings.get('http_proxy'):
|
|
os.putenv('http_proxy', self.settings.get('http_proxy'))
|
|
if not self.settings.get('https_proxy'):
|
|
os.putenv('https_proxy', self.settings.get('http_proxy'))
|
|
if self.settings.get('https_proxy'):
|
|
os.putenv('https_proxy', self.settings.get('https_proxy'))
|
|
|
|
while tries > 0:
|
|
tries -= 1
|
|
try:
|
|
result = self.execute(command)
|
|
self.clean_tmp_file()
|
|
return result
|
|
except NonCleanExitError as e:
|
|
error_line = ''
|
|
with open(self.tmp_file) as f:
|
|
for line in list(f):
|
|
if re.search('ERROR[: ]|failed: ', line):
|
|
error_line = line
|
|
break
|
|
|
|
if e.returncode == 8:
|
|
regex = re.compile('^.*ERROR (\d+):.*', re.S)
|
|
if re.sub(regex, '\\1', error_line) == '503':
|
|
# GitHub and BitBucket seem to rate limit via 503
|
|
print('%s: Downloading %s was rate limited, trying again' % (__name__, url))
|
|
continue
|
|
error_string = 'HTTP error ' + re.sub('^.*? ERROR ', '',
|
|
error_line)
|
|
|
|
elif e.returncode == 4:
|
|
error_string = re.sub('^.*?failed: ', '', error_line)
|
|
# GitHub and BitBucket seem to time out a lot
|
|
if error_string.find('timed out') != -1:
|
|
print('%s: Downloading %s timed out, trying again' % (__name__, url))
|
|
continue
|
|
|
|
else:
|
|
error_string = re.sub('^.*?(ERROR[: ]|failed: )', '\\1',
|
|
error_line)
|
|
|
|
error_string = re.sub('\\.?\s*\n\s*$', '', error_string)
|
|
print('%s: %s %s downloading %s.' % (__name__, error_message,
|
|
error_string, url))
|
|
self.clean_tmp_file()
|
|
break
|
|
return False
|
|
|
|
|
|
class CurlDownloader(CliDownloader):
|
|
def __init__(self, settings):
|
|
self.settings = settings
|
|
self.curl = self.find_binary('curl')
|
|
|
|
def download(self, url, error_message, timeout, tries):
|
|
if not self.curl:
|
|
return False
|
|
command = [self.curl, '-f', '--user-agent', 'Emmet PyV8 Loader',
|
|
'--connect-timeout', str(int(timeout)), '-sS']
|
|
|
|
command.append(url)
|
|
|
|
if self.settings.get('http_proxy'):
|
|
os.putenv('http_proxy', self.settings.get('http_proxy'))
|
|
if not self.settings.get('https_proxy'):
|
|
os.putenv('HTTPS_PROXY', self.settings.get('http_proxy'))
|
|
if self.settings.get('https_proxy'):
|
|
os.putenv('HTTPS_PROXY', self.settings.get('https_proxy'))
|
|
|
|
while tries > 0:
|
|
tries -= 1
|
|
try:
|
|
return self.execute(command)
|
|
except NonCleanExitError as e:
|
|
if e.returncode == 22:
|
|
code = re.sub('^.*?(\d+)\s*$', '\\1', e.output)
|
|
if code == '503':
|
|
# GitHub and BitBucket seem to rate limit via 503
|
|
print('%s: Downloading %s was rate limited, trying again' % (__name__, url))
|
|
continue
|
|
error_string = 'HTTP error ' + code
|
|
elif e.returncode == 6:
|
|
error_string = 'URL error host not found'
|
|
elif e.returncode == 28:
|
|
# GitHub and BitBucket seem to time out a lot
|
|
print('%s: Downloading %s timed out, trying again' % (__name__, url))
|
|
continue
|
|
else:
|
|
error_string = e.output.rstrip()
|
|
|
|
print('%s: %s %s downloading %s.' % (__name__, error_message, error_string, url))
|
|
break
|
|
return False
|
|
|
|
|
|
class UrlLib2Downloader():
|
|
def __init__(self, settings):
|
|
self.settings = settings
|
|
|
|
def download(self, url, error_message, timeout, tries):
|
|
http_proxy = self.settings.get('http_proxy')
|
|
https_proxy = self.settings.get('https_proxy')
|
|
if http_proxy or https_proxy:
|
|
proxies = {}
|
|
if http_proxy:
|
|
proxies['http'] = http_proxy
|
|
if not https_proxy:
|
|
proxies['https'] = http_proxy
|
|
if https_proxy:
|
|
proxies['https'] = https_proxy
|
|
proxy_handler = url_req.ProxyHandler(proxies)
|
|
else:
|
|
proxy_handler = url_req.ProxyHandler()
|
|
handlers = [proxy_handler]
|
|
|
|
# secure_url_match = re.match('^https://([^/]+)', url)
|
|
# if secure_url_match != None:
|
|
# secure_domain = secure_url_match.group(1)
|
|
# bundle_path = self.check_certs(secure_domain, timeout)
|
|
# if not bundle_path:
|
|
# return False
|
|
# handlers.append(VerifiedHTTPSHandler(ca_certs=bundle_path))
|
|
url_req.install_opener(url_req.build_opener(*handlers))
|
|
|
|
while tries > 0:
|
|
tries -= 1
|
|
try:
|
|
request = url_req.Request(url, headers={"User-Agent":
|
|
"Emmet PyV8 Loader"})
|
|
http_file = url_req.urlopen(request, timeout=timeout)
|
|
return http_file.read()
|
|
|
|
except url_err.HTTPError as e:
|
|
# Bitbucket and Github ratelimit using 503 a decent amount
|
|
if str(e.code) == '503':
|
|
print('%s: Downloading %s was rate limited, trying again' % (__name__, url))
|
|
continue
|
|
print('%s: %s HTTP error %s downloading %s.' % (__name__, error_message, str(e.code), url))
|
|
|
|
except url_err.URLError as e:
|
|
# Bitbucket and Github timeout a decent amount
|
|
if str(e.reason) == 'The read operation timed out' or \
|
|
str(e.reason) == 'timed out':
|
|
print('%s: Downloading %s timed out, trying again' % (__name__, url))
|
|
continue
|
|
print('%s: %s URL error %s downloading %s.' % (__name__, error_message, str(e.reason), url))
|
|
break
|
|
return False
|
|
|
|
class PyV8Loader(threading.Thread):
|
|
def __init__(self, arch, download_path, config, delegate=None):
|
|
self.arch = arch
|
|
self.config = config
|
|
self.download_path = download_path
|
|
self.exit_code = 0
|
|
self.result = None
|
|
self.delegate = delegate or LoaderDelegate()
|
|
self.is_pyv8_thread = True
|
|
|
|
threading.Thread.__init__(self)
|
|
self.delegate.log('Creating thread')
|
|
|
|
def download_url(self, url, error_message):
|
|
# TODO add settings
|
|
has_ssl = 'ssl' in sys.modules and hasattr(url_req, 'HTTPSHandler')
|
|
is_ssl = re.search('^https://', url) != None
|
|
|
|
if (is_ssl and has_ssl) or not is_ssl:
|
|
downloader = UrlLib2Downloader(self.delegate.settings)
|
|
else:
|
|
for downloader_class in [CurlDownloader, WgetDownloader]:
|
|
try:
|
|
downloader = downloader_class(self.delegate.settings)
|
|
break
|
|
except BinaryNotFoundError:
|
|
pass
|
|
|
|
if not downloader:
|
|
self.delegate.log('Unable to download PyV8 binary due to invalid downloader')
|
|
return False
|
|
|
|
timeout = self.delegate.settings.get('timeout', 60)
|
|
# timeout = 3
|
|
return downloader.download(url.replace(' ', '%20'), error_message, timeout, 3)
|
|
|
|
def run(self):
|
|
# get list of available packages first
|
|
self.delegate.log('Loading %s' % PACKAGES_URL)
|
|
try:
|
|
packages = self.download_url(PACKAGES_URL, 'Unable to download packages list.')
|
|
except Exception as e:
|
|
self.delegate.log('Unable to download file: %s' % e)
|
|
self.exit_code = 4
|
|
return
|
|
|
|
if not packages:
|
|
self.exit_code = 1
|
|
return
|
|
|
|
if isinstance(packages, bytes):
|
|
packages = packages.decode('utf-8')
|
|
|
|
files = json.loads(packages)
|
|
|
|
# find package for current architecture
|
|
cur_item = None
|
|
bundle_name = 'pyv8-%s.zip' % self.arch
|
|
for item in files:
|
|
if bundle_name == item['name']:
|
|
cur_item = item
|
|
break
|
|
|
|
if not cur_item:
|
|
self.delegate.log('Unable to find binary for %s architecture' % self.arch)
|
|
self.exit_code = 2
|
|
return
|
|
|
|
if cur_item['sha'] == self.config['last_id']:
|
|
self.delegate.log('You have the most recent PyV8 binary')
|
|
return
|
|
|
|
url = 'https://raw.github.com/emmetio/pyv8-binaries/master/%s' % cur_item['name']
|
|
self.delegate.log('Loading PyV8 binary from %s' % url)
|
|
package = self.download_url(url, 'Unable to download package from %s' % url)
|
|
if not package:
|
|
self.exit_code = 3
|
|
return
|
|
|
|
# we should only save downloaded package and delegate module
|
|
# loading/unloading to main thread since improper PyV8 unload
|
|
# may cause editor crash
|
|
try:
|
|
os.makedirs(self.download_path)
|
|
except Exception as e:
|
|
pass
|
|
|
|
fp = open(os.path.join(self.download_path, 'pack.zip'), 'wb')
|
|
fp.write(package)
|
|
fp.close()
|
|
|
|
self.result = cur_item['sha']
|
|
# Done!
|
|
|