You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
292 lines
9.7 KiB
292 lines
9.7 KiB
#!c:\e\python3\python3.exe
|
|
|
|
import sys
|
|
import os
|
|
import glob
|
|
import tempfile
|
|
import shutil
|
|
import time
|
|
from six.moves.urllib.request import (build_opener, install_opener,
|
|
HTTPRedirectHandler as UrllibHTTPRedirectHandler,
|
|
Request, urlopen)
|
|
from six.moves.urllib.error import HTTPError, URLError
|
|
import netrc
|
|
import json
|
|
from optparse import OptionParser
|
|
from six.moves.urllib.parse import urlparse, urljoin
|
|
from subprocess import Popen, PIPE, check_call
|
|
|
|
from w3lib.form import encode_multipart
|
|
import setuptools # not used in code but needed in runtime, don't remove!
|
|
|
|
from scrapy.utils.project import inside_project
|
|
from scrapy.utils.http import basic_auth_header
|
|
from scrapy.utils.python import retry_on_eintr
|
|
from scrapy.utils.conf import get_config, closest_scrapy_cfg
|
|
|
|
_SETUP_PY_TEMPLATE = \
|
|
"""# Automatically created by: scrapyd-deploy
|
|
|
|
from setuptools import setup, find_packages
|
|
|
|
setup(
|
|
name = 'project',
|
|
version = '1.0',
|
|
packages = find_packages(),
|
|
entry_points = {'scrapy': ['settings = %(settings)s']},
|
|
)
|
|
"""
|
|
|
|
def parse_opts():
|
|
parser = OptionParser(usage="%prog [options] [ [target] | -l | -L <target> ]",
|
|
description="Deploy Scrapy project to Scrapyd server")
|
|
parser.add_option("-p", "--project",
|
|
help="the project name in the target")
|
|
parser.add_option("-v", "--version",
|
|
help="the version to deploy. Defaults to current timestamp")
|
|
parser.add_option("-l", "--list-targets", action="store_true", \
|
|
help="list available targets")
|
|
parser.add_option("-a", "--deploy-all-targets",action="store_true", help="deploy all targets")
|
|
parser.add_option("-d", "--debug", action="store_true",
|
|
help="debug mode (do not remove build dir)")
|
|
parser.add_option("-L", "--list-projects", metavar="TARGET", \
|
|
help="list available projects on TARGET")
|
|
parser.add_option("--egg", metavar="FILE",
|
|
help="use the given egg, instead of building it")
|
|
parser.add_option("--build-egg", metavar="FILE",
|
|
help="only build the egg, don't deploy it")
|
|
return parser.parse_args()
|
|
|
|
def main():
|
|
opts, args = parse_opts()
|
|
exitcode = 0
|
|
if not inside_project():
|
|
_log("Error: no Scrapy project found in this location")
|
|
sys.exit(1)
|
|
|
|
install_opener(
|
|
build_opener(HTTPRedirectHandler)
|
|
)
|
|
|
|
if opts.list_targets:
|
|
for name, target in _get_targets().items():
|
|
print("%-20s %s" % (name, target['url']))
|
|
return
|
|
|
|
if opts.list_projects:
|
|
target = _get_target(opts.list_projects)
|
|
req = Request(_url(target, 'listprojects.json'))
|
|
_add_auth_header(req, target)
|
|
f = urlopen(req)
|
|
projects = json.loads(f.read())['projects']
|
|
print(os.linesep.join(projects))
|
|
return
|
|
|
|
tmpdir = None
|
|
|
|
if opts.build_egg: # build egg only
|
|
egg, tmpdir = _build_egg()
|
|
_log("Writing egg to %s" % opts.build_egg)
|
|
shutil.copyfile(egg, opts.build_egg)
|
|
elif opts.deploy_all_targets:
|
|
version = None
|
|
for name, target in _get_targets().items():
|
|
if version is None:
|
|
version = _get_version(target, opts)
|
|
_build_egg_and_deploy_target(target, version, opts)
|
|
else: # buld egg and deploy
|
|
target_name = _get_target_name(args)
|
|
target = _get_target(target_name)
|
|
version = _get_version(target, opts)
|
|
exitcode, tmpdir = _build_egg_and_deploy_target(target, version, opts)
|
|
|
|
if tmpdir:
|
|
if opts.debug:
|
|
_log("Output dir not removed: %s" % tmpdir)
|
|
else:
|
|
shutil.rmtree(tmpdir)
|
|
|
|
sys.exit(exitcode)
|
|
|
|
def _build_egg_and_deploy_target(target, version, opts):
|
|
exitcode = 0
|
|
tmpdir = None
|
|
|
|
project = _get_project(target, opts)
|
|
if opts.egg:
|
|
_log("Using egg: %s" % opts.egg)
|
|
egg = opts.egg
|
|
else:
|
|
_log("Packing version %s" % version)
|
|
egg, tmpdir = _build_egg()
|
|
if not _upload_egg(target, egg, project, version):
|
|
exitcode = 1
|
|
return exitcode, tmpdir
|
|
|
|
def _log(message):
|
|
sys.stderr.write(message + os.linesep)
|
|
|
|
def _fail(message, code=1):
|
|
_log(message)
|
|
sys.exit(code)
|
|
|
|
def _get_target_name(args):
|
|
if len(args) > 1:
|
|
raise _fail("Error: Too many arguments: %s" % ' '.join(args))
|
|
elif args:
|
|
return args[0]
|
|
elif len(args) < 1:
|
|
return 'default'
|
|
|
|
def _get_project(target, opts):
|
|
project = opts.project or target.get('project')
|
|
if not project:
|
|
raise _fail("Error: Missing project")
|
|
return project
|
|
|
|
def _get_option(section, option, default=None):
|
|
cfg = get_config()
|
|
return cfg.get(section, option) if cfg.has_option(section, option) \
|
|
else default
|
|
|
|
def _get_targets():
|
|
cfg = get_config()
|
|
baset = dict(cfg.items('deploy')) if cfg.has_section('deploy') else {}
|
|
targets = {}
|
|
if 'url' in baset:
|
|
targets['default'] = baset
|
|
for x in cfg.sections():
|
|
if x.startswith('deploy:'):
|
|
t = baset.copy()
|
|
t.update(cfg.items(x))
|
|
targets[x[7:]] = t
|
|
return targets
|
|
|
|
def _get_target(name):
|
|
try:
|
|
return _get_targets()[name]
|
|
except KeyError:
|
|
raise _fail("Unknown target: %s" % name)
|
|
|
|
def _url(target, action):
|
|
return urljoin(target['url'], action)
|
|
|
|
def _get_version(target, opts):
|
|
version = opts.version or target.get('version')
|
|
if version == 'HG':
|
|
p = Popen(['hg', 'tip', '--template', '{rev}'], stdout=PIPE, universal_newlines=True)
|
|
d = 'r%s' % p.communicate()[0]
|
|
p = Popen(['hg', 'branch'], stdout=PIPE, universal_newlines=True)
|
|
b = p.communicate()[0].strip('\n')
|
|
return '%s-%s' % (d, b)
|
|
elif version == 'GIT':
|
|
p = Popen(['git', 'describe'], stdout=PIPE, universal_newlines=True)
|
|
d = p.communicate()[0].strip('\n')
|
|
if p.wait() != 0:
|
|
p = Popen(['git', 'rev-list', '--count', 'HEAD'], stdout=PIPE, universal_newlines=True)
|
|
d = 'r%s' % p.communicate()[0].strip('\n')
|
|
|
|
p = Popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stdout=PIPE, universal_newlines=True)
|
|
b = p.communicate()[0].strip('\n')
|
|
return '%s-%s' % (d, b)
|
|
elif version:
|
|
return version
|
|
else:
|
|
return str(int(time.time()))
|
|
|
|
def _upload_egg(target, eggpath, project, version):
|
|
with open(eggpath, 'rb') as f:
|
|
eggdata = f.read()
|
|
data = {
|
|
'project': project,
|
|
'version': version,
|
|
'egg': ('project.egg', eggdata),
|
|
}
|
|
body, boundary = encode_multipart(data)
|
|
url = _url(target, 'addversion.json')
|
|
headers = {
|
|
'Content-Type': 'multipart/form-data; boundary=%s' % boundary,
|
|
'Content-Length': str(len(body)),
|
|
}
|
|
req = Request(url, body, headers)
|
|
_add_auth_header(req, target)
|
|
_log('Deploying to project "%s" in %s' % (project, url))
|
|
return _http_post(req)
|
|
|
|
def _add_auth_header(request, target):
|
|
if 'username' in target:
|
|
u, p = target.get('username'), target.get('password', '')
|
|
request.add_header('Authorization', basic_auth_header(u, p))
|
|
else: # try netrc
|
|
try:
|
|
host = urlparse(target['url']).hostname
|
|
a = netrc.netrc().authenticators(host)
|
|
request.add_header('Authorization', basic_auth_header(a[0], a[2]))
|
|
except (netrc.NetrcParseError, IOError, TypeError):
|
|
pass
|
|
|
|
def _http_post(request):
|
|
try:
|
|
f = urlopen(request)
|
|
_log("Server response (%s):" % f.code)
|
|
print(f.read().decode('utf-8'))
|
|
return True
|
|
except HTTPError as e:
|
|
_log("Deploy failed (%s):" % e.code)
|
|
resp = e.read().decode('utf-8')
|
|
try:
|
|
d = json.loads(resp)
|
|
except ValueError:
|
|
print(resp)
|
|
else:
|
|
if "status" in d and "message" in d:
|
|
print("Status: %(status)s" % d)
|
|
print("Message:\n%(message)s" % d)
|
|
else:
|
|
print(json.dumps(d, indent=3))
|
|
except URLError as e:
|
|
_log("Deploy failed: %s" % e)
|
|
|
|
def _build_egg():
|
|
closest = closest_scrapy_cfg()
|
|
os.chdir(os.path.dirname(closest))
|
|
if not os.path.exists('setup.py'):
|
|
settings = get_config().get('settings', 'default')
|
|
_create_default_setup_py(settings=settings)
|
|
d = tempfile.mkdtemp(prefix="scrapydeploy-")
|
|
o = open(os.path.join(d, "stdout"), "wb")
|
|
e = open(os.path.join(d, "stderr"), "wb")
|
|
retry_on_eintr(check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d], stdout=o, stderr=e)
|
|
o.close()
|
|
e.close()
|
|
egg = glob.glob(os.path.join(d, '*.egg'))[0]
|
|
return egg, d
|
|
|
|
def _create_default_setup_py(**kwargs):
|
|
with open('setup.py', 'w') as f:
|
|
f.write(_SETUP_PY_TEMPLATE % kwargs)
|
|
|
|
|
|
class HTTPRedirectHandler(UrllibHTTPRedirectHandler):
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
newurl = newurl.replace(' ', '%20')
|
|
if code in (301, 307):
|
|
return Request(newurl,
|
|
data=req.get_data(),
|
|
headers=req.headers,
|
|
origin_req_host=req.get_origin_req_host(),
|
|
unverifiable=True)
|
|
elif code in (302, 303):
|
|
newheaders = dict((k, v) for k, v in req.headers.items()
|
|
if k.lower() not in ("content-length", "content-type"))
|
|
return Request(newurl,
|
|
headers=newheaders,
|
|
origin_req_host=req.get_origin_req_host(),
|
|
unverifiable=True)
|
|
else:
|
|
raise HTTPError(req.get_full_url(), code, msg, headers, fp)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|