Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: ensure_dependencies.py

Issue 29531669: Issue 5596 - Update buildtools dependency to ba4805b5eda4 (Closed)
Patch Set: Created Aug. 30, 2017, 2:30 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « dependencies ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 2
3 # This Source Code Form is subject to the terms of the Mozilla Public 3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this 4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 6
7 import sys 7 import sys
8 import os 8 import os
9 import posixpath 9 import posixpath
10 import re 10 import re
11 import io 11 import io
12 import errno 12 import errno
13 import logging 13 import logging
14 import subprocess 14 import subprocess
15 import urlparse 15 import urlparse
16 import argparse 16 import argparse
17 import json
17 18
18 from collections import OrderedDict 19 from collections import OrderedDict
19 from ConfigParser import RawConfigParser 20 from ConfigParser import RawConfigParser
20 21
21 USAGE = ''' 22 USAGE = '''
22 A dependencies file should look like this: 23 A dependencies file should look like this:
23 24
24 # VCS-specific root URLs for the repositories 25 # VCS-specific root URLs for the repositories
25 _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/ 26 _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
26 # File to update this script from (optional) 27 # File to update this script from (optional)
(...skipping 211 matching lines...) Expand 10 before | Expand all | Expand 10 after
238 return os.path.join(path, *normpath.split(posixpath.sep)) 239 return os.path.join(path, *normpath.split(posixpath.sep))
239 240
240 241
241 def get_repo_type(repo): 242 def get_repo_type(repo):
242 for name, repotype in repo_types.iteritems(): 243 for name, repotype in repo_types.iteritems():
243 if repotype.istype(repo): 244 if repotype.istype(repo):
244 return name 245 return name
245 return 'hg' 246 return 'hg'
246 247
247 248
249 def resolve_npm_dependencies(target, vcs):
250 """Install Node.js production-only dependencies if necessary and desired.
251
252 When the target dependency has additional Node.js dependencies declared
253 run "npm install --only=production --loglevel=warn" to resolve the declared
254 dependencies.
255
256 Additionally, make sure that any VCS will ignore the installed files.
257
258 Requires Node.js to be installed locally.
259 """
260 try:
261 with open(os.path.join(target, 'package.json'), 'r') as fp:
262 package_data = json.load(fp)
263
264 # In case a package.json does not exist at all or if there are no
265 # production dependencies declared, we don't need to run npm and can
266 # bail out early.
267 if not package_data.get('dependencies', False):
268 return
269 except IOError:
270 return
271
272 try:
273 cmd = ['npm', 'install', '--only=production', '--loglevel=warn']
274 subprocess.check_output(cmd, cwd=target)
275
276 repo_types[vcs].ignore(os.path.join(target, 'node_modules'), target)
277 except OSError as e:
278 import errno
279 if e.errno == errno.ENOENT:
280 logging.error('Failed to install Node.js dependencies for %s,'
281 ' please ensure Node.js is installed.', target)
282 else:
283 raise
284
285
248 def ensure_repo(parentrepo, parenttype, target, type, root, sourcename): 286 def ensure_repo(parentrepo, parenttype, target, type, root, sourcename):
249 if os.path.exists(target): 287 if os.path.exists(target):
250 return 288 return False
251 289
252 if SKIP_DEPENDENCY_UPDATES: 290 if SKIP_DEPENDENCY_UPDATES:
253 logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, ' 291 logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, '
254 '%s not cloned', target) 292 '%s not cloned', target)
255 return 293 return False
256 294
257 postprocess_url = repo_types[type].postprocess_url 295 postprocess_url = repo_types[type].postprocess_url
258 root = postprocess_url(root) 296 root = postprocess_url(root)
259 sourcename = postprocess_url(sourcename) 297 sourcename = postprocess_url(sourcename)
260 298
261 if os.path.exists(root): 299 if os.path.exists(root):
262 url = os.path.join(root, sourcename) 300 url = os.path.join(root, sourcename)
263 else: 301 else:
264 url = urlparse.urljoin(root, sourcename) 302 url = urlparse.urljoin(root, sourcename)
265 303
266 logging.info('Cloning repository %s into %s' % (url, target)) 304 logging.info('Cloning repository %s into %s' % (url, target))
267 repo_types[type].clone(url, target) 305 repo_types[type].clone(url, target)
268 repo_types[parenttype].ignore(target, parentrepo) 306 repo_types[parenttype].ignore(target, parentrepo)
307 return True
269 308
270 309
271 def update_repo(target, type, revision): 310 def update_repo(target, type, revision):
272 resolved_revision = repo_types[type].get_revision_id(target, revision) 311 resolved_revision = repo_types[type].get_revision_id(target, revision)
273 current_revision = repo_types[type].get_revision_id(target) 312 current_revision = repo_types[type].get_revision_id(target)
274 313
275 if resolved_revision != current_revision: 314 if resolved_revision != current_revision:
276 if SKIP_DEPENDENCY_UPDATES: 315 if SKIP_DEPENDENCY_UPDATES:
277 logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, ' 316 logging.warning('SKIP_DEPENDENCY_UPDATES environment variable set, '
278 '%s not checked out to %s', target, revision) 317 '%s not checked out to %s', target, revision)
279 return 318 return False
280 319
281 if not resolved_revision: 320 if not resolved_revision:
282 logging.info('Revision %s is unknown, downloading remote changes' % revision) 321 logging.info('Revision %s is unknown, downloading remote changes' % revision)
283 repo_types[type].pull(target) 322 repo_types[type].pull(target)
284 resolved_revision = repo_types[type].get_revision_id(target, revisio n) 323 resolved_revision = repo_types[type].get_revision_id(target, revisio n)
285 if not resolved_revision: 324 if not resolved_revision:
286 raise Exception('Failed to resolve revision %s' % revision) 325 raise Exception('Failed to resolve revision %s' % revision)
287 326
288 logging.info('Updating repository %s to revision %s' % (target, resolved _revision)) 327 logging.info('Updating repository %s to revision %s' % (target, resolved _revision))
289 repo_types[type].update(target, resolved_revision, revision) 328 repo_types[type].update(target, resolved_revision, revision)
329 return True
330 return False
290 331
291 332
292 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdep endencies=set()): 333 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdep endencies=set()):
293 config = read_deps(repodir) 334 config = read_deps(repodir)
294 if config is None: 335 if config is None:
295 if level == 0: 336 if level == 0:
296 logging.warning('No dependencies file in directory %s, nothing to do ...\n%s' % (repodir, USAGE)) 337 logging.warning('No dependencies file in directory %s, nothing to do ...\n%s' % (repodir, USAGE))
297 return 338 return
298 if level >= 10: 339 if level >= 10:
299 logging.warning('Too much subrepository nesting, ignoring %s' % repo) 340 logging.warning('Too much subrepository nesting, ignoring %s' % repo)
(...skipping 13 matching lines...) Expand all
313 354
314 for key in sources.keys() + _root.keys(): 355 for key in sources.keys() + _root.keys():
315 if key == parenttype or key is None and vcs != '*': 356 if key == parenttype or key is None and vcs != '*':
316 vcs = key 357 vcs = key
317 source, rev = merge_seqs(sources.get('*'), sources.get(vcs)) 358 source, rev = merge_seqs(sources.get('*'), sources.get(vcs))
318 359
319 if not (vcs and source and rev): 360 if not (vcs and source and rev):
320 logging.warning('No valid source / revision found to create %s' % ta rget) 361 logging.warning('No valid source / revision found to create %s' % ta rget)
321 continue 362 continue
322 363
323 ensure_repo(repodir, parenttype, target, vcs, _root.get(vcs, ''), source ) 364 repo_cloned = ensure_repo(repodir, parenttype, target, vcs,
324 update_repo(target, vcs, rev) 365 _root.get(vcs, ''), source)
366 repo_updated = update_repo(target, vcs, rev)
367 if repo_cloned or repo_updated:
368 resolve_npm_dependencies(target, vcs)
325 resolve_deps(target, level + 1, self_update=False, 369 resolve_deps(target, level + 1, self_update=False,
326 overrideroots=overrideroots, skipdependencies=skipdependenc ies) 370 overrideroots=overrideroots, skipdependencies=skipdependenc ies)
327 371
328 if self_update and '_self' in config and '*' in config['_self']: 372 if self_update and '_self' in config and '*' in config['_self']:
329 source = safe_join(repodir, config['_self']['*']) 373 source = safe_join(repodir, config['_self']['*'])
330 try: 374 try:
331 with io.open(source, 'rb') as handle: 375 with io.open(source, 'rb') as handle:
332 sourcedata = handle.read() 376 sourcedata = handle.read()
333 except IOError as e: 377 except IOError as e:
334 if e.errno != errno.ENOENT: 378 if e.errno != errno.ENOENT:
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after
371 args = parser.parse_args() 415 args = parser.parse_args()
372 416
373 if args.quiet: 417 if args.quiet:
374 logging.disable(logging.INFO) 418 logging.disable(logging.INFO)
375 419
376 repos = args.repos 420 repos = args.repos
377 if not len(repos): 421 if not len(repos):
378 repos = [os.path.dirname(__file__)] 422 repos = [os.path.dirname(__file__)]
379 for repo in repos: 423 for repo in repos:
380 resolve_deps(repo) 424 resolve_deps(repo)
OLDNEW
« no previous file with comments | « dependencies ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld