Issue #6681 - Add linting for .gitlab-ci.yml

build.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import argparse
 import logging
+import io
+import json
 import os
 import re
 import shutil
 import subprocess
 import sys
+from urllib import urlencode
+import urllib2
 from functools import partial
 from StringIO import StringIO
 from zipfile import ZipFile
 from buildtools.localeTools import read_locale_config
 
 KNOWN_PLATFORMS = {'chrome', 'gecko', 'edge', 'generic'}
 
 MAIN_PARSER = argparse.ArgumentParser(
     description=__doc__,
     formatter_class=argparse.RawDescriptionHelpFormatter)
 
 SUB_PARSERS = MAIN_PARSER.add_subparsers(title='Commands', dest='action',
                                          metavar='[command]')
 
 ALL_COMMANDS = []
 
 
 def make_argument(*args, **kwargs):
     def _make_argument(*args, **kwargs):
         parser = kwargs.pop('parser')
         parser.add_argument(*args, **kwargs)
 
     return partial(_make_argument, *args, **kwargs)
 
 
-def argparse_command(valid_platforms=None, multi_platform=False, arguments=()):
+def argparse_command(valid_platforms=None, multi_platform=False,
+                     no_platform=False, arguments=()):
     def wrapper(func):
         def func_wrapper(*args, **kwargs):
             return func(*args, **kwargs)
 
         short_desc, long_desc = func.__doc__.split('\n\n', 1)
 
         ALL_COMMANDS.append({
             'name': func.__name__,
             'description': long_desc,
             'help_text': short_desc,
             'valid_platforms': valid_platforms or KNOWN_PLATFORMS,
             'multi_platform': multi_platform,
             'function': func,
             'arguments': arguments,
+            'no_platform': no_platform,
         })
         return func_wrapper
     return wrapper
 
 
 def make_subcommand(name, description, help_text, function, arguments):
     new_parser = SUB_PARSERS.add_parser(
-        name, description=description, help=help_text,
+        name.replace('_', '-'), description=description, help=help_text,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
 
     for argument in arguments:
         argument(parser=new_parser)
 
     new_parser.set_defaults(function=function)
     return new_parser
 
 
@@ skipping 18 matching lines @@
 
     if len(types) == 0:
         logging.error('No metadata file found in this repository. Expecting '
                       'one or more of {} to be present.'.format(
                           ', '.join('metadata.' + p for p in KNOWN_PLATFORMS)))
         build_available_subcommands._result = False
         return False
 
     for command_params in ALL_COMMANDS:
         multi_platform = command_params.pop('multi_platform')
+        no_platform = command_params.pop('no_platform')
         platforms = types.intersection(command_params.pop('valid_platforms'))
         if len(platforms) > 1:
             if multi_platform:
                 help_text = ('Multiple types may be specifed (each preceded '
                              'by -t/--type)')
                 action = 'append'
             else:
                 help_text = None
                 action = 'store'
-
-            command_params['arguments'] += (
-                make_argument('-t', '--type', dest='platform', required=True,
-                              choices=platforms, action=action,
-                              help=help_text),
-            )
+            if not no_platform:
+                command_params['arguments'] += (
+                    make_argument('-t', '--type', dest='platform',
+                                  required=True, choices=platforms,
+                                  action=action, help=help_text),
+                )
             make_subcommand(**command_params)
         elif len(platforms) == 1:
             sub_parser = make_subcommand(**command_params)
             sub_parser.set_defaults(platform=platforms.pop())
 
     build_available_subcommands._result = True
     return True
 
 
 build_available_subcommands._result = None
@@ skipping 245 matching lines @@
 def updatepsl(base_dir, **kwargs):
     """Update Public Suffix List.
 
     Downloads Public Suffix List (see http://publicsuffix.org/) and generates
     lib/publicSuffixList.js from it.
     """
     import buildtools.publicSuffixListUpdater as publicSuffixListUpdater
     publicSuffixListUpdater.updatePSL(base_dir)
 
 
+@argparse_command(no_platform=True)
+def lint_gitlab_ci(base_dir, **kwargs):
+    """Lint the .gitlab-ci.yaml file.
+
+    Test the .gitlab-ci.yaml file for validity. (Note: You need to have PyYAML
+    installed.)
+    """
+    import yaml
+    filename = '.gitlab-ci.yml'
+    try:
+        with io.open(os.path.join(base_dir, filename), 'rt') as fp:
+            yaml_data = yaml.load(fp.read())
+
+        post_data = {'content': json.dumps(yaml_data)}
+        request = urllib2.Request('https://gitlab.com/api/v4/ci/lint/',
+                                  data=urlencode(post_data))
+        print urllib2.urlopen(request).read()

[Vasily Kuznetsov, 2018/07/09 17:59:48]

Could we maybe set the return code based on the results of the linting? Seems
like it would be a useful feature. Or is the response too complicated to
interpret?

[tlucas, 2018/07/10 09:41:35]

It's not hard to interpret at all ( on success it returns
'{"status":"valid","errors":[]}' ) but i would stick to 0 or 1 as a return code.
What do you think?

+    except IOError:
+        print 'No valid {} found.'.format(filename)
+
+
 def process_args(base_dir, *args):
     if build_available_subcommands(base_dir):
         MAIN_PARSER.set_defaults(base_dir=base_dir)
 
         # If no args are provided, this module is run directly from the command
         # line. argparse will take care of consuming sys.argv.
         arguments = MAIN_PARSER.parse_args(args if len(args) > 0 else None)
 
         function = arguments.function
         del arguments.function
         function(**vars(arguments))