Issue #5352 - generate_static_pages cannot deal with directories being turned into regular pages

cms/bin/generate_static_pages.py

 # This file is part of the Adblock Plus web scripts,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import re
 import codecs
 import ConfigParser
 import logging
 from argparse import ArgumentParser
 import shutil
 
 from cms.utils import get_page_params, process_page
 from cms.sources import create_source
 
 MIN_TRANSLATED = 0.3
 
 
-def resolve_dirs(partial_path, path_parts):

[Vasily Kuznetsov, 2018/09/27 11:05:07]

It seems like resolve_dirs doesn't quite capture what this function is doing.
Basically it creates directories unless they exist so it's kind of like
os.makedirs() but it also handles some special cases (like stale files in the
way). My first thought is "makedirs2()" but that's kind of lame. Maybe
"ensure_dir" (because it ensures that the target directory exists and is a
directory)?

[Tudor Avram, 2018/10/04 06:34:18]

Done.

+def ensure_dirs(partial_path, path_parts):
     """Create an entire path of directories.
 
     This is a recursive function, that also treats these special cases, if
     the partial path we reached so far exists and is:
 
     1. a directory - do nothing, just move on,
     2. a file - remove the file, create a directory in its place, and move on.
     3. neither a file, nor a directory - raise and exception.
 
     Parameters
@@ skipping 15 matching lines @@
     elif os.path.exists(partial_path) and not os.path.isdir(partial_path):
         raise Exception('The object at {} is not recognisable! It is neither '
                         'a file, nor a directory!'.format(partial_path))
 
     if not os.path.isdir(partial_path):
         os.mkdir(partial_path)
 
     if len(path_parts) == 0:
         return
 
-    resolve_dirs(os.path.join(partial_path, path_parts[0]), path_parts[1:])
+    ensure_dirs(os.path.join(partial_path, path_parts[0]), path_parts[1:])
 
 
 def is_in_previous_version(path, new_contents, encoding):
     """Test if a file we try to create already is in the output directory.
 
     It tests if the pre-existent file has all the expected content.
     It also handles the following two cases:
 
     1. The path is a directory - If this happens, it removes the directory from
     the file tree.
@@ skipping 28 matching lines @@
                 return True
     elif os.path.isdir(path):
         shutil.rmtree(path)
     elif os.path.exists(path):
         raise Exception('The object at {} is not recognisable! It is '
                         'neither a file, nor a directory!'.format(path))
 
     return False
 
 
 def generate_pages(repo, output_dir):

[Tudor Avram, 2018/10/04 06:34:18]

Nit: Also, does `repo` really make sense here now? I know that previously, we
used to have Mercurial sources. But as that's not the case, maybe `website` or
`raw_dir` would make a little bit more sense?
What do you think? 

(This would be handled in a separate ticket, obviously).

[Vasily Kuznetsov, 2018/10/05 09:40:43]

Yes, makes sense, `repo` is a confusing name. Perhaps `source_dir` would work so
we have `generate_pages(source_dir, output_dir)`.

     known_files = set()
 
     def write_file(path_parts, contents, binary=False):
         encoding = None if binary else 'utf-8'
         outfile = os.path.join(output_dir, *path_parts)
         if outfile in known_files:
             logging.warning('File %s has multiple sources', outfile)
             return
         known_files.add(outfile)
 
         if is_in_previous_version(outfile, contents, encoding):
             return
 
-        resolve_dirs(output_dir, path_parts[:-1])
+        ensure_dirs(output_dir, path_parts[:-1])
 
         with codecs.open(outfile, 'wb', encoding=encoding) as handle:
             handle.write(contents)
 
     with create_source(repo, cached=True) as source:
         config = source.read_config()
         defaultlocale = config.get('general', 'defaultlocale')
         locales = list(source.list_locales())
         if defaultlocale not in locales:
             locales.append(defaultlocale)
@@ skipping 61 matching lines @@
                     os.rmdir(path)
     remove_unknown(output_dir)
 
 
 if __name__ == '__main__':
     parser = ArgumentParser('Convert website source to static website')
     parser.add_argument('source', help="Path to website's repository")
     parser.add_argument('output', help='Path to desired output directory')
     args = parser.parse_args()
     generate_pages(args.source, args.output)