Merge pull request #302 from elemoine/debug

Hosted examples in debug mode

bin/closure/depswriter.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+#
+# Copyright 2009 The Closure Library Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Generates out a Closure deps.js file given a list of JavaScript sources.
+
+Paths can be specified as arguments or (more commonly) specifying trees
+with the flags (call with --help for descriptions).
+
+Usage: depswriter.py [path/to/js1.js [path/to/js2.js] ...]
+"""
+
+import logging
+import optparse
+import os
+import posixpath
+import shlex
+import sys
+
+import source
+import treescan
+
+
+__author__ = 'nnaze@google.com (Nathan Naze)'
+
+
+def MakeDepsFile(source_map):
+  """Make a generated deps file.
+
+  Args:
+    source_map: A dict map of the source path to source.Source object.
+
+  Returns:
+    str, A generated deps file source.
+  """
+
+  # Write in path alphabetical order
+  paths = sorted(source_map.keys())
+
+  lines = []
+
+  for path in paths:
+    js_source = source_map[path]
+
+    # We don't need to add entries that don't provide anything.
+    if js_source.provides:
+      lines.append(_GetDepsLine(path, js_source))
+
+  return ''.join(lines)
+
+
+def _GetDepsLine(path, js_source):
+  """Get a deps.js file string for a source."""
+
+  provides = sorted(js_source.provides)
+  requires = sorted(js_source.requires)
+
+  return 'goog.addDependency(\'%s\', %s, %s);\n' % (path, provides, requires)
+
+
+def _GetOptionsParser():
+  """Get the options parser."""
+
+  parser = optparse.OptionParser(__doc__)
+
+  parser.add_option('--output_file',
+                    dest='output_file',
+                    action='store',
+                    help=('If specified, write output to this path instead of '
+                          'writing to standard output.'))
+  parser.add_option('--root',
+                    dest='roots',
+                    default=[],
+                    action='append',
+                    help='A root directory to scan for JS source files. '
+                    'Paths of JS files in generated deps file will be '
+                    'relative to this path.  This flag may be specified '
+                    'multiple times.')
+  parser.add_option('--root_with_prefix',
+                    dest='roots_with_prefix',
+                    default=[],
+                    action='append',
+                    help='A root directory to scan for JS source files, plus '
+                    'a prefix (if either contains a space, surround with '
+                    'quotes).  Paths in generated deps file will be relative '
+                    'to the root, but preceded by the prefix.  This flag '
+                    'may be specified multiple times.')
+  parser.add_option('--path_with_depspath',
+                    dest='paths_with_depspath',
+                    default=[],
+                    action='append',
+                    help='A path to a source file and an alternate path to '
+                    'the file in the generated deps file (if either contains '
+                    'a space, surround with whitespace). This flag may be '
+                    'specified multiple times.')
+  return parser
+
+
+def _NormalizePathSeparators(path):
+  """Replaces OS-specific path separators with POSIX-style slashes.
+
+  Args:
+    path: str, A file path.
+
+  Returns:
+    str, The path with any OS-specific path separators (such as backslash on
+      Windows) replaced with URL-compatible forward slashes. A no-op on systems
+      that use POSIX paths.
+  """
+  return path.replace(os.sep, posixpath.sep)
+
+
+def _GetRelativePathToSourceDict(root, prefix=''):
+  """Scans a top root directory for .js sources.
+
+  Args:
+    root: str, Root directory.
+    prefix: str, Prefix for returned paths.
+
+  Returns:
+    dict, A map of relative paths (with prefix, if given), to source.Source
+      objects.
+  """
+  # Remember and restore the cwd when we're done. We work from the root so
+  # that paths are relative from the root.
+  start_wd = os.getcwd()
+  os.chdir(root)
+
+  path_to_source = {}
+  for path in treescan.ScanTreeForJsFiles('.'):
+    prefixed_path = _NormalizePathSeparators(os.path.join(prefix, path))
+    path_to_source[prefixed_path] = source.Source(source.GetFileContents(path))
+
+  os.chdir(start_wd)
+
+  return path_to_source
+
+
+def _GetPair(s):
+  """Return a string as a shell-parsed tuple.  Two values expected."""
+  try:
+    # shlex uses '\' as an escape character, so they must be escaped.
+    s = s.replace('\\', '\\\\')
+    first, second = shlex.split(s)
+    return (first, second)
+  except:
+    raise Exception('Unable to parse input line as a pair: %s' % s)
+
+
+def main():
+  """CLI frontend to MakeDepsFile."""
+  logging.basicConfig(format=(sys.argv[0] + ': %(message)s'),
+                      level=logging.INFO)
+  options, args = _GetOptionsParser().parse_args()
+
+  path_to_source = {}
+
+  # Roots without prefixes
+  for root in options.roots:
+    path_to_source.update(_GetRelativePathToSourceDict(root))
+
+  # Roots with prefixes
+  for root_and_prefix in options.roots_with_prefix:
+    root, prefix = _GetPair(root_and_prefix)
+    path_to_source.update(_GetRelativePathToSourceDict(root, prefix=prefix))
+
+  # Source paths
+  for path in args:
+    path_to_source[path] = source.Source(source.GetFileContents(path))
+
+  # Source paths with alternate deps paths
+  for path_with_depspath in options.paths_with_depspath:
+    srcpath, depspath = _GetPair(path_with_depspath)
+    path_to_source[depspath] = source.Source(source.GetFileContents(srcpath))
+
+  # Make our output pipe.
+  if options.output_file:
+    out = open(options.output_file, 'w')
+  else:
+    out = sys.stdout
+
+  out.write('// This file was autogenerated by %s.\n' % sys.argv[0])
+  out.write('// Please do not edit.\n')
+
+  out.write(MakeDepsFile(path_to_source))
+
+
+if __name__ == '__main__':
+  main()

bin/closure/source.py

@@ -0,0 +1,114 @@
+# Copyright 2009 The Closure Library Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Scans a source JS file for its provided and required namespaces.
+
+Simple class to scan a JavaScript file and express its dependencies.
+"""
+
+__author__ = 'nnaze@google.com'
+
+
+import re
+
+_BASE_REGEX_STRING = '^\s*goog\.%s\(\s*[\'"](.+)[\'"]\s*\)'
+_PROVIDE_REGEX = re.compile(_BASE_REGEX_STRING % 'provide')
+_REQUIRES_REGEX = re.compile(_BASE_REGEX_STRING % 'require')
+
+# This line identifies base.js and should match the line in that file.
+_GOOG_BASE_LINE = (
+    'var goog = goog || {}; // Identifies this file as the Closure base.')
+
+
+class Source(object):
+  """Scans a JavaScript source for its provided and required namespaces."""
+
+  # Matches a "/* ... */" comment.
+  # Note: We can't definitively distinguish a "/*" in a string literal without a
+  # state machine tokenizer. We'll assume that a line starting with whitespace
+  # and "/*" is a comment.
+  _COMMENT_REGEX = re.compile(
+      r"""
+      ^\s*   # Start of a new line and whitespace
+      /\*    # Opening "/*"
+      .*?    # Non greedy match of any characters (including newlines)
+      \*/    # Closing "*/""",
+      re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+  def __init__(self, source):
+    """Initialize a source.
+
+    Args:
+      source: str, The JavaScript source.
+    """
+
+    self.provides = set()
+    self.requires = set()
+
+    self._source = source
+    self._ScanSource()
+
+  def __str__(self):
+    return 'Source %s' % self._path
+
+  def GetSource(self):
+    """Get the source as a string."""
+    return self._source
+
+  @classmethod
+  def _StripComments(cls, source):
+    return cls._COMMENT_REGEX.sub('', source)
+
+  def _ScanSource(self):
+    """Fill in provides and requires by scanning the source."""
+
+    source = self._StripComments(self.GetSource())
+
+    source_lines = source.splitlines()
+    for line in source_lines:
+      match = _PROVIDE_REGEX.match(line)
+      if match:
+        self.provides.add(match.group(1))
+      match = _REQUIRES_REGEX.match(line)
+      if match:
+        self.requires.add(match.group(1))
+
+    # Closure's base file implicitly provides 'goog'.
+    for line in source_lines:
+      if line == _GOOG_BASE_LINE:
+        if len(self.provides) or len(self.requires):
+          raise Exception(
+              'Base files should not provide or require namespaces.')
+        self.provides.add('goog')
+
+
+def GetFileContents(path):
+  """Get a file's contents as a string.
+
+  Args:
+    path: str, Path to file.
+
+  Returns:
+    str, Contents of file.
+
+  Raises:
+    IOError: An error occurred opening or reading the file.
+
+  """
+  fileobj = open(path)
+  try:
+    return fileobj.read()
+  finally:
+    fileobj.close()

bin/closure/treescan.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+#
+# Copyright 2010 The Closure Library Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Shared utility functions for scanning directory trees."""
+
+import os
+import re
+
+
+__author__ = 'nnaze@google.com (Nathan Naze)'
+
+
+# Matches a .js file path.
+_JS_FILE_REGEX = re.compile(r'^.+\.js$')
+
+
+def ScanTreeForJsFiles(root):
+  """Scans a directory tree for JavaScript files.
+
+  Args:
+    root: str, Path to a root directory.
+
+  Returns:
+    An iterable of paths to JS files, relative to cwd.
+  """
+  return ScanTree(root, path_filter=_JS_FILE_REGEX)
+
+
+def ScanTree(root, path_filter=None, ignore_hidden=True):
+  """Scans a directory tree for files.
+
+  Args:
+    root: str, Path to a root directory.
+    path_filter: A regular expression filter.  If set, only paths matching
+      the path_filter are returned.
+    ignore_hidden: If True, do not follow or return hidden directories or files
+      (those starting with a '.' character).
+
+  Yields:
+    A string path to files, relative to cwd.
+  """
+
+  def OnError(os_error):
+    raise os_error
+
+  for dirpath, dirnames, filenames in os.walk(root, onerror=OnError):
+    # os.walk allows us to modify dirnames to prevent decent into particular
+    # directories.  Avoid hidden directories.
+    for dirname in dirnames:
+      if ignore_hidden and dirname.startswith('.'):
+        dirnames.remove(dirname)
+
+    for filename in filenames:
+
+      # nothing that starts with '.'
+      if ignore_hidden and filename.startswith('.'):
+        continue
+
+      fullpath = os.path.join(dirpath, filename)
+
+      if path_filter and not path_filter.match(fullpath):
+        continue
+
+      yield os.path.normpath(fullpath)

build.py

@@ -30,6 +30,7 @@ else:
     variables.GIT = 'git'
     variables.GJSLINT = 'gjslint'
     variables.JAVA = 'java'
+    variables.JAR = 'jar'
     variables.JSDOC = 'jsdoc'
     variables.PYTHON = 'python'
     variables.PHANTOMJS = 'phantomjs'
@@ -359,6 +360,37 @@ def jsdoc_BRANCH_timestamp(t):
     t.touch()
 
 
+def split_example_file(example, dst_dir):
+    lines = open(example).readlines()
+
+    target_lines = []
+    target_require_lines = []
+
+    found_requires = False
+    found_code = False
+    for line in lines:
+        m = re.match(r'goog.require\(\'(.*)\'\);', line)
+        if m:
+            found_requires = True
+            target_require_lines.append(line)
+        elif found_requires:
+            if found_code or line not in ('\n', '\r\n'):
+                found_code = True
+                target_lines.append(line)
+
+    target = open(
+        os.path.join(dst_dir, os.path.basename(example)), 'w')
+    target_require = open(
+        os.path.join(dst_dir,
+            os.path.basename(example).replace('.js', '-require.js')), 'w')
+
+    target.writelines(target_lines)
+    target.close()
+
+    target_require.writelines(target_require_lines)
+    target_require.close()
+
+
 @target('hostexamples', 'build', 'examples', phony=True)
 def hostexamples(t):
     examples_dir = 'build/gh-pages/%(BRANCH)s/examples'
@@ -367,8 +399,9 @@ def hostexamples(t):
     t.makedirs(examples_dir)
     t.rm_rf(build_dir)
     t.makedirs(build_dir)
-    t.cp(EXAMPLES, (path.replace('.html', '.js') for path in EXAMPLES),
-        'examples/examples.css', examples_dir)
+    t.cp(EXAMPLES, 'examples/examples.css', examples_dir)
+    for example in [path.replace('.html', '.js') for path in EXAMPLES]:
+        split_example_file(example, examples_dir % vars(variables))
     t.cp_r('examples/data', examples_dir + '/data')
     t.cp_r('examples/bootstrap', examples_dir + '/bootstrap')
     t.cp_r('examples/font-awesome', examples_dir + '/font-awesome')
@@ -378,13 +411,28 @@ def hostexamples(t):
     t.cp('examples/example-list.html', examples_dir + '/index.html')
     t.cp('examples/example-list.js', 'examples/example-list.xml',
         'examples/Jugl.js', examples_dir)
+    t.rm_rf('build/gh-pages/%(BRANCH)s/closure-library')
+    t.makedirs('build/gh-pages/%(BRANCH)s/closure-library')
+    with t.chdir('build/gh-pages/%(BRANCH)s/closure-library'):
+        t.run('%(JAR)s', 'xf', '../../../../' + PLOVR_JAR, 'closure')
+        t.run('%(JAR)s', 'xf', '../../../../' + PLOVR_JAR, 'third_party')
+    t.rm_rf('build/gh-pages/%(BRANCH)s/ol')
+    t.makedirs('build/gh-pages/%(BRANCH)s/ol')
+    t.cp_r('src/ol', 'build/gh-pages/%(BRANCH)s/ol/ol')
+    t.run('%(PYTHON)s', 'bin/closure/depswriter.py',
+        '--root_with_prefix', 'src ../../../ol',
+        '--root', 'build/gh-pages/%(BRANCH)s/closure-library/closure/goog',
+        '--root_with_prefix', 'build/gh-pages/%(BRANCH)s/closure-library/third_party ../../third_party',
+        '--output_file', 'build/gh-pages/%(BRANCH)s/build/ol-deps.js')
 
 
 @target('check-examples', 'hostexamples', phony=True)
 def check_examples(t):
     directory = 'build/gh-pages/%(BRANCH)s/'
     examples = ['build/gh-pages/%(BRANCH)s/' + e for e in EXAMPLES]
-    all_examples = [e + '?mode=whitespace' for e in examples] + \
+    all_examples = \
+        [e + '?mode=raw' for e in examples] + \
+        [e + '?mode=whitespace' for e in examples] + \
         [e + '?mode=simple' for e in examples] + \
         examples
     for example in all_examples:

build/loader_hosted_examples.js

@@ -1,17 +1,15 @@
 /**
- *
  * Loader to add ol.css, ol.js and the example-specific js file to the
  * documents.
  *
  * This loader is used for the hosted examples. It is used in place of the
  * development loader (examples/loader.js).
  *
- * ol.css, ol.js, ol-simple.js, and ol-whitespace.js are built with
- * Plovr/Closure.  `build.py build` builds them. They are located in the
- * ../build/ directory, relatively to this script.
+ * ol.css, ol.js, ol-simple.js, ol-whitespace.js, and ol-deps.js are built
+ * by OL3's build.py script. They are located in the ../build/ directory,
+ * relatively to this script.
  *
- * The script should be named loader.js. So it needs to be renamed to
- * loader.js from loader_hosted_examples.js.
+ * The script must be named loader.js.
  *
  * Usage:
  *
@@ -59,17 +57,23 @@
   var oljs = 'ol.js', mode;
   if ('mode' in pageParams) {
     mode = pageParams.mode.toLowerCase();
-    if (mode != 'advanced') {
+    if (mode == 'debug') {
+      mode = 'raw';
+    }
+    if (mode != 'advanced' && mode != 'raw') {
       oljs = 'ol-' + mode + '.js';
     }
   }
 
-  document.write('<link rel="stylesheet" href="../build/ol.css" '+
-                 'type="text/css">');
-  document.write('<scr' + 'ipt type="text/javascript" ' +
-                 'src="../build/' + oljs + '">' +
-                 '</scr' + 'ipt>');
-  document.write('<scr' + 'ipt type="text/javascript" ' +
-                 'src="' + encodeURIComponent(scriptParams.id) + '.js">' +
-                 '</scr' + 'ipt>');
+  var scriptId = encodeURIComponent(scriptParams.id);
+  document.write('<link rel="stylesheet" href="../build/ol.css" type="text/css">');
+  if (mode != 'raw') {
+    document.write('<scr' + 'ipt type="text/javascript" src="../build/' + oljs + '"></scr' + 'ipt>');
+  } else {
+    window.CLOSURE_NO_DEPS = true; // we've got our own deps file
+    document.write('<scr' + 'ipt type="text/javascript" src="../closure-library/closure/goog/base.js"></scr' + 'ipt>');
+    document.write('<scr' + 'ipt type="text/javascript" src="../build/ol-deps.js"></scr' + 'ipt>');
+    document.write('<scr' + 'ipt type="text/javascript" src="' + scriptId + '-require.js"></scr' + 'ipt>');
+  }
+  document.write('<scr' + 'ipt type="text/javascript" src="' + scriptId + '.js"></scr' + 'ipt>');
 }());

readme.md

@@ -7,10 +7,15 @@
 
 The examples are hosted on GitHub (as GitHub pages): http://openlayers.github.com/ol3/master/examples/.
 
-By default the examples use the `ol.js` script, which is compiled using Closure Compiler's ADVANVCED mode.
-By appending `?mode=simple` or `?mode=whitespace` to the URL you can make the example page load `ol-simple.js`
-or `ol-whitespace.js` instead of `ol.js`. As their names suggest it, `ol-simple.js` and `ol-whitespace.js`
-are compiled using the SIMPLE and WHITESPACE modes, respectively. For example:
+By default the examples use the `ol.js` script, which is compiled using Closure
+Compiler's ADVANCED mode.  By appending `?mode=simple` or `?mode=whitespace` to
+the URL you can make the example page load the `ol-simple.js` or
+`ol-whitespace.js` builds instead of `ol.js`. As their names suggest it,
+`ol-simple.js` and `ol-whitespace.js` are compiled using the SIMPLE and
+WHITESPACE modes, respectively. And by appending `?mode=debug` or `?mode=raw`
+you will make the example work in full debug mode.
+
+For example:
 http://openlayers.github.com/ol3/master/examples/full-screen.html?mode=simple.
 
 ## Build OpenLayers 3
@@ -47,12 +52,14 @@ on GitHub. Start by executing the `hostexamples` build target:
 
     $ ./build.py hostexamples
     
-This will build `ol.js`, `ol-simple.js`, `ol-whitespace.js`, and `ol.css`, create the examples index page,
-and copy everything to `build/gh-pages/<branch_name>/`, where `<branch_name>` is the name of the local
-checked out Git branch. You can now open the examples index page in the browser, for example:
-<http://localhost:8000/build/gh-pages/master/examples/>. To make an example use `ol-simple.js` or
-`ol-whitespace.js` instead of `ol.js` append `?mode=simple` or `?mode=whitespace` to the example
-URL.
+This will build `ol.js`, `ol-simple.js`, `ol-whitespace.js`, and `ol.css`,
+create the examples index page, and copy everything to
+`build/gh-pages/<branch_name>/`, where `<branch_name>` is the name of the local
+checked out Git branch. You can now open the examples index page in the
+browser, for example: <http://localhost:8000/build/gh-pages/master/examples/>.
+To make an example use `ol-simple.js` or `ol-whitespace.js` instead of `ol.js`
+append `?mode=simple` or `?mode=whitespace` to the example URL. And append
+`?mode=debug` or `?mode=raw` to make the example work in full debug mode.
 
 ## Run tests