From 8ce86ac2c992cdee578493c6a93de4f8eca52957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Lemoine?= Date: Wed, 6 Mar 2013 18:14:09 +0100 Subject: [PATCH 1/5] Add depswriter scripts Copied from http://closure-library.googlecode.com/git/closure/bin/build/ --- bin/closure/depswriter.py | 202 ++++++++++++++++++++++++++++++++++++++ bin/closure/source.py | 114 +++++++++++++++++++++ bin/closure/treescan.py | 78 +++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 bin/closure/depswriter.py create mode 100644 bin/closure/source.py create mode 100644 bin/closure/treescan.py diff --git a/bin/closure/depswriter.py b/bin/closure/depswriter.py new file mode 100644 index 0000000000..dfecc4bf74 --- /dev/null +++ b/bin/closure/depswriter.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python +# +# Copyright 2009 The Closure Library Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Generates out a Closure deps.js file given a list of JavaScript sources. + +Paths can be specified as arguments or (more commonly) specifying trees +with the flags (call with --help for descriptions). + +Usage: depswriter.py [path/to/js1.js [path/to/js2.js] ...] +""" + +import logging +import optparse +import os +import posixpath +import shlex +import sys + +import source +import treescan + + +__author__ = 'nnaze@google.com (Nathan Naze)' + + +def MakeDepsFile(source_map): + """Make a generated deps file. + + Args: + source_map: A dict map of the source path to source.Source object. + + Returns: + str, A generated deps file source. + """ + + # Write in path alphabetical order + paths = sorted(source_map.keys()) + + lines = [] + + for path in paths: + js_source = source_map[path] + + # We don't need to add entries that don't provide anything. + if js_source.provides: + lines.append(_GetDepsLine(path, js_source)) + + return ''.join(lines) + + +def _GetDepsLine(path, js_source): + """Get a deps.js file string for a source.""" + + provides = sorted(js_source.provides) + requires = sorted(js_source.requires) + + return 'goog.addDependency(\'%s\', %s, %s);\n' % (path, provides, requires) + + +def _GetOptionsParser(): + """Get the options parser.""" + + parser = optparse.OptionParser(__doc__) + + parser.add_option('--output_file', + dest='output_file', + action='store', + help=('If specified, write output to this path instead of ' + 'writing to standard output.')) + parser.add_option('--root', + dest='roots', + default=[], + action='append', + help='A root directory to scan for JS source files. ' + 'Paths of JS files in generated deps file will be ' + 'relative to this path. This flag may be specified ' + 'multiple times.') + parser.add_option('--root_with_prefix', + dest='roots_with_prefix', + default=[], + action='append', + help='A root directory to scan for JS source files, plus ' + 'a prefix (if either contains a space, surround with ' + 'quotes). Paths in generated deps file will be relative ' + 'to the root, but preceded by the prefix. This flag ' + 'may be specified multiple times.') + parser.add_option('--path_with_depspath', + dest='paths_with_depspath', + default=[], + action='append', + help='A path to a source file and an alternate path to ' + 'the file in the generated deps file (if either contains ' + 'a space, surround with whitespace). This flag may be ' + 'specified multiple times.') + return parser + + +def _NormalizePathSeparators(path): + """Replaces OS-specific path separators with POSIX-style slashes. + + Args: + path: str, A file path. + + Returns: + str, The path with any OS-specific path separators (such as backslash on + Windows) replaced with URL-compatible forward slashes. A no-op on systems + that use POSIX paths. + """ + return path.replace(os.sep, posixpath.sep) + + +def _GetRelativePathToSourceDict(root, prefix=''): + """Scans a top root directory for .js sources. + + Args: + root: str, Root directory. + prefix: str, Prefix for returned paths. + + Returns: + dict, A map of relative paths (with prefix, if given), to source.Source + objects. + """ + # Remember and restore the cwd when we're done. We work from the root so + # that paths are relative from the root. + start_wd = os.getcwd() + os.chdir(root) + + path_to_source = {} + for path in treescan.ScanTreeForJsFiles('.'): + prefixed_path = _NormalizePathSeparators(os.path.join(prefix, path)) + path_to_source[prefixed_path] = source.Source(source.GetFileContents(path)) + + os.chdir(start_wd) + + return path_to_source + + +def _GetPair(s): + """Return a string as a shell-parsed tuple. Two values expected.""" + try: + # shlex uses '\' as an escape character, so they must be escaped. + s = s.replace('\\', '\\\\') + first, second = shlex.split(s) + return (first, second) + except: + raise Exception('Unable to parse input line as a pair: %s' % s) + + +def main(): + """CLI frontend to MakeDepsFile.""" + logging.basicConfig(format=(sys.argv[0] + ': %(message)s'), + level=logging.INFO) + options, args = _GetOptionsParser().parse_args() + + path_to_source = {} + + # Roots without prefixes + for root in options.roots: + path_to_source.update(_GetRelativePathToSourceDict(root)) + + # Roots with prefixes + for root_and_prefix in options.roots_with_prefix: + root, prefix = _GetPair(root_and_prefix) + path_to_source.update(_GetRelativePathToSourceDict(root, prefix=prefix)) + + # Source paths + for path in args: + path_to_source[path] = source.Source(source.GetFileContents(path)) + + # Source paths with alternate deps paths + for path_with_depspath in options.paths_with_depspath: + srcpath, depspath = _GetPair(path_with_depspath) + path_to_source[depspath] = source.Source(source.GetFileContents(srcpath)) + + # Make our output pipe. + if options.output_file: + out = open(options.output_file, 'w') + else: + out = sys.stdout + + out.write('// This file was autogenerated by %s.\n' % sys.argv[0]) + out.write('// Please do not edit.\n') + + out.write(MakeDepsFile(path_to_source)) + + +if __name__ == '__main__': + main() diff --git a/bin/closure/source.py b/bin/closure/source.py new file mode 100644 index 0000000000..c2ee1fbb26 --- /dev/null +++ b/bin/closure/source.py @@ -0,0 +1,114 @@ +# Copyright 2009 The Closure Library Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Scans a source JS file for its provided and required namespaces. + +Simple class to scan a JavaScript file and express its dependencies. +""" + +__author__ = 'nnaze@google.com' + + +import re + +_BASE_REGEX_STRING = '^\s*goog\.%s\(\s*[\'"](.+)[\'"]\s*\)' +_PROVIDE_REGEX = re.compile(_BASE_REGEX_STRING % 'provide') +_REQUIRES_REGEX = re.compile(_BASE_REGEX_STRING % 'require') + +# This line identifies base.js and should match the line in that file. +_GOOG_BASE_LINE = ( + 'var goog = goog || {}; // Identifies this file as the Closure base.') + + +class Source(object): + """Scans a JavaScript source for its provided and required namespaces.""" + + # Matches a "/* ... */" comment. + # Note: We can't definitively distinguish a "/*" in a string literal without a + # state machine tokenizer. We'll assume that a line starting with whitespace + # and "/*" is a comment. + _COMMENT_REGEX = re.compile( + r""" + ^\s* # Start of a new line and whitespace + /\* # Opening "/*" + .*? # Non greedy match of any characters (including newlines) + \*/ # Closing "*/""", + re.MULTILINE | re.DOTALL | re.VERBOSE) + + def __init__(self, source): + """Initialize a source. + + Args: + source: str, The JavaScript source. + """ + + self.provides = set() + self.requires = set() + + self._source = source + self._ScanSource() + + def __str__(self): + return 'Source %s' % self._path + + def GetSource(self): + """Get the source as a string.""" + return self._source + + @classmethod + def _StripComments(cls, source): + return cls._COMMENT_REGEX.sub('', source) + + def _ScanSource(self): + """Fill in provides and requires by scanning the source.""" + + source = self._StripComments(self.GetSource()) + + source_lines = source.splitlines() + for line in source_lines: + match = _PROVIDE_REGEX.match(line) + if match: + self.provides.add(match.group(1)) + match = _REQUIRES_REGEX.match(line) + if match: + self.requires.add(match.group(1)) + + # Closure's base file implicitly provides 'goog'. + for line in source_lines: + if line == _GOOG_BASE_LINE: + if len(self.provides) or len(self.requires): + raise Exception( + 'Base files should not provide or require namespaces.') + self.provides.add('goog') + + +def GetFileContents(path): + """Get a file's contents as a string. + + Args: + path: str, Path to file. + + Returns: + str, Contents of file. + + Raises: + IOError: An error occurred opening or reading the file. + + """ + fileobj = open(path) + try: + return fileobj.read() + finally: + fileobj.close() diff --git a/bin/closure/treescan.py b/bin/closure/treescan.py new file mode 100644 index 0000000000..6694593aab --- /dev/null +++ b/bin/closure/treescan.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# +# Copyright 2010 The Closure Library Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Shared utility functions for scanning directory trees.""" + +import os +import re + + +__author__ = 'nnaze@google.com (Nathan Naze)' + + +# Matches a .js file path. +_JS_FILE_REGEX = re.compile(r'^.+\.js$') + + +def ScanTreeForJsFiles(root): + """Scans a directory tree for JavaScript files. + + Args: + root: str, Path to a root directory. + + Returns: + An iterable of paths to JS files, relative to cwd. + """ + return ScanTree(root, path_filter=_JS_FILE_REGEX) + + +def ScanTree(root, path_filter=None, ignore_hidden=True): + """Scans a directory tree for files. + + Args: + root: str, Path to a root directory. + path_filter: A regular expression filter. If set, only paths matching + the path_filter are returned. + ignore_hidden: If True, do not follow or return hidden directories or files + (those starting with a '.' character). + + Yields: + A string path to files, relative to cwd. + """ + + def OnError(os_error): + raise os_error + + for dirpath, dirnames, filenames in os.walk(root, onerror=OnError): + # os.walk allows us to modify dirnames to prevent decent into particular + # directories. Avoid hidden directories. + for dirname in dirnames: + if ignore_hidden and dirname.startswith('.'): + dirnames.remove(dirname) + + for filename in filenames: + + # nothing that starts with '.' + if ignore_hidden and filename.startswith('.'): + continue + + fullpath = os.path.join(dirpath, filename) + + if path_filter and not path_filter.match(fullpath): + continue + + yield os.path.normpath(fullpath) From 74b8fea679765027b03e903f951238dfb16658eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Lemoine?= Date: Wed, 6 Mar 2013 19:18:40 +0100 Subject: [PATCH 2/5] Extent hostexamples target to copy ol and closure --- build.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 5294577271..826614a406 100755 --- a/build.py +++ b/build.py @@ -30,6 +30,7 @@ else: variables.GIT = 'git' variables.GJSLINT = 'gjslint' variables.JAVA = 'java' + variables.JAR = 'jar' variables.JSDOC = 'jsdoc' variables.PYTHON = 'python' variables.PHANTOMJS = 'phantomjs' @@ -359,6 +360,37 @@ def jsdoc_BRANCH_timestamp(t): t.touch() +def split_example_file(example, dst_dir): + lines = open(example).readlines() + + target_lines = [] + target_require_lines = [] + + found_requires = False + found_code = False + for line in lines: + m = re.match(r'goog.require\(\'(.*)\'\);', line) + if m: + found_requires = True + target_require_lines.append(line) + elif found_requires: + if found_code or line not in ('\n', '\r\n'): + found_code = True + target_lines.append(line) + + target = open( + os.path.join(dst_dir, os.path.basename(example)), 'w') + target_require = open( + os.path.join(dst_dir, + os.path.basename(example).replace('.js', '-require.js')), 'w') + + target.writelines(target_lines) + target.close() + + target_require.writelines(target_require_lines) + target_require.close() + + @target('hostexamples', 'build', 'examples', phony=True) def hostexamples(t): examples_dir = 'build/gh-pages/%(BRANCH)s/examples' @@ -367,8 +399,9 @@ def hostexamples(t): t.makedirs(examples_dir) t.rm_rf(build_dir) t.makedirs(build_dir) - t.cp(EXAMPLES, (path.replace('.html', '.js') for path in EXAMPLES), - 'examples/examples.css', examples_dir) + t.cp(EXAMPLES, 'examples/examples.css', examples_dir) + for example in [path.replace('.html', '.js') for path in EXAMPLES]: + split_example_file(example, examples_dir % vars(variables)) t.cp_r('examples/data', examples_dir + '/data') t.cp_r('examples/bootstrap', examples_dir + '/bootstrap') t.cp_r('examples/font-awesome', examples_dir + '/font-awesome') @@ -378,6 +411,19 @@ def hostexamples(t): t.cp('examples/example-list.html', examples_dir + '/index.html') t.cp('examples/example-list.js', 'examples/example-list.xml', 'examples/Jugl.js', examples_dir) + t.rm_rf('build/gh-pages/%(BRANCH)s/closure-library') + t.makedirs('build/gh-pages/%(BRANCH)s/closure-library') + with t.chdir('build/gh-pages/%(BRANCH)s/closure-library'): + t.run('%(JAR)s', 'xf', '../../../../' + PLOVR_JAR, 'closure') + t.run('%(JAR)s', 'xf', '../../../../' + PLOVR_JAR, 'third_party') + t.rm_rf('build/gh-pages/%(BRANCH)s/ol') + t.makedirs('build/gh-pages/%(BRANCH)s/ol') + t.cp_r('src/ol', 'build/gh-pages/%(BRANCH)s/ol/ol') + t.run('%(PYTHON)s', 'bin/closure/depswriter.py', + '--root_with_prefix', 'src ../../../ol', + '--root', 'build/gh-pages/%(BRANCH)s/closure-library/closure/goog', + '--root_with_prefix', 'build/gh-pages/%(BRANCH)s/closure-library/third_party ../../third_party', + '--output_file', 'build/gh-pages/%(BRANCH)s/build/ol-deps.js') @target('check-examples', 'hostexamples', phony=True) From 757401313ba4d89df80c58073dadd4ae08f3d0f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Lemoine?= Date: Wed, 6 Mar 2013 19:19:11 +0100 Subject: [PATCH 3/5] Extent hosted loader to support debug mode --- build/loader_hosted_examples.js | 34 ++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/build/loader_hosted_examples.js b/build/loader_hosted_examples.js index 92cefa8e76..19c5d2ae04 100644 --- a/build/loader_hosted_examples.js +++ b/build/loader_hosted_examples.js @@ -1,17 +1,15 @@ /** - * * Loader to add ol.css, ol.js and the example-specific js file to the * documents. * * This loader is used for the hosted examples. It is used in place of the * development loader (examples/loader.js). * - * ol.css, ol.js, ol-simple.js, and ol-whitespace.js are built with - * Plovr/Closure. `build.py build` builds them. They are located in the - * ../build/ directory, relatively to this script. + * ol.css, ol.js, ol-simple.js, ol-whitespace.js, and ol-deps.js are built + * by OL3's build.py script. They are located in the ../build/ directory, + * relatively to this script. * - * The script should be named loader.js. So it needs to be renamed to - * loader.js from loader_hosted_examples.js. + * The script must be named loader.js. * * Usage: * @@ -59,17 +57,23 @@ var oljs = 'ol.js', mode; if ('mode' in pageParams) { mode = pageParams.mode.toLowerCase(); - if (mode != 'advanced') { + if (mode == 'debug') { + mode = 'raw'; + } + if (mode != 'advanced' && mode != 'raw') { oljs = 'ol-' + mode + '.js'; } } - document.write(''); - document.write('' + - ''); - document.write('' + - ''); + var scriptId = encodeURIComponent(scriptParams.id); + document.write(''); + if (mode != 'raw') { + document.write(''); + } else { + window.CLOSURE_NO_DEPS = true; // we've got our own deps file + document.write(''); + document.write(''); + document.write(''); + } + document.write(''); }()); From 306447fac7d3d3a2af926305288199dbec24f5b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Lemoine?= Date: Wed, 6 Mar 2013 20:45:08 +0100 Subject: [PATCH 4/5] Update readme.md --- readme.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/readme.md b/readme.md index b94f437aa9..33f8844149 100644 --- a/readme.md +++ b/readme.md @@ -7,10 +7,15 @@ The examples are hosted on GitHub (as GitHub pages): http://openlayers.github.com/ol3/master/examples/. -By default the examples use the `ol.js` script, which is compiled using Closure Compiler's ADVANVCED mode. -By appending `?mode=simple` or `?mode=whitespace` to the URL you can make the example page load `ol-simple.js` -or `ol-whitespace.js` instead of `ol.js`. As their names suggest it, `ol-simple.js` and `ol-whitespace.js` -are compiled using the SIMPLE and WHITESPACE modes, respectively. For example: +By default the examples use the `ol.js` script, which is compiled using Closure +Compiler's ADVANCED mode. By appending `?mode=simple` or `?mode=whitespace` to +the URL you can make the example page load the `ol-simple.js` or +`ol-whitespace.js` builds instead of `ol.js`. As their names suggest it, +`ol-simple.js` and `ol-whitespace.js` are compiled using the SIMPLE and +WHITESPACE modes, respectively. And by appending `?mode=debug` or `?mode=raw` +you will make the example work in full debug mode. + +For example: http://openlayers.github.com/ol3/master/examples/full-screen.html?mode=simple. ## Build OpenLayers 3 @@ -47,12 +52,14 @@ on GitHub. Start by executing the `hostexamples` build target: $ ./build.py hostexamples -This will build `ol.js`, `ol-simple.js`, `ol-whitespace.js`, and `ol.css`, create the examples index page, -and copy everything to `build/gh-pages//`, where `` is the name of the local -checked out Git branch. You can now open the examples index page in the browser, for example: -. To make an example use `ol-simple.js` or -`ol-whitespace.js` instead of `ol.js` append `?mode=simple` or `?mode=whitespace` to the example -URL. +This will build `ol.js`, `ol-simple.js`, `ol-whitespace.js`, and `ol.css`, +create the examples index page, and copy everything to +`build/gh-pages//`, where `` is the name of the local +checked out Git branch. You can now open the examples index page in the +browser, for example: . +To make an example use `ol-simple.js` or `ol-whitespace.js` instead of `ol.js` +append `?mode=simple` or `?mode=whitespace` to the example URL. And append +`?mode=debug` or `?mode=raw` to make the example work in full debug mode. ## Run tests From bbc8249e28ec85e99b4a014ce8a07c4b8eb20c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Lemoine?= Date: Thu, 7 Mar 2013 10:33:40 +0100 Subject: [PATCH 5/5] Check the examples in raw mode as well --- build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 826614a406..7c13dcfd45 100755 --- a/build.py +++ b/build.py @@ -430,7 +430,9 @@ def hostexamples(t): def check_examples(t): directory = 'build/gh-pages/%(BRANCH)s/' examples = ['build/gh-pages/%(BRANCH)s/' + e for e in EXAMPLES] - all_examples = [e + '?mode=whitespace' for e in examples] + \ + all_examples = \ + [e + '?mode=raw' for e in examples] + \ + [e + '?mode=whitespace' for e in examples] + \ [e + '?mode=simple' for e in examples] + \ examples for example in all_examples: