From 9d0034576daa9926e12cf6fb4b4665575dc9e515 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Tue, 26 Nov 2019 08:58:16 -0500 Subject: [PATCH 1/7] start extracting secrets --- .gitignore | 1 + .../Account/Account.xcodeproj/project.pbxproj | 4 ++ .../FeedWrangler/FeedWranglerConfig.swift | 2 +- NetNewsWire.xcodeproj/project.pbxproj | 19 ++++++++++ Shared/Secrets.swift.gyb | 37 +++++++++++++++++++ updateSecrets.sh | 7 ++++ 6 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 Shared/Secrets.swift.gyb create mode 100755 updateSecrets.sh diff --git a/.gitignore b/.gitignore index 336a128d5..8497f0b61 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ fastlane/Preview.html fastlane/screenshots fastlane/test_output +/Shared/Secrets.swift diff --git a/Frameworks/Account/Account.xcodeproj/project.pbxproj b/Frameworks/Account/Account.xcodeproj/project.pbxproj index a835e2639..f622a5a2b 100644 --- a/Frameworks/Account/Account.xcodeproj/project.pbxproj +++ b/Frameworks/Account/Account.xcodeproj/project.pbxproj @@ -7,6 +7,7 @@ objects = { /* Begin PBXBuildFile section */ + 3B3A33E7238D3D6800314204 /* Secrets.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B3A33E6238D3D6800314204 /* Secrets.swift */; }; 3B826DA72385C81C00FC1ADB /* FeedWranglerAuthorizationResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B826D9E2385C81C00FC1ADB /* FeedWranglerAuthorizationResult.swift */; }; 3B826DA82385C81C00FC1ADB /* FeedWranglerFeedItem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B826D9F2385C81C00FC1ADB /* FeedWranglerFeedItem.swift */; }; 3B826DA92385C81C00FC1ADB /* FeedWranglerAPICaller.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B826DA02385C81C00FC1ADB /* FeedWranglerAPICaller.swift */; }; @@ -219,6 +220,7 @@ /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ + 3B3A33E6238D3D6800314204 /* Secrets.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Secrets.swift; path = ../../Shared/Secrets.swift; sourceTree = ""; }; 3B826D9E2385C81C00FC1ADB /* FeedWranglerAuthorizationResult.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FeedWranglerAuthorizationResult.swift; sourceTree = ""; }; 3B826D9F2385C81C00FC1ADB /* FeedWranglerFeedItem.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FeedWranglerFeedItem.swift; sourceTree = ""; }; 3B826DA02385C81C00FC1ADB /* FeedWranglerAPICaller.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FeedWranglerAPICaller.swift; sourceTree = ""; }; @@ -563,6 +565,7 @@ 848934EC1F62484F00CEBD24 = { isa = PBXGroup; children = ( + 3B3A33E6238D3D6800314204 /* Secrets.swift */, 848935101F62486800CEBD24 /* Account.swift */, 841974241F6DDCE4006346C4 /* AccountDelegate.swift */, 51BB7B83233531BC008E8144 /* AccountBehaviors.swift */, @@ -1115,6 +1118,7 @@ 9EAEC62823331C350085D7C9 /* FeedlyCategory.swift in Sources */, 3B826DAE2385C81C00FC1ADB /* FeedWranglerSubscriptionsRequest.swift in Sources */, 9E964EB823754AC400A7AF2E /* OAuthAuthorizationClient+Feedly.swift in Sources */, + 3B3A33E7238D3D6800314204 /* Secrets.swift in Sources */, 9EF1B10923590E93000A486A /* FeedlyStreamIds.swift in Sources */, 84D09623217418DC00D77525 /* FeedbinTagging.swift in Sources */, 84CAD7161FDF2E22000F0755 /* FeedbinEntry.swift in Sources */, diff --git a/Frameworks/Account/FeedWrangler/FeedWranglerConfig.swift b/Frameworks/Account/FeedWrangler/FeedWranglerConfig.swift index 16205707f..24386daab 100644 --- a/Frameworks/Account/FeedWrangler/FeedWranglerConfig.swift +++ b/Frameworks/Account/FeedWrangler/FeedWranglerConfig.swift @@ -10,7 +10,7 @@ import Foundation enum FeedWranglerConfig { static let pageSize = 100 - static let clientKey = "{FEEDWRANGLERKEY}" // Add FEED_WRANGLER_KEY = XYZ to SharedXcodeSettings/DeveloperSettings.xcconfig + static let clientKey = Secrets.FeedWranglerClientKey // Add FEED_WRANGLER_KEY = XYZ to SharedXcodeSettings/DeveloperSettings.xcconfig static let clientPath = "https://feedwrangler.net/api/v2/" static let clientURL = { URL(string: FeedWranglerConfig.clientPath)! diff --git a/NetNewsWire.xcodeproj/project.pbxproj b/NetNewsWire.xcodeproj/project.pbxproj index a9acae6f3..b25763f28 100644 --- a/NetNewsWire.xcodeproj/project.pbxproj +++ b/NetNewsWire.xcodeproj/project.pbxproj @@ -2893,6 +2893,7 @@ isa = PBXNativeTarget; buildConfigurationList = 849C647A1ED37A5D003D8FC0 /* Build configuration list for PBXNativeTarget "NetNewsWire" */; buildPhases = ( + 3B3A341B238D3FB200314204 /* Run Script: Update Secrets */, 51D6803823330CFF0097A009 /* Run Script: Update ArticleExtractorConfig.swift */, 849C645C1ED37A5D003D8FC0 /* Sources */, 517D2D82233A53D600FF3E35 /* Run Script: Reset ArticleExtractorConfig.swift */, @@ -3513,6 +3514,24 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ + 3B3A341B238D3FB200314204 /* Run Script: Update Secrets */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Run Script: Update Secrets"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "./updateSecrets.sh\n\n"; + }; 515D50802326D02600EE1167 /* Run Script: Verify No Build Settings */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; diff --git a/Shared/Secrets.swift.gyb b/Shared/Secrets.swift.gyb new file mode 100644 index 000000000..e65f74297 --- /dev/null +++ b/Shared/Secrets.swift.gyb @@ -0,0 +1,37 @@ +// Secrets.swift.gyb +%{ +import os + +def chunks(seq, size): + return (seq[i:(i + size)] for i in range(0, len(seq), size)) + +def encode(string, salt): + bytes = string.encode("UTF-8") + return [ord(bytes[i]) ^ salt[i % len(salt)] for i in range(0, len(bytes))] + +salt = [ord(byte) for byte in os.urandom(64)] +}% +enum Secrets { + + static var FeedWranglerClientKey: String { + let encoded: [UInt8] = [ + % for chunk in chunks(encode(os.environ.get('FEED_WRANGLER_KEY'), salt), 8): + ${"".join(["0x%02x, " % byte for byte in chunk])} + % end + ] + + return decode(encoded, salt: salt) + } + + private static let salt: [UInt8] = [ + % for chunk in chunks(salt, 8): + ${"".join(["0x%02x, " % byte for byte in chunk])} + % end + ] + + private static func decode(_ encoded: [UInt8], salt: [UInt8]) -> String { + String(decoding: encoded.enumerated().map { (offset, element) in + element ^ salt[offset % salt.count] + }, as: UTF8.self) + } +} \ No newline at end of file diff --git a/updateSecrets.sh b/updateSecrets.sh new file mode 100755 index 000000000..d5fa38cc9 --- /dev/null +++ b/updateSecrets.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +find . -name '*.gyb' | \ + while read file; do \ + echo "running ${file%.gyb}"; \ + gyb --line-directive '' -o "${file%.gyb}" "$file"; \ + done \ No newline at end of file From eda98d166d5f7eacd8efdaffbf348714cfb83ecb Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Wed, 27 Nov 2019 11:17:46 -0500 Subject: [PATCH 2/7] remove FeedWrangler build scripts --- .../Account/Account.xcodeproj/project.pbxproj | 38 ------------------- .../FeedWrangler/FeedWranglerConfig.swift | 2 +- 2 files changed, 1 insertion(+), 39 deletions(-) diff --git a/Frameworks/Account/Account.xcodeproj/project.pbxproj b/Frameworks/Account/Account.xcodeproj/project.pbxproj index f622a5a2b..1b300ce20 100644 --- a/Frameworks/Account/Account.xcodeproj/project.pbxproj +++ b/Frameworks/Account/Account.xcodeproj/project.pbxproj @@ -776,10 +776,8 @@ buildConfigurationList = 8489350A1F62485000CEBD24 /* Build configuration list for PBXNativeTarget "Account" */; buildPhases = ( 9E964EBB2375512300A7AF2E /* Run Script: Update OAuthAuthorizationClient+Feedly.swift */, - 3B826DCF2385CE1B00FC1ADB /* Run Script: Update FeedWranglerConfig.swift */, 848934F11F62484F00CEBD24 /* Sources */, 9E964EBC2375517100A7AF2E /* Run Script: Reset OAuthAuthorizationClient+Feedly.swift */, - 3B826DD02385CE9500FC1ADB /* Run Script: Reset FeedWranglerConfig.swift */, 848934F21F62484F00CEBD24 /* Frameworks */, 848934F31F62484F00CEBD24 /* Headers */, 848934F41F62484F00CEBD24 /* Resources */, @@ -934,42 +932,6 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ - 3B826DCF2385CE1B00FC1ADB /* Run Script: Update FeedWranglerConfig.swift */ = { - isa = PBXShellScriptBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - inputFileListPaths = ( - ); - inputPaths = ( - ); - name = "Run Script: Update FeedWranglerConfig.swift"; - outputFileListPaths = ( - ); - outputPaths = ( - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "FAILED=false\n\nif [ -z \"${FEED_WRANGLER_KEY}\" ]; then\nFAILED=true\nfi\n\nif [ \"$FAILED\" = true ]; then\necho \"Missing Feed Wrangler Key. FeedWranglerConfig.swift not changed.\"\nexit 0\nfi\n\nsed -i .tmp \"s|{FEEDWRANGLERKEY}|${FEED_WRANGLER_KEY}|g; s|{FEEDWRANGLERKEY}|${FEED_WRANGLER_KEY}|g\" \"${SRCROOT}/FeedWrangler/FeedWranglerConfig.swift\"\n\nrm -f \"${SRCROOT}/FeedWrangler/FeedWranglerConfig.swift.tmp\"\n\necho \"All Feed Wrangler env values found!\"\n\n"; - }; - 3B826DD02385CE9500FC1ADB /* Run Script: Reset FeedWranglerConfig.swift */ = { - isa = PBXShellScriptBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - inputFileListPaths = ( - ); - inputPaths = ( - ); - name = "Run Script: Reset FeedWranglerConfig.swift"; - outputFileListPaths = ( - ); - outputPaths = ( - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "git checkout \"${SRCROOT}/FeedWrangler/FeedWranglerConfig.swift\"\n"; - }; 51C8F34C234FB14B0048ED95 /* Run Script: Verify No Build Settings */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; diff --git a/Frameworks/Account/FeedWrangler/FeedWranglerConfig.swift b/Frameworks/Account/FeedWrangler/FeedWranglerConfig.swift index 24386daab..2de3b62e9 100644 --- a/Frameworks/Account/FeedWrangler/FeedWranglerConfig.swift +++ b/Frameworks/Account/FeedWrangler/FeedWranglerConfig.swift @@ -10,7 +10,7 @@ import Foundation enum FeedWranglerConfig { static let pageSize = 100 - static let clientKey = Secrets.FeedWranglerClientKey // Add FEED_WRANGLER_KEY = XYZ to SharedXcodeSettings/DeveloperSettings.xcconfig + static let clientKey = Secrets.feedWranglerKey // Add FEED_WRANGLER_KEY = XYZ to SharedXcodeSettings/DeveloperSettings.xcconfig static let clientPath = "https://feedwrangler.net/api/v2/" static let clientURL = { URL(string: FeedWranglerConfig.clientPath)! From d3dd7498e680d4ac899933e1264f8a3948edc727 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Wed, 27 Nov 2019 11:20:43 -0500 Subject: [PATCH 3/7] generate the secret property automatically --- Shared/Secrets.swift.gyb | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/Shared/Secrets.swift.gyb b/Shared/Secrets.swift.gyb index e65f74297..8157424a2 100644 --- a/Shared/Secrets.swift.gyb +++ b/Shared/Secrets.swift.gyb @@ -1,27 +1,39 @@ -// Secrets.swift.gyb +// Generated by Secrets.swift.gyb %{ import os +secrets = ['FEED_WRANGLER_KEY'] + def chunks(seq, size): - return (seq[i:(i + size)] for i in range(0, len(seq), size)) + return (seq[i:(i + size)] for i in range(0, len(seq), size)) def encode(string, salt): - bytes = string.encode("UTF-8") - return [ord(bytes[i]) ^ salt[i % len(salt)] for i in range(0, len(bytes))] + bytes = string.encode("UTF-8") + return [ord(bytes[i]) ^ salt[i % len(salt)] for i in range(0, len(bytes))] + +def snake_to_camel(snake_str): + components = snake_str.split('_') + return components[0].lower() + ''.join(x.title() for x in components[1:]) salt = [ord(byte) for byte in os.urandom(64)] }% enum Secrets { - static var FeedWranglerClientKey: String { - let encoded: [UInt8] = [ - % for chunk in chunks(encode(os.environ.get('FEED_WRANGLER_KEY'), salt), 8): + % for secret in secrets: + static var ${snake_to_camel(secret)}: String { + let encoded: [UInt8] = [ + % for chunk in chunks(encode(os.environ.get(secret), salt), 8): ${"".join(["0x%02x, " % byte for byte in chunk])} % end ] return decode(encoded, salt: salt) } + % end + +} + +private extension Secrets { private static let salt: [UInt8] = [ % for chunk in chunks(salt, 8): @@ -34,4 +46,5 @@ enum Secrets { element ^ salt[offset % salt.count] }, as: UTF8.self) } + } \ No newline at end of file From 544daf57560317f1bd3c5e2ce35d70a849a1c4f9 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Wed, 27 Nov 2019 11:30:27 -0500 Subject: [PATCH 4/7] handle missing secrets Missing secrets will give a blank string instead of exploding. Also, an example of a custom property --- Shared/Secrets.swift.gyb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Shared/Secrets.swift.gyb b/Shared/Secrets.swift.gyb index 8157424a2..b20e8bf56 100644 --- a/Shared/Secrets.swift.gyb +++ b/Shared/Secrets.swift.gyb @@ -18,11 +18,11 @@ def snake_to_camel(snake_str): salt = [ord(byte) for byte in os.urandom(64)] }% enum Secrets { - % for secret in secrets: + static var ${snake_to_camel(secret)}: String { let encoded: [UInt8] = [ - % for chunk in chunks(encode(os.environ.get(secret), salt), 8): + % for chunk in chunks(encode(os.environ.get(secret) or "", salt), 8): ${"".join(["0x%02x, " % byte for byte in chunk])} % end ] @@ -30,7 +30,10 @@ enum Secrets { return decode(encoded, salt: salt) } % end - + + %{ + # custom example: static let myVariable = "${os.environ.get('MY_CUSTOM_VARIABLE')}" + }% } private extension Secrets { From 69ebecaf40f76241c682f2783ed6bd3c81f2eb49 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Wed, 27 Nov 2019 11:40:24 -0500 Subject: [PATCH 5/7] display instructions if gyb is missing --- updateSecrets.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/updateSecrets.sh b/updateSecrets.sh index d5fa38cc9..9033d9d8d 100755 --- a/updateSecrets.sh +++ b/updateSecrets.sh @@ -1,7 +1,12 @@ #!/bin/bash -find . -name '*.gyb' | \ - while read file; do \ - echo "running ${file%.gyb}"; \ - gyb --line-directive '' -o "${file%.gyb}" "$file"; \ +if [ ! $(type -P "gyb") ]; then + echo "gyb not installed. Install via: brew install nshipster/formulae/gyb" + exit 1 +fi + +find . -name '*.gyb' | + while read file; do + echo "running ${file%.gyb}"; + gyb --line-directive '' -o "${file%.gyb}" "$file"; done \ No newline at end of file From a4f6adbd852abe9467abd31f7ab0dd30343f5c04 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Wed, 27 Nov 2019 14:06:32 -0500 Subject: [PATCH 6/7] vendor the gyb dependency --- .gitignore | 3 + Vendor/gyb | 3 + Vendor/gyb.py | 1267 ++++++++++++++++++++++++++++++++++++++++++++++ updateSecrets.sh | 7 +- 4 files changed, 1274 insertions(+), 6 deletions(-) create mode 100755 Vendor/gyb create mode 100644 Vendor/gyb.py diff --git a/.gitignore b/.gitignore index 8497f0b61..51163b4ba 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,7 @@ fastlane/Preview.html fastlane/screenshots fastlane/test_output + + /Shared/Secrets.swift +*.py[cod] diff --git a/Vendor/gyb b/Vendor/gyb new file mode 100755 index 000000000..dece788e5 --- /dev/null +++ b/Vendor/gyb @@ -0,0 +1,3 @@ +#!/usr/bin/env python2.7 +import gyb +gyb.main() diff --git a/Vendor/gyb.py b/Vendor/gyb.py new file mode 100644 index 000000000..d9ce0e7b3 --- /dev/null +++ b/Vendor/gyb.py @@ -0,0 +1,1267 @@ +#!/usr/bin/env python +# GYB: Generate Your Boilerplate (improved names welcome; at least +# this one's short). See -h output for instructions + +from __future__ import print_function + +import os +import re +import sys +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO +import textwrap +import tokenize +from bisect import bisect + +try: + basestring +except NameError: + basestring = str + + +def get_line_starts(s): + """Return a list containing the start index of each line in s. + + The list also contains a sentinel index for the end of the string, + so there will be one more element in the list than there are lines + in the string + """ + starts = [0] + + for line in s.split('\n'): + starts.append(starts[-1] + len(line) + 1) + + starts[-1] -= 1 + return starts + + +def strip_trailing_nl(s): + """If s ends with a newline, drop it; else return s intact""" + return s[:-1] if s.endswith('\n') else s + + +def split_lines(s): + """Split s into a list of lines, each of which has a trailing newline + + If the lines are later concatenated, the result is s, possibly + with a single appended newline. + """ + return [l + '\n' for l in s.split('\n')] + + +# text on a line up to the first '$$', '${', or '%%' +literalText = r'(?: [^$\n%] | \$(?![${]) | %(?!%) )*' + +# The part of an '%end' line that follows the '%' sign +linesClose = r'[\ \t]* end [\ \t]* (?: \# .* )? $' + +# Note: Where "# Absorb" appears below, the regexp attempts to eat up +# through the end of ${...} and %{...}% constructs. In reality we +# handle this with the Python tokenizer, which avoids mis-detections +# due to nesting, comments and strings. This extra absorption in the +# regexp facilitates testing the regexp on its own, by preventing the +# interior of some of these constructs from being treated as literal +# text. +tokenize_re = re.compile( + r''' +# %-lines and %{...}-blocks + # \n? # absorb one preceding newline + ^ + (?: + (?P + (?P<_indent> [\ \t]* % (?! [{%] ) [\ \t]* ) (?! [\ \t] | ''' + + linesClose + r''' ) .* + ( \n (?P=_indent) (?! ''' + linesClose + r''' ) .* ) * + ) + | (?P [\ \t]* % [ \t]* ''' + linesClose + r''' ) + | [\ \t]* (?P %\{ ) + (?: [^}]| \} (?!%) )* \}% # Absorb + ) + \n? # absorb one trailing newline + +# Substitutions +| (?P \$\{ ) + [^}]* \} # Absorb + +# %% and $$ are literal % and $ respectively +| (?P[$%]) (?P=symbol) + +# Literal text +| (?P ''' + literalText + r''' + (?: + # newline that doesn't precede space+% + (?: \n (?! [\ \t]* %[^%] ) ) + ''' + literalText + r''' + )* + \n? + ) +''', re.VERBOSE | re.MULTILINE) + +gyb_block_close = re.compile(r'\}%[ \t]*\n?') + + +def token_pos_to_index(token_pos, start, line_starts): + """Translate a tokenize (line, column) pair into an absolute + position in source text given the position where we started + tokenizing and a list that maps lines onto their starting + character indexes. + """ + relative_token_line_plus1, token_col = token_pos + + # line number where we started tokenizing + start_line_num = bisect(line_starts, start) - 1 + + # line number of the token in the whole text + abs_token_line = relative_token_line_plus1 - 1 + start_line_num + + # if found in the first line, adjust the end column to account + # for the extra text + if relative_token_line_plus1 == 1: + token_col += start - line_starts[start_line_num] + + # Sometimes tokenizer errors report a line beyond the last one + if abs_token_line >= len(line_starts): + return line_starts[-1] + + return line_starts[abs_token_line] + token_col + + +def tokenize_python_to_unmatched_close_curly(source_text, start, line_starts): + """Apply Python's tokenize to source_text starting at index start + while matching open and close curly braces. When an unmatched + close curly brace is found, return its index. If not found, + return len(source_text). If there's a tokenization error, return + the position of the error. + """ + stream = StringIO(source_text) + stream.seek(start) + nesting = 0 + + try: + for kind, text, token_start, token_end, line_text \ + in tokenize.generate_tokens(stream.readline): + + if text == '{': + nesting += 1 + elif text == '}': + nesting -= 1 + if nesting < 0: + return token_pos_to_index(token_start, start, line_starts) + + except tokenize.TokenError as error: + (message, error_pos) = error.args + return token_pos_to_index(error_pos, start, line_starts) + + return len(source_text) + + +def tokenize_template(template_text): + r"""Given the text of a template, returns an iterator over + (tokenType, token, match) tuples. + + **Note**: this is template syntax tokenization, not Python + tokenization. + + When a non-literal token is matched, a client may call + iter.send(pos) on the iterator to reset the position in + template_text at which scanning will resume. + + This function provides a base level of tokenization which is + then refined by ParseContext.token_generator. + + >>> from pprint import * + >>> pprint(list((kind, text) for kind, text, _ in tokenize_template( + ... '%for x in range(10):\n% print x\n%end\njuicebox'))) + [('gybLines', '%for x in range(10):\n% print x'), + ('gybLinesClose', '%end'), + ('literal', 'juicebox')] + + >>> pprint(list((kind, text) for kind, text, _ in tokenize_template( + ... '''Nothing + ... % if x: + ... % for i in range(3): + ... ${i} + ... % end + ... % else: + ... THIS SHOULD NOT APPEAR IN THE OUTPUT + ... '''))) + [('literal', 'Nothing\n'), + ('gybLines', '% if x:\n% for i in range(3):'), + ('substitutionOpen', '${'), + ('literal', '\n'), + ('gybLinesClose', '% end'), + ('gybLines', '% else:'), + ('literal', 'THIS SHOULD NOT APPEAR IN THE OUTPUT\n')] + + >>> for kind, text, _ in tokenize_template(''' + ... This is $some$ literal stuff containing a ${substitution} + ... followed by a %{...} block: + ... %{ + ... # Python code + ... }% + ... and here $${are} some %-lines: + ... % x = 1 + ... % y = 2 + ... % if z == 3: + ... % print '${hello}' + ... % end + ... % for x in zz: + ... % print x + ... % # different indentation + ... % twice + ... and some lines that literally start with a %% token + ... %% first line + ... %% second line + ... '''): + ... print((kind, text.strip().split('\n',1)[0])) + ('literal', 'This is $some$ literal stuff containing a') + ('substitutionOpen', '${') + ('literal', 'followed by a %{...} block:') + ('gybBlockOpen', '%{') + ('literal', 'and here ${are} some %-lines:') + ('gybLines', '% x = 1') + ('gybLinesClose', '% end') + ('gybLines', '% for x in zz:') + ('gybLines', '% # different indentation') + ('gybLines', '% twice') + ('literal', 'and some lines that literally start with a % token') + """ + pos = 0 + end = len(template_text) + + saved_literal = [] + literal_first_match = None + + while pos < end: + m = tokenize_re.match(template_text, pos, end) + + # pull out the one matched key (ignoring internal patterns starting + # with _) + ((kind, text), ) = ( + (kind, text) for (kind, text) in m.groupdict().items() + if text is not None and kind[0] != '_') + + if kind in ('literal', 'symbol'): + if len(saved_literal) == 0: + literal_first_match = m + # literals and symbols get batched together + saved_literal.append(text) + pos = None + else: + # found a non-literal. First yield any literal we've accumulated + if saved_literal != []: + yield 'literal', ''.join(saved_literal), literal_first_match + saved_literal = [] + + # Then yield the thing we found. If we get a reply, it's + # the place to resume tokenizing + pos = yield kind, text, m + + # If we were not sent a new position by our client, resume + # tokenizing at the end of this match. + if pos is None: + pos = m.end(0) + else: + # Client is not yet ready to process next token + yield + + if saved_literal != []: + yield 'literal', ''.join(saved_literal), literal_first_match + + +def split_gyb_lines(source_lines): + r"""Return a list of lines at which to split the incoming source + + These positions represent the beginnings of python line groups that + will require a matching %end construct if they are to be closed. + + >>> src = split_lines('''\ + ... if x: + ... print x + ... if y: # trailing comment + ... print z + ... if z: # another comment\ + ... ''') + >>> s = split_gyb_lines(src) + >>> len(s) + 2 + >>> src[s[0]] + ' print z\n' + >>> s[1] - len(src) + 0 + + >>> src = split_lines('''\ + ... if x: + ... if y: print 1 + ... if z: + ... print 2 + ... pass\ + ... ''') + >>> s = split_gyb_lines(src) + >>> len(s) + 1 + >>> src[s[0]] + ' if y: print 1\n' + + >>> src = split_lines('''\ + ... if x: + ... if y: + ... print 1 + ... print 2 + ... ''') + >>> s = split_gyb_lines(src) + >>> len(s) + 2 + >>> src[s[0]] + ' if y:\n' + >>> src[s[1]] + ' print 1\n' + """ + last_token_text, last_token_kind = None, None + unmatched_indents = [] + + dedents = 0 + try: + for token_kind, token_text, token_start, \ + (token_end_line, token_end_col), line_text \ + in tokenize.generate_tokens(lambda i=iter(source_lines): + next(i)): + + if token_kind in (tokenize.COMMENT, tokenize.ENDMARKER): + continue + + if token_text == '\n' and last_token_text == ':': + unmatched_indents.append(token_end_line) + + # The tokenizer appends dedents at EOF; don't consider + # those as matching indentations. Instead just save them + # up... + if last_token_kind == tokenize.DEDENT: + dedents += 1 + # And count them later, when we see something real. + if token_kind != tokenize.DEDENT and dedents > 0: + unmatched_indents = unmatched_indents[:-dedents] + dedents = 0 + + last_token_text, last_token_kind = token_text, token_kind + + except tokenize.TokenError: + # Let the later compile() call report the error + return [] + + if last_token_text == ':': + unmatched_indents.append(len(source_lines)) + + return unmatched_indents + + +def code_starts_with_dedent_keyword(source_lines): + r"""Return True iff the incoming Python source_lines begin with "else", + "elif", "except", or "finally". + + Initial comments and whitespace are ignored. + + >>> code_starts_with_dedent_keyword(split_lines('if x in y: pass')) + False + >>> code_starts_with_dedent_keyword(split_lines('except ifSomethingElse:')) + True + >>> code_starts_with_dedent_keyword( + ... split_lines('\n# comment\nelse: # yes')) + True + """ + token_text = None + for token_kind, token_text, _, _, _ \ + in tokenize.generate_tokens(lambda i=iter(source_lines): next(i)): + + if token_kind != tokenize.COMMENT and token_text.strip() != '': + break + + return token_text in ('else', 'elif', 'except', 'finally') + + +class ParseContext(object): + + """State carried through a parse of a template""" + + filename = '' + template = '' + line_starts = [] + code_start_line = -1 + code_text = None + tokens = None # The rest of the tokens + close_lines = False + + def __init__(self, filename, template=None): + self.filename = os.path.abspath(filename) + if sys.platform == 'win32': + self.filename = self.filename.replace('\\', '/') + if template is None: + with open(filename) as f: + self.template = f.read() + else: + self.template = template + self.line_starts = get_line_starts(self.template) + self.tokens = self.token_generator(tokenize_template(self.template)) + self.next_token() + + def pos_to_line(self, pos): + return bisect(self.line_starts, pos) - 1 + + def token_generator(self, base_tokens): + r"""Given an iterator over (kind, text, match) triples (see + tokenize_template above), return a refined iterator over + token_kinds. + + Among other adjustments to the elements found by base_tokens, + this refined iterator tokenizes python code embedded in + template text to help determine its true extent. The + expression "base_tokens.send(pos)" is used to reset the index at + which base_tokens resumes scanning the underlying text. + + >>> ctx = ParseContext('dummy', ''' + ... %for x in y: + ... % print x + ... % end + ... literally + ... ''') + >>> while ctx.token_kind: + ... print((ctx.token_kind, ctx.code_text or ctx.token_text)) + ... ignored = ctx.next_token() + ('literal', '\n') + ('gybLinesOpen', 'for x in y:\n') + ('gybLines', ' print x\n') + ('gybLinesClose', '% end') + ('literal', 'literally\n') + + >>> ctx = ParseContext('dummy', + ... '''Nothing + ... % if x: + ... % for i in range(3): + ... ${i} + ... % end + ... % else: + ... THIS SHOULD NOT APPEAR IN THE OUTPUT + ... ''') + >>> while ctx.token_kind: + ... print((ctx.token_kind, ctx.code_text or ctx.token_text)) + ... ignored = ctx.next_token() + ('literal', 'Nothing\n') + ('gybLinesOpen', 'if x:\n') + ('gybLinesOpen', ' for i in range(3):\n') + ('substitutionOpen', 'i') + ('literal', '\n') + ('gybLinesClose', '% end') + ('gybLinesOpen', 'else:\n') + ('literal', 'THIS SHOULD NOT APPEAR IN THE OUTPUT\n') + + >>> ctx = ParseContext('dummy', + ... '''% for x in [1, 2, 3]: + ... % if x == 1: + ... literal1 + ... % elif x > 1: # add output line here to fix bug + ... % if x == 2: + ... literal2 + ... % end + ... % end + ... % end + ... ''') + >>> while ctx.token_kind: + ... print((ctx.token_kind, ctx.code_text or ctx.token_text)) + ... ignored = ctx.next_token() + ('gybLinesOpen', 'for x in [1, 2, 3]:\n') + ('gybLinesOpen', ' if x == 1:\n') + ('literal', 'literal1\n') + ('gybLinesOpen', 'elif x > 1: # add output line here to fix bug\n') + ('gybLinesOpen', ' if x == 2:\n') + ('literal', 'literal2\n') + ('gybLinesClose', '% end') + ('gybLinesClose', '% end') + ('gybLinesClose', '% end') + """ + for self.token_kind, self.token_text, self.token_match in base_tokens: + kind = self.token_kind + self.code_text = None + + # Do we need to close the current lines? + self.close_lines = kind == 'gybLinesClose' + + # %{...}% and ${...} constructs + if kind.endswith('Open'): + + # Tokenize text that follows as Python up to an unmatched '}' + code_start = self.token_match.end(kind) + self.code_start_line = self.pos_to_line(code_start) + + close_pos = tokenize_python_to_unmatched_close_curly( + self.template, code_start, self.line_starts) + self.code_text = self.template[code_start:close_pos] + yield kind + + if (kind == 'gybBlockOpen'): + # Absorb any '}% \n' + m2 = gyb_block_close.match(self.template, close_pos) + if not m2: + raise ValueError("Invalid block closure") + next_pos = m2.end(0) + else: + assert kind == 'substitutionOpen' + # skip past the closing '}' + next_pos = close_pos + 1 + + # Resume tokenizing after the end of the code. + base_tokens.send(next_pos) + + elif kind == 'gybLines': + + self.code_start_line = self.pos_to_line( + self.token_match.start('gybLines')) + indentation = self.token_match.group('_indent') + + # Strip off the leading indentation and %-sign + source_lines = re.split( + '^' + re.escape(indentation), + self.token_match.group('gybLines') + '\n', + flags=re.MULTILINE)[1:] + + if code_starts_with_dedent_keyword(source_lines): + self.close_lines = True + + last_split = 0 + for line in split_gyb_lines(source_lines): + self.token_kind = 'gybLinesOpen' + self.code_text = ''.join(source_lines[last_split:line]) + yield self.token_kind + last_split = line + self.code_start_line += line - last_split + self.close_lines = False + + self.code_text = ''.join(source_lines[last_split:]) + if self.code_text: + self.token_kind = 'gybLines' + yield self.token_kind + else: + yield self.token_kind + + def next_token(self): + """Move to the next token""" + for kind in self.tokens: + return self.token_kind + + self.token_kind = None + + +_default_line_directive = \ + '// ###sourceLocation(file: "%(file)s", line: %(line)d)' + + +class ExecutionContext(object): + + """State we pass around during execution of a template""" + + def __init__(self, line_directive=_default_line_directive, + **local_bindings): + self.local_bindings = local_bindings + self.line_directive = line_directive + self.local_bindings['__context__'] = self + self.result_text = [] + self.last_file_line = None + + def append_text(self, text, file, line): + # see if we need to inject a line marker + if self.line_directive: + if (file, line) != self.last_file_line: + # We can only insert the line directive at a line break + if len(self.result_text) == 0 \ + or self.result_text[-1].endswith('\n'): + substitutions = {'file': file, 'line': line + 1} + format_str = self.line_directive + '\n' + self.result_text.append(format_str % substitutions) + # But if the new text contains any line breaks, we can create + # one + elif '\n' in text: + i = text.find('\n') + self.result_text.append(text[:i + 1]) + # and try again + self.append_text(text[i + 1:], file, line + 1) + return + + self.result_text.append(text) + self.last_file_line = (file, line + text.count('\n')) + + +class ASTNode(object): + + """Abstract base class for template AST nodes""" + + def __init__(self): + raise NotImplementedError("ASTNode.__init__ is not implemented.") + + def execute(self, context): + raise NotImplementedError("ASTNode.execute is not implemented.") + + def __str__(self, indent=''): + raise NotImplementedError("ASTNode.__str__ is not implemented.") + + def format_children(self, indent): + if not self.children: + return ' []' + + return '\n'.join( + ['', indent + '['] + + [x.__str__(indent + 4 * ' ') for x in self.children] + + [indent + ']']) + + +class Block(ASTNode): + + """A sequence of other AST nodes, to be executed in order""" + + children = [] + + def __init__(self, context): + self.children = [] + + while context.token_kind and not context.close_lines: + if context.token_kind == 'literal': + node = Literal + else: + node = Code + self.children.append(node(context)) + + def execute(self, context): + for x in self.children: + x.execute(context) + + def __str__(self, indent=''): + return indent + 'Block:' + self.format_children(indent) + + +class Literal(ASTNode): + + """An AST node that generates literal text""" + + def __init__(self, context): + self.text = context.token_text + start_position = context.token_match.start(context.token_kind) + self.start_line_number = context.pos_to_line(start_position) + self.filename = context.filename + context.next_token() + + def execute(self, context): + context.append_text(self.text, self.filename, self.start_line_number) + + def __str__(self, indent=''): + return '\n'.join( + [indent + x for x in ['Literal:'] + + strip_trailing_nl(self.text).split('\n')]) + + +class Code(ASTNode): + + """An AST node that is evaluated as Python""" + + code = None + children = () + kind = None + + def __init__(self, context): + + source = '' + source_line_count = 0 + + def accumulate_code(): + s = source + (context.code_start_line - source_line_count) * '\n' \ + + textwrap.dedent(context.code_text) + line_count = context.code_start_line + \ + context.code_text.count('\n') + context.next_token() + return s, line_count + + eval_exec = 'exec' + if context.token_kind.startswith('substitution'): + eval_exec = 'eval' + source, source_line_count = accumulate_code() + source = '(' + source.strip() + ')' + + else: + while context.token_kind == 'gybLinesOpen': + source, source_line_count = accumulate_code() + source += ' __children__[%d].execute(__context__)\n' % len( + self.children) + source_line_count += 1 + + self.children += (Block(context),) + + if context.token_kind == 'gybLinesClose': + context.next_token() + + if context.token_kind == 'gybLines': + source, source_line_count = accumulate_code() + + # Only handle a substitution as part of this code block if + # we don't already have some %-lines. + elif context.token_kind == 'gybBlockOpen': + + # Opening ${...} and %{...}% constructs + source, source_line_count = accumulate_code() + + self.filename = context.filename + self.start_line_number = context.code_start_line + self.code = compile(source, context.filename, eval_exec) + self.source = source + + def execute(self, context): + # Save __children__ from the local bindings + save_children = context.local_bindings.get('__children__') + # Execute the code with our __children__ in scope + context.local_bindings['__children__'] = self.children + context.local_bindings['__file__'] = self.filename + result = eval(self.code, context.local_bindings) + + if context.local_bindings['__children__'] is not self.children: + raise ValueError("The code is not allowed to mutate __children__") + # Restore the bindings + context.local_bindings['__children__'] = save_children + + # If we got a result, the code was an expression, so append + # its value + if result is not None \ + or (isinstance(result, basestring) and result != ''): + from numbers import Number, Integral + result_string = None + if isinstance(result, Number) and not isinstance(result, Integral): + result_string = repr(result) + else: + result_string = str(result) + context.append_text( + result_string, self.filename, self.start_line_number) + + def __str__(self, indent=''): + source_lines = re.sub(r'^\n', '', strip_trailing_nl( + self.source), flags=re.MULTILINE).split('\n') + if len(source_lines) == 1: + s = indent + 'Code: {' + source_lines[0] + '}' + else: + s = indent + 'Code:\n' + indent + '{\n' + '\n'.join( + indent + 4 * ' ' + l for l in source_lines + ) + '\n' + indent + '}' + return s + self.format_children(indent) + + +def expand(filename, line_directive=_default_line_directive, **local_bindings): + r"""Return the contents of the given template file, executed with the given + local bindings. + + >>> from tempfile import NamedTemporaryFile + >>> # On Windows, the name of a NamedTemporaryFile cannot be used to open + >>> # the file for a second time if delete=True. Therefore, we have to + >>> # manually handle closing and deleting this file to allow us to open + >>> # the file by its name across all platforms. + >>> f = NamedTemporaryFile(delete=False) + >>> f.write( + ... r'''--- + ... % for i in range(int(x)): + ... a pox on ${i} for epoxy + ... % end + ... ${120 + + ... + ... 3} + ... abc + ... ${"w\nx\nX\ny"} + ... z + ... ''') + >>> f.flush() + >>> result = expand( + ... f.name, + ... line_directive='//#sourceLocation(file: "%(file)s", ' + \ + ... 'line: %(line)d)', + ... x=2 + ... ).replace( + ... '"%s"' % f.name.replace('\\', '/'), '"dummy.file"') + >>> print(result, end='') + //#sourceLocation(file: "dummy.file", line: 1) + --- + //#sourceLocation(file: "dummy.file", line: 3) + a pox on 0 for epoxy + //#sourceLocation(file: "dummy.file", line: 3) + a pox on 1 for epoxy + //#sourceLocation(file: "dummy.file", line: 5) + 123 + //#sourceLocation(file: "dummy.file", line: 8) + abc + w + x + X + y + //#sourceLocation(file: "dummy.file", line: 10) + z + >>> f.close() + >>> os.remove(f.name) + """ + with open(filename) as f: + t = parse_template(filename, f.read()) + d = os.getcwd() + os.chdir(os.path.dirname(os.path.abspath(filename))) + try: + return execute_template( + t, line_directive=line_directive, **local_bindings) + finally: + os.chdir(d) + + +def parse_template(filename, text=None): + r"""Return an AST corresponding to the given template file. + + If text is supplied, it is assumed to be the contents of the file, + as a string. + + >>> print(parse_template('dummy.file', text= + ... '''% for x in [1, 2, 3]: + ... % if x == 1: + ... literal1 + ... % elif x > 1: # add output line after this line to fix bug + ... % if x == 2: + ... literal2 + ... % end + ... % end + ... % end + ... ''')) + Block: + [ + Code: + { + for x in [1, 2, 3]: + __children__[0].execute(__context__) + } + [ + Block: + [ + Code: + { + if x == 1: + __children__[0].execute(__context__) + elif x > 1: # add output line after this line to fix bug + __children__[1].execute(__context__) + } + [ + Block: + [ + Literal: + literal1 + ] + Block: + [ + Code: + { + if x == 2: + __children__[0].execute(__context__) + } + [ + Block: + [ + Literal: + literal2 + ] + ] + ] + ] + ] + ] + ] + + >>> print(parse_template( + ... 'dummy.file', + ... text='%for x in range(10):\n% print(x)\n%end\njuicebox')) + Block: + [ + Code: + { + for x in range(10): + __children__[0].execute(__context__) + } + [ + Block: + [ + Code: {print(x)} [] + ] + ] + Literal: + juicebox + ] + + >>> print(parse_template('/dummy.file', text= + ... '''Nothing + ... % if x: + ... % for i in range(3): + ... ${i} + ... % end + ... % else: + ... THIS SHOULD NOT APPEAR IN THE OUTPUT + ... ''')) + Block: + [ + Literal: + Nothing + Code: + { + if x: + __children__[0].execute(__context__) + else: + __children__[1].execute(__context__) + } + [ + Block: + [ + Code: + { + for i in range(3): + __children__[0].execute(__context__) + } + [ + Block: + [ + Code: {(i)} [] + Literal: + + ] + ] + ] + Block: + [ + Literal: + THIS SHOULD NOT APPEAR IN THE OUTPUT + ] + ] + ] + + >>> print(parse_template('dummy.file', text='''% + ... %for x in y: + ... % print(y) + ... ''')) + Block: + [ + Code: + { + for x in y: + __children__[0].execute(__context__) + } + [ + Block: + [ + Code: {print(y)} [] + ] + ] + ] + + >>> print(parse_template('dummy.file', text='''% + ... %if x: + ... % print(y) + ... AAAA + ... %else: + ... BBBB + ... ''')) + Block: + [ + Code: + { + if x: + __children__[0].execute(__context__) + else: + __children__[1].execute(__context__) + } + [ + Block: + [ + Code: {print(y)} [] + Literal: + AAAA + ] + Block: + [ + Literal: + BBBB + ] + ] + ] + + >>> print(parse_template('dummy.file', text='''% + ... %if x: + ... % print(y) + ... AAAA + ... %# This is a comment + ... %else: + ... BBBB + ... ''')) + Block: + [ + Code: + { + if x: + __children__[0].execute(__context__) + # This is a comment + else: + __children__[1].execute(__context__) + } + [ + Block: + [ + Code: {print(y)} [] + Literal: + AAAA + ] + Block: + [ + Literal: + BBBB + ] + ] + ] + + >>> print(parse_template('dummy.file', text='''\ + ... %for x in y: + ... AAAA + ... %if x: + ... BBBB + ... %end + ... CCCC + ... ''')) + Block: + [ + Code: + { + for x in y: + __children__[0].execute(__context__) + } + [ + Block: + [ + Literal: + AAAA + Code: + { + if x: + __children__[0].execute(__context__) + } + [ + Block: + [ + Literal: + BBBB + ] + ] + Literal: + CCCC + ] + ] + ] + """ + return Block(ParseContext(filename, text)) + + +def execute_template( + ast, line_directive=_default_line_directive, **local_bindings): + r"""Return the text generated by executing the given template AST. + + Keyword arguments become local variable bindings in the execution context + + >>> root_directory = os.path.abspath('/') + >>> file_name = (root_directory + 'dummy.file').replace('\\', '/') + >>> ast = parse_template(file_name, text= + ... '''Nothing + ... % if x: + ... % for i in range(3): + ... ${i} + ... % end + ... % else: + ... THIS SHOULD NOT APPEAR IN THE OUTPUT + ... ''') + >>> out = execute_template(ast, + ... line_directive='//#sourceLocation(file: "%(file)s", line: %(line)d)', + ... x=1) + >>> out = out.replace(file_name, "DUMMY-FILE") + >>> print(out, end="") + //#sourceLocation(file: "DUMMY-FILE", line: 1) + Nothing + //#sourceLocation(file: "DUMMY-FILE", line: 4) + 0 + //#sourceLocation(file: "DUMMY-FILE", line: 4) + 1 + //#sourceLocation(file: "DUMMY-FILE", line: 4) + 2 + + >>> ast = parse_template(file_name, text= + ... '''Nothing + ... % a = [] + ... % for x in range(3): + ... % a.append(x) + ... % end + ... ${a} + ... ''') + >>> out = execute_template(ast, + ... line_directive='//#sourceLocation(file: "%(file)s", line: %(line)d)', + ... x=1) + >>> out = out.replace(file_name, "DUMMY-FILE") + >>> print(out, end="") + //#sourceLocation(file: "DUMMY-FILE", line: 1) + Nothing + //#sourceLocation(file: "DUMMY-FILE", line: 6) + [0, 1, 2] + + >>> ast = parse_template(file_name, text= + ... '''Nothing + ... % a = [] + ... % for x in range(3): + ... % a.append(x) + ... % end + ... ${a} + ... ''') + >>> out = execute_template(ast, + ... line_directive='#line %(line)d "%(file)s"', x=1) + >>> out = out.replace(file_name, "DUMMY-FILE") + >>> print(out, end="") + #line 1 "DUMMY-FILE" + Nothing + #line 6 "DUMMY-FILE" + [0, 1, 2] + """ + execution_context = ExecutionContext( + line_directive=line_directive, **local_bindings) + ast.execute(execution_context) + return ''.join(execution_context.result_text) + + +def main(): + """ + Lint this file. + >>> import sys + >>> gyb_path = os.path.realpath(__file__).replace('.pyc', '.py') + >>> sys.path.append(os.path.dirname(gyb_path)) + >>> import python_lint + >>> python_lint.lint([gyb_path], verbose=False) + 0 + """ + + import argparse + import sys + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description='Generate Your Boilerplate!', epilog=''' + A GYB template consists of the following elements: + + - Literal text which is inserted directly into the output + + - %% or $$ in literal text, which insert literal '%' and '$' + symbols respectively. + + - Substitutions of the form ${}. The Python + expression is converted to a string and the result is inserted + into the output. + + - Python code delimited by %{...}%. Typically used to inject + definitions (functions, classes, variable bindings) into the + evaluation context of the template. Common indentation is + stripped, so you can add as much indentation to the beginning + of this code as you like + + - Lines beginning with optional whitespace followed by a single + '%' and Python code. %-lines allow you to nest other + constructs inside them. To close a level of nesting, use the + "%end" construct. + + - Lines beginning with optional whitespace and followed by a + single '%' and the token "end", which close open constructs in + %-lines. + + Example template: + + - Hello - + %{ + x = 42 + def succ(a): + return a+1 + }% + + I can assure you that ${x} < ${succ(x)} + + % if int(y) > 7: + % for i in range(3): + y is greater than seven! + % end + % else: + y is less than or equal to seven + % end + + - The End. - + + When run with "gyb -Dy=9", the output is + + - Hello - + + I can assure you that 42 < 43 + + y is greater than seven! + y is greater than seven! + y is greater than seven! + + - The End. - +''' + ) + parser.add_argument( + '-D', action='append', dest='defines', metavar='NAME=VALUE', + default=[], + help='''Bindings to be set in the template's execution context''') + + parser.add_argument( + 'file', type=argparse.FileType(), + help='Path to GYB template file (defaults to stdin)', nargs='?', + default=sys.stdin) + parser.add_argument( + '-o', dest='target', type=argparse.FileType('w'), + help='Output file (defaults to stdout)', default=sys.stdout) + parser.add_argument( + '--test', action='store_true', + default=False, help='Run a self-test') + parser.add_argument( + '--verbose-test', action='store_true', + default=False, help='Run a verbose self-test') + parser.add_argument( + '--dump', action='store_true', + default=False, help='Dump the parsed template to stdout') + parser.add_argument( + '--line-directive', + default=_default_line_directive, + help=''' + Line directive format string, which will be + provided 2 substitutions, `%%(line)d` and `%%(file)s`. + + Example: `#sourceLocation(file: "%%(file)s", line: %%(line)d)` + + The default works automatically with the `line-directive` tool, + which see for more information. + ''') + + args = parser.parse_args(sys.argv[1:]) + + if args.test or args.verbose_test: + import doctest + selfmod = sys.modules[__name__] + if doctest.testmod(selfmod, verbose=args.verbose_test or None).failed: + sys.exit(1) + + bindings = dict(x.split('=', 1) for x in args.defines) + ast = parse_template(args.file.name, args.file.read()) + if args.dump: + print(ast) + # Allow the template to open files and import .py files relative to its own + # directory + os.chdir(os.path.dirname(os.path.abspath(args.file.name))) + sys.path = ['.'] + sys.path + + args.target.write(execute_template(ast, args.line_directive, **bindings)) + + +if __name__ == '__main__': + main() diff --git a/updateSecrets.sh b/updateSecrets.sh index 9033d9d8d..981723438 100755 --- a/updateSecrets.sh +++ b/updateSecrets.sh @@ -1,12 +1,7 @@ #!/bin/bash -if [ ! $(type -P "gyb") ]; then - echo "gyb not installed. Install via: brew install nshipster/formulae/gyb" - exit 1 -fi - find . -name '*.gyb' | while read file; do echo "running ${file%.gyb}"; - gyb --line-directive '' -o "${file%.gyb}" "$file"; + ./Vendor/gyb --line-directive '' -o "${file%.gyb}" "$file"; done \ No newline at end of file From 88664bd3e69418ec0ce3ea752692ea4651c1ec5c Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Wed, 27 Nov 2019 17:20:56 -0500 Subject: [PATCH 7/7] use pre-build hooks to trigger gyb script --- .../xcschemes/NetNewsWire MAS.xcscheme | 20 ++++++++++++++++++- .../xcschemes/NetNewsWire-iOS.xcscheme | 20 ++++++++++++++++++- .../xcschemes/NetNewsWire.xcscheme | 20 ++++++++++++++++++- updateSecrets.sh | 6 +++--- 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/NetNewsWire.xcodeproj/xcshareddata/xcschemes/NetNewsWire MAS.xcscheme b/NetNewsWire.xcodeproj/xcshareddata/xcschemes/NetNewsWire MAS.xcscheme index 29320b85f..7e5194df8 100644 --- a/NetNewsWire.xcodeproj/xcshareddata/xcschemes/NetNewsWire MAS.xcscheme +++ b/NetNewsWire.xcodeproj/xcshareddata/xcschemes/NetNewsWire MAS.xcscheme @@ -1,10 +1,28 @@ + version = "1.7"> + + + + + + + + + + + version = "1.7"> + + + + + + + + + + + version = "1.7"> + + + + + + + + + +