summaryrefslogtreecommitdiff
path: root/scripts
authorGiulio Cesare Solaroli <giulio.cesare@clipperz.com>2011-10-02 23:56:18 (UTC)
committer Giulio Cesare Solaroli <giulio.cesare@clipperz.com>2011-10-02 23:56:18 (UTC)
commitef68436ac04da078ffdcacd7e1f785473a303d45 (patch) (unidiff)
treec403752d66a2c4775f00affd4fa8431b29c5b68c /scripts
parent597ecfbc0249d83e1b856cbd558340c01237a360 (diff)
downloadclipperz-ef68436ac04da078ffdcacd7e1f785473a303d45.zip
clipperz-ef68436ac04da078ffdcacd7e1f785473a303d45.tar.gz
clipperz-ef68436ac04da078ffdcacd7e1f785473a303d45.tar.bz2
First version of the newly restructured repository
Diffstat (limited to 'scripts') (more/less context) (ignore whitespace changes)
-rwxr-xr-xscripts/build5
-rw-r--r--scripts/builder/backendBuilder.py89
-rw-r--r--scripts/builder/cssmin.py223
-rw-r--r--scripts/builder/frontendBuilder.py398
-rw-r--r--scripts/builder/jsmin.py246
-rwxr-xr-xscripts/builder/main.py166
-rw-r--r--scripts/builder/phpBuilder.py14
-rw-r--r--scripts/builder/pythonBuilder.py14
8 files changed, 1155 insertions, 0 deletions
diff --git a/scripts/build b/scripts/build
new file mode 100755
index 0000000..b17a18f
--- a/dev/null
+++ b/scripts/build
@@ -0,0 +1,5 @@
1#!/bin/bash
2
3readonly CURR_DIR=$(cd "$(dirname "$0")"; pwd -P)
4
5${CURR_DIR}/builder/main.py $@ \ No newline at end of file
diff --git a/scripts/builder/backendBuilder.py b/scripts/builder/backendBuilder.py
new file mode 100644
index 0000000..f5dc7b2
--- a/dev/null
+++ b/scripts/builder/backendBuilder.py
@@ -0,0 +1,89 @@
1#!/usr/bin/python
2# -*- coding: UTF-8 -*-
3
4import sys, os, json
5import shutil
6import main
7import hashlib
8
9class BackendBuilder:
10
11 def __init__ (self, projectTargetDir, frontends, versions, settings):
12 self.projectTargetDir = projectTargetDir
13 self.frontends = frontends
14 self.versions = versions
15 self.settings = settings
16
17 def name (self):
18 raise NotImplementedError()
19
20 def relativePath (self):
21 raise NotImplementedError()
22
23 def compileCode (self):
24 pass
25
26 def copyCompiledCodeToTargetDir (self):
27 src = self.sourceFolder()
28 dst = self.targetFolder()
29 main.createFolder(os.path.dirname(dst))
30 shutil.copytree(src, dst)
31
32 def sourceFolder (self):
33 return main.projectBaseDir() + '/backend/' + self.relativePath() + '/src'
34
35
36 def targetFolder (self):
37 return self.projectTargetDir + self.relativePath()
38
39 def createTargetFolder (self):
40 main.createFolder(self.targetFolder())
41
42
43 #def copyFrontendResources (self, frontend):
44 # print "copying resources for frontend: " + frontend
45 # print "SETTINGS: " + str(self.settings)
46
47
48 def writeToTargetFolder (self, filename, content):
49 file = open(self.targetFolder() + '/' + filename, 'w')
50 file.write(content.encode('utf-8'))
51 file.close()
52
53
54 def configureIndexContent (self, indexContent):
55 result = indexContent
56 result = result.replace( '@request.path@', self.settings['request.path'] )
57 result = result.replace( '@should.pay.toll@', self.settings['should.pay.toll'] )
58
59 return result
60
61
62 def logChecksums (self, content, message):
63 md5Digest = hashlib.md5(content.encode('utf-8')).hexdigest()
64 shaDigest = hashlib.sha1(content.encode('utf-8')).hexdigest()
65 sha256Digest= hashlib.sha256(content.encode('utf-8')).hexdigest()
66 print message + ": " + md5Digest + " (md5)"
67 print message + ": " + shaDigest + " (sha1)"
68 print message + ": " + sha256Digest + " (sha256)"
69
70
71
72 def run (self):
73 print self.name() + " - RUN"
74
75 self.compileCode()
76 self.copyCompiledCodeToTargetDir()
77
78 for frontend in self.frontends:
79 frontendPath = frontend.module + '/'
80 if 'debug' in self.versions:
81 frontend.copyResourcesToTargetFolder(self.targetFolder())
82 #self.writeToTargetFolder(frontendPath + 'index_debug.html', self.configureIndexContent(frontend.assembleDebugVersion()))
83 self.writeToTargetFolder(frontendPath + 'index_debug.html', self.configureIndexContent(frontend.assemble(assemblyMode='DEBUG', versionType='DEBUG')))
84
85 if 'install' in self.versions:
86 index = self.configureIndexContent(frontend.assemble())
87 self.writeToTargetFolder(frontendPath + 'index.html', index)
88 self.logChecksums(index, "[" + self.name() + " - " + frontend.module + "] index.html checksum")
89
diff --git a/scripts/builder/cssmin.py b/scripts/builder/cssmin.py
new file mode 100644
index 0000000..32ddf77
--- a/dev/null
+++ b/scripts/builder/cssmin.py
@@ -0,0 +1,223 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4"""`cssmin` - A Python port of the YUI CSS compressor."""
5
6
7from StringIO import StringIO # The pure-Python StringIO supports unicode.
8import re
9
10
11__version__ = '0.1.4'
12
13
14def remove_comments(css):
15 """Remove all CSS comment blocks."""
16
17 iemac = False
18 preserve = False
19 comment_start = css.find("/*")
20 while comment_start >= 0:
21 # Preserve comments that look like `/*!...*/`.
22 # Slicing is used to make sure we don"t get an IndexError.
23 preserve = css[comment_start + 2:comment_start + 3] == "!"
24
25 comment_end = css.find("*/", comment_start + 2)
26 if comment_end < 0:
27 if not preserve:
28 css = css[:comment_start]
29 break
30 elif comment_end >= (comment_start + 2):
31 if css[comment_end - 1] == "\\":
32 # This is an IE Mac-specific comment; leave this one and the
33 # following one alone.
34 comment_start = comment_end + 2
35 iemac = True
36 elif iemac:
37 comment_start = comment_end + 2
38 iemac = False
39 elif not preserve:
40 css = css[:comment_start] + css[comment_end + 2:]
41 else:
42 comment_start = comment_end + 2
43 comment_start = css.find("/*", comment_start)
44
45 return css
46
47
48def remove_unnecessary_whitespace(css):
49 """Remove unnecessary whitespace characters."""
50
51 def pseudoclasscolon(css):
52
53 """
54 Prevents 'p :link' from becoming 'p:link'.
55
56 Translates 'p :link' into 'p ___PSEUDOCLASSCOLON___link'; this is
57 translated back again later.
58 """
59
60 regex = re.compile(r"(^|\})(([^\{\:])+\:)+([^\{]*\{)")
61 match = regex.search(css)
62 while match:
63 css = ''.join([
64 css[:match.start()],
65 match.group().replace(":", "___PSEUDOCLASSCOLON___"),
66 css[match.end():]])
67 match = regex.search(css)
68 return css
69
70 css = pseudoclasscolon(css)
71 # Remove spaces from before things.
72 css = re.sub(r"\s+([!{};:>+\(\)\],])", r"\1", css)
73
74 # If there is a `@charset`, then only allow one, and move to the beginning.
75 css = re.sub(r"^(.*)(@charset \"[^\"]*\";)", r"\2\1", css)
76 css = re.sub(r"^(\s*@charset [^;]+;\s*)+", r"\1", css)
77
78 # Put the space back in for a few cases, such as `@media screen` and
79 # `(-webkit-min-device-pixel-ratio:0)`.
80 css = re.sub(r"\band\(", "and (", css)
81
82 # Put the colons back.
83 css = css.replace('___PSEUDOCLASSCOLON___', ':')
84
85 # Remove spaces from after things.
86 css = re.sub(r"([!{}:;>+\(\[,])\s+", r"\1", css)
87
88 return css
89
90
91def remove_unnecessary_semicolons(css):
92 """Remove unnecessary semicolons."""
93
94 return re.sub(r";+\}", "}", css)
95
96
97def remove_empty_rules(css):
98 """Remove empty rules."""
99
100 return re.sub(r"[^\}\{]+\{\}", "", css)
101
102
103def normalize_rgb_colors_to_hex(css):
104 """Convert `rgb(51,102,153)` to `#336699`."""
105
106 regex = re.compile(r"rgb\s*\(\s*([0-9,\s]+)\s*\)")
107 match = regex.search(css)
108 while match:
109 colors = map(lambda s: s.strip(), match.group(1).split(","))
110 hexcolor = '#%.2x%.2x%.2x' % tuple(map(int, colors))
111 css = css.replace(match.group(), hexcolor)
112 match = regex.search(css)
113 return css
114
115
116def condense_zero_units(css):
117 """Replace `0(px, em, %, etc)` with `0`."""
118
119 return re.sub(r"([\s:])(0)(px|em|%|in|cm|mm|pc|pt|ex)", r"\1\2", css)
120
121
122def condense_multidimensional_zeros(css):
123 """Replace `:0 0 0 0;`, `:0 0 0;` etc. with `:0;`."""
124
125 css = css.replace(":0 0 0 0;", ":0;")
126 css = css.replace(":0 0 0;", ":0;")
127 css = css.replace(":0 0;", ":0;")
128
129 # Revert `background-position:0;` to the valid `background-position:0 0;`.
130 css = css.replace("background-position:0;", "background-position:0 0;")
131
132 return css
133
134
135def condense_floating_points(css):
136 """Replace `0.6` with `.6` where possible."""
137
138 return re.sub(r"(:|\s)0+\.(\d+)", r"\1.\2", css)
139
140
141def condense_hex_colors(css):
142 """Shorten colors from #AABBCC to #ABC where possible."""
143
144 regex = re.compile(r"([^\"'=\s])(\s*)#([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])([0-9a-fA-F])")
145 match = regex.search(css)
146 while match:
147 first = match.group(3) + match.group(5) + match.group(7)
148 second = match.group(4) + match.group(6) + match.group(8)
149 if first.lower() == second.lower():
150 css = css.replace(match.group(), match.group(1) + match.group(2) + '#' + first)
151 match = regex.search(css, match.end() - 3)
152 else:
153 match = regex.search(css, match.end())
154 return css
155
156
157def condense_whitespace(css):
158 """Condense multiple adjacent whitespace characters into one."""
159
160 return re.sub(r"\s+", " ", css)
161
162
163def condense_semicolons(css):
164 """Condense multiple adjacent semicolon characters into one."""
165
166 return re.sub(r";;+", ";", css)
167
168
169def wrap_css_lines(css, line_length):
170 """Wrap the lines of the given CSS to an approximate length."""
171
172 lines = []
173 line_start = 0
174 for i, char in enumerate(css):
175 # It's safe to break after `}` characters.
176 if char == '}' and (i - line_start >= line_length):
177 lines.append(css[line_start:i + 1])
178 line_start = i + 1
179
180 if line_start < len(css):
181 lines.append(css[line_start:])
182 return '\n'.join(lines)
183
184
185def cssmin(css, wrap=None):
186 css = remove_comments(css)
187 css = condense_whitespace(css)
188 # A pseudo class for the Box Model Hack
189 # (see http://tantek.com/CSS/Examples/boxmodelhack.html)
190 css = css.replace('"\\"}\\""', "___PSEUDOCLASSBMH___")
191 css = remove_unnecessary_whitespace(css)
192 css = remove_unnecessary_semicolons(css)
193 css = condense_zero_units(css)
194 css = condense_multidimensional_zeros(css)
195 css = condense_floating_points(css)
196 css = normalize_rgb_colors_to_hex(css)
197 css = condense_hex_colors(css)
198 if wrap is not None:
199 css = wrap_css_lines(css, wrap)
200 css = css.replace("___PSEUDOCLASSBMH___", '"\\"}\\""')
201 css = condense_semicolons(css)
202 return css.strip()
203
204
205def main():
206 import optparse
207 import sys
208
209 p = optparse.OptionParser(
210 prog="cssmin", version=__version__,
211 usage="%prog [--wrap N]",
212 description="""Reads raw CSS from stdin, and writes compressed CSS to stdout.""")
213
214 p.add_option(
215 '-w', '--wrap', type='int', default=None, metavar='N',
216 help="Wrap output to approximately N chars per line.")
217
218 options, args = p.parse_args()
219 sys.stdout.write(cssmin(sys.stdin.read(), wrap=options.wrap))
220
221
222if __name__ == '__main__':
223 main()
diff --git a/scripts/builder/frontendBuilder.py b/scripts/builder/frontendBuilder.py
new file mode 100644
index 0000000..b796438
--- a/dev/null
+++ b/scripts/builder/frontendBuilder.py
@@ -0,0 +1,398 @@
1#!/usr/bin/python
2# -*- coding: UTF-8 -*-
3
4import sys, os, re
5import cssmin
6import jsmin
7import codecs
8import shutil
9import StringIO
10import urllib
11
12#from mercurial import ui, hg
13#from mercurial.node import hex
14from dulwich.repo import Repo
15
16import main
17
18
19
20class FrontendBuilder:
21
22 def __init__ (self, frontend, settings):
23 if '.' in frontend:
24 moduleComponents = frontend.split('.')
25 self.module = moduleComponents[0]
26 self.submodule = moduleComponents[1]
27 else:
28 self.module = frontend
29 self.submodule = frontend
30
31 self.settings = settings
32 self.projectDir = main.projectBaseDir()
33 self.processedFiles = {}
34
35
36 def mercurialRepositoryVersion (self):
37 repo = hg.repository(ui.ui(), self.projectDir)
38 context = repo['tip']
39 result = str(context)
40
41 return result
42
43
44 def gitRepositoryVersion (self):
45 repo = Repo(self.projectDir)
46 #if repo.is_dirty():
47 #print "WARNING: build run with dirty repository"
48 result = repo.refs['HEAD']
49
50 return result
51
52
53
54 def repositoryVersion (self):
55 cacheKey = 'repositoryVersion'
56 if not self.processedFiles.has_key(cacheKey):
57 #result = self.mercurialRepositoryVersion()
58 result = self.gitRepositoryVersion()
59 self.processedFiles[cacheKey] = result
60 else:
61 result = self.processedFiles[cacheKey]
62
63 return result
64
65
66 #def relativePath (self):
67 #return self.module
68 #
69
70 def log (self, message):
71 print "frontend [" + self.module + "]: " + message
72
73
74 def absolutePathForSourceFile (self, folder, basePath, file):
75 return folder + '/frontend/' + self.module + '/' + basePath + '/' + file
76
77
78 def absolutePathForTargetFile (self, folder, basePath, file):
79 return folder + '/' + self.module + '/' + basePath + '/' + file
80
81 def filterFiles (self, files):
82 result = []
83
84 for file in files:
85 if file.startswith('--'):
86 pass
87 else:
88 result.append(file)
89
90 return result
91
92
93 def copyResources (self, sourceFolder, destinationFolder, fileType):
94 for file in self.filterFiles(self.settings[fileType]):
95 src = self.absolutePathForSourceFile(sourceFolder, fileType, file)
96 dst = self.absolutePathForTargetFile(destinationFolder, fileType, file)
97 main.createFolder(os.path.dirname(dst))
98 shutil.copy2(src, dst)
99
100
101 def copyResourcesToTargetFolder (self, targetFolder):
102 self.copyResources(self.projectDir, targetFolder, 'css')
103 self.copyResources(self.projectDir, targetFolder, 'js')
104
105
106 def loadFilesContent (self, basePath, files):
107 result = ""
108
109 for file in self.filterFiles(files):
110 try:
111 fileHandler = codecs.open(self.absolutePathForSourceFile(self.projectDir, basePath, file), 'r', 'utf-8')
112 except:
113 print "FILE: " + file
114
115 result += fileHandler.read() + '\n'
116 fileHandler.close()
117
118 return result
119
120
121 def template (self):
122 processedFile = 'html_template'
123 if not self.processedFiles.has_key(processedFile):
124 self.processedFiles[processedFile] = self.loadFilesContent('html', ['index_template.html'])
125
126 return self.processedFiles[processedFile]
127
128
129 def cssminCompressor (self, css):
130 # package found here:
131 # - http://stackoverflow.com/questions/222581/python-script-for-minifying-css/2396777#2396777
132 # actual downloaded version: http://pypi.python.org/pypi/cssmin/0.1.4
133 return cssmin.cssmin(css)
134
135
136 def regexCssCompressor (self, css):
137 # http://stackoverflow.com/questions/222581/python-script-for-minifying-css/223689#223689
138
139 # remove comments - this will break a lot of hacks :-P
140 css = re.sub( r'\s*/\*\s*\*/', "$$HACK1$$", css ) # preserve IE<6 comment hack
141 css = re.sub( r'/\*[\s\S]*?\*/', "", css )
142 css = css.replace( "$$HACK1$$", '/**/' ) # preserve IE<6 comment hack
143
144 # url() doesn't need quotes
145 css = re.sub( r'url\((["\'])([^)]*)\1\)', r'url(\2)', css )
146
147 # spaces may be safely collapsed as generated content will collapse them anyway
148 css = re.sub( r'\s+', ' ', css )
149
150 # shorten collapsable colors: #aabbcc to #abc
151 css = re.sub( r'#([0-9a-f])\1([0-9a-f])\2([0-9a-f])\3(\s|;)', r'#\1\2\3\4', css )
152
153 # fragment values can loose zeros
154 css = re.sub( r':\s*0(\.\d+([cm]m|e[mx]|in|p[ctx]))\s*;', r':\1;', css )
155
156 for rule in re.findall( r'([^{]+){([^}]*)}', css ):
157
158 # we don't need spaces around operators
159 selectors = [re.sub( r'(?<=[\[\(>+=])\s+|\s+(?=[=~^$*|>+\]\)])', r'', selector.strip() ) for selector in rule[0].split( ',' )]
160
161 # order is important, but we still want to discard repetitions
162 properties = {}
163 porder = []
164 for prop in re.findall( '(.*?):(.*?)(;|$)', rule[1] ):
165 key = prop[0].strip().lower()
166 if key not in porder: porder.append( key )
167 properties[ key ] = prop[1].strip()
168
169 # output rule if it contains any declarations
170 if properties:
171 print "%s{%s}" % ( ','.join( selectors ), ''.join(['%s:%s;' % (key, properties[key]) for key in porder])[:-1] )
172
173 return css
174
175
176 def compressCSS (self, css):
177 self.log("compressing CSS")
178 #return self.regexCssCompressor(css)
179 return self.cssminCompressor(css)
180
181
182 #==========================================================================
183
184 def compressJS_jsmin (self, js):
185 self.log("compressing JS code")
186 original = StringIO.StringIO(js)
187 output = StringIO.StringIO()
188
189 jsMinifier = jsmin.JavascriptMinify()
190 jsMinifier.minify(original, output)
191
192 result = output.getvalue()
193
194 original.close()
195 output.close()
196
197 return result
198
199 def compressJS_closureCompiler (self, js):
200 #Googles Closure compiler
201 #java -jar compiler.jar --js=in1.js --js=in2.js ... --js_output_file=out.js
202
203 result = js
204
205 return result
206
207
208 def compressJS (self, js):
209 return self.compressJS_jsmin(js)
210 #return self.compressJS_closureCompiler(js)
211
212
213 #==========================================================================
214
215 def packBookmarklet (self, bookmakeletCode):
216 replacers = [
217 ('isLoginForm', 'ilf'),
218 ('findLoginForm', 'flf'),
219 ('findLoginForm', 'flf'),
220 ('formParameters', 'fp' ),
221 ('pageParameters', 'pp' ),
222 ('serializeJSON', 'sj' ),
223 ('reprString', 'rs' ),
224 ('logFormParameters', 'lfp'),
225 ('loadClipperzBookmarklet','lcb'),
226 ('loginForm', 'lf' ),
227 ('parameters', 'p' ),
228 ('inputElementValues', 'iev'),
229 ]
230 result = self.compressJS(bookmakeletCode)
231
232 result = re.sub('\n', ' ', result) #Fit all in a single line
233 # result = re.sub('\s+', ' ', result) #Collapse "redundant" spaces. WARNING: this could have some evil side effects on constant strings used inside to code!!
234 # result = re.sub('\s?([,\+=\(\)\{\};])\s?', '\\1', result)
235
236 for replacer in replacers:
237 result = re.sub(replacer[0], replacer[1], result)
238
239 # <!-- escaping required to handle the bookmarklet code within the javascript code -->
240 result = re.sub('\://', '%3a%2f%2f',result)
241 result = re.sub('/', '%2f', result)
242 # result = re.sub('"', '%22', result)
243 result = re.sub('"', '\\"', result)
244 result = re.sub('\"', '%22', result)
245 result = re.sub('\'', '%22', result)
246 result = re.sub('\\\\', '%5c', result)
247 result = result.strip()
248 result = 'javascript:' + result
249
250 # replacers = [
251 # ('aForm', '_1' ),
252 # ('inputFields', '_2' ),
253 # ('passwordFieldsFound','_3' ),
254 # ('aDocument', '_6' ),
255 # ('aLevel', '_7' ),
256 # # ('result', '_8' ),
257 # ('documentForms', '_9' ),
258 # ('iFrames', '_c' ),
259 # ('anInputElement', '_d' ),
260 # ('options', '_f' ),
261 # ('option', '_12'),
262 # ('aLoginForm', '_13'),
263 # # ('action', '_17'),
264 # ('radioValues', '_18'),
265 # ('radioValueName', '_19'),
266 # ('inputElement', '_1a'),
267 # ('elementValues', '_1b'),
268 # ('radioValue', '_1c'),
269 # ('values', '_1d'),
270 # ('objtype', '_21'),
271 # ('useKey', '_27'),
272 # ('bookmarkletDiv', '_28'),
273 # ('someParameters', '_29'),
274 # ('anException', '_2a'),
275 # ('newDiv', '_2b'),
276 # ('base_url', '_2c'),
277 # ('help_url', '_2d'),
278 # ('logo_image_url', '_2e'),
279 # ('background_image_url','_2f'),
280 # ('close_image_url', '_30'),
281 # #('bookmarklet_textarea','_31'),
282 # ('innerHTML', '_32'),
283 # ]
284 # for replacer in replacers:
285 # result = re.sub('([^\.])' + replacer[0], '\\1' + replacer[1], result)
286
287 # replacers = [
288 # ('headNode', '_1' ),
289 # ('clipperzScriptNode','_2' ),
290 # ]
291 # for replacer in replacers:
292 # result = re.sub('([^\.])' + replacer[0], '\\1' + replacer[1], result)
293
294 # result = re.sub(';', ';\n', result)
295
296 return result
297
298
299
300 def bookmarklet (self):
301 cacheKey = 'bookmarklet'
302 if not self.processedFiles.has_key(cacheKey):
303 result = 'bookmarklet="' + self.packBookmarklet(self.loadFilesContent('js', ['Bookmarklet.js'])) + '";bookmarklet_ie="' + self.packBookmarklet(self.loadFilesContent('js', ['Bookmarklet_IE.js'])) + '";'
304 self.processedFiles[cacheKey] = result
305 else:
306 result = self.processedFiles[cacheKey]
307
308 return result
309
310
311 def replaceTemplatePlaceholders (self, assemblyMode, pageTitle, copyright, css, code, version, versionType):
312 result = self.template()
313
314 result = result.replace('@page.title@', pageTitle, 1)
315 result = result.replace('@copyright@', copyright, 1)
316 result = result.replace('@css@', css, 1)
317 #result = result.replace('@bookmarklet@', bookmarklet,1)
318 result = result.replace('@application.version@', version, 1)
319 result = result.replace('@application.version.type@', versionType,1)
320 result = result.replace('@js_' + assemblyMode + '@', code, 1)
321
322 result = re.sub('@js_[^@]+@', '', result)
323
324 return result
325
326
327 def assembleCopyrightHeader (self):
328 processedFile = 'copyright'
329 if not self.processedFiles.has_key(processedFile):
330 #self.log("assembling copyright header")
331 copyrightValues = self.settings['copyright.values']
332 license = self.loadFilesContent('../../properties', ['license.txt'])
333 result = self.loadFilesContent('properties', ['creditsAndCopyrights.txt'])
334
335 result = re.sub('@clipperz.license@', license, result)
336 for key in copyrightValues:
337 result = re.sub('@'+key+'@', copyrightValues[key], result)
338
339 self.processedFiles[processedFile] = result
340
341 return self.processedFiles[processedFile]
342
343
344 def cssTagsForFiles (self, basePath, files):
345 #<link rel="stylesheet" type="text/css" href="./css/reset-min.css" />
346 return '\n'.join(map(lambda file: '<link rel="stylesheet" type="text/css" href="./' + basePath + '/' + file + '" />', files))
347
348
349 def cssTagForContent (self, content):
350 return '<style type="text/css">' + content + '</style>'
351
352
353 def scriptTagsForFiles (self, basePath, files):
354 #<script type='text/javascript' src='./js/src/bookmarklet.js'></script>
355 return '\n'.join(map(lambda file: '<script type="text/javascript" src="./' + basePath + '/' + file + '"></script>', files))
356
357
358 def scriptTagForContent (self, content):
359 return '<script>' + content + '</script>'
360
361
362 def assembleVersion (self, assemblyMode, pageTitle, copyright, css, js, version, versionType):
363 cacheKey = version + "-" + versionType
364 if not self.processedFiles.has_key(cacheKey):
365 result = self.replaceTemplatePlaceholders(assemblyMode, pageTitle, copyright, css, js, version, versionType)
366 self.processedFiles[cacheKey] = result
367 else:
368 result = self.processedFiles[cacheKey]
369
370 #self.log("# cacheKey:\n" + result)
371 return result
372
373
374 def assemble (self, assemblyMode='INSTALL', versionType='LIVE'):
375 pageTitle = "Clipperz - " + self.module
376 if versionType != 'LIVE':
377 pageTitle += " [" + versionType + " - " + assemblyMode +"]"
378
379 if assemblyMode == 'INSTALL':
380 css= self.cssTagForContent(self.compressCSS(self.loadFilesContent('css', self.settings['css'])))
381 js= self.scriptTagForContent(self.bookmarklet() + '\n' + self.compressJS(self.loadFilesContent('js', self.settings['js'])))
382 else:
383 css= self.cssTagsForFiles('css', self.filterFiles(self.settings['css']))
384 js= self.scriptTagForContent(self.bookmarklet()) + '\n' + self.scriptTagsForFiles('js', self.filterFiles(self.settings['js']))
385
386 return self.assembleVersion(
387 assemblyMode= assemblyMode,
388 pageTitle = pageTitle,
389 copyright = self.assembleCopyrightHeader(),
390 css = css,
391 js = js,
392 version = self.repositoryVersion(),
393 versionType = versionType
394 )
395
396
397
398
diff --git a/scripts/builder/jsmin.py b/scripts/builder/jsmin.py
new file mode 100644
index 0000000..91d6307
--- a/dev/null
+++ b/scripts/builder/jsmin.py
@@ -0,0 +1,246 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4import os, os.path, shutil
5
6# This code is original from jsmin by Douglas Crockford, it was translated to
7# Python by Baruch Even. The original code had the following copyright and
8# license.
9#
10# /* jsmin.c
11# 2007-05-22
12#
13# Copyright (c) 2002 Douglas Crockford (www.crockford.com)
14#
15# Permission is hereby granted, free of charge, to any person obtaining a copy of
16# this software and associated documentation files (the "Software"), to deal in
17# the Software without restriction, including without limitation the rights to
18# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
19# of the Software, and to permit persons to whom the Software is furnished to do
20# so, subject to the following conditions:
21#
22# The above copyright notice and this permission notice shall be included in all
23# copies or substantial portions of the Software.
24#
25# The Software shall be used for Good, not Evil.
26#
27# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
28# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
29# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
30# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
31# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
32# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33# SOFTWARE.
34# */
35
36from StringIO import StringIO
37
38def jsmin(js):
39 ins = StringIO(js)
40 outs = StringIO()
41 JavascriptMinify().minify(ins, outs)
42 str = outs.getvalue()
43 if len(str) > 0 and str[0] == '\n':
44 str = str[1:]
45 return str
46
47def isAlphanum(c):
48 """return true if the character is a letter, digit, underscore,
49 dollar sign, or non-ASCII character.
50 """
51 return ((c >= 'a' and c <= 'z') or (c >= '0' and c <= '9') or
52 (c >= 'A' and c <= 'Z') or c == '_' or c == '$' or c == '\\' or (c is not None and ord(c) > 126));
53
54class UnterminatedComment(Exception):
55 pass
56
57class UnterminatedStringLiteral(Exception):
58 pass
59
60class UnterminatedRegularExpression(Exception):
61 pass
62
63class JavascriptMinify(object):
64
65 def _outA(self):
66 self.outstream.write(self.theA)
67 def _outB(self):
68 self.outstream.write(self.theB)
69
70 def _get(self):
71 """return the next character from stdin. Watch out for lookahead. If
72 the character is a control character, translate it to a space or
73 linefeed.
74 """
75 c = self.theLookahead
76 self.theLookahead = None
77 if c == None:
78 c = self.instream.read(1)
79 if c >= ' ' or c == '\n':
80 return c
81 if c == '': # EOF
82 return '\000'
83 if c == '\r':
84 return '\n'
85 return ' '
86
87 def _peek(self):
88 self.theLookahead = self._get()
89 return self.theLookahead
90
91 def _next(self):
92 """get the next character, excluding comments. peek() is used to see
93 if an unescaped '/' is followed by a '/' or '*'.
94 """
95 c = self._get()
96 if c == '/' and self.theA != '\\':
97 p = self._peek()
98 if p == '/':
99 c = self._get()
100 while c > '\n':
101 c = self._get()
102 return c
103 if p == '*':
104 c = self._get()
105 while 1:
106 c = self._get()
107 if c == '*':
108 if self._peek() == '/':
109 self._get()
110 return ' '
111 if c == '\000':
112 raise UnterminatedComment()
113
114 return c
115
116 def _action(self, action):
117 """do something! What you do is determined by the argument:
118 1 Output A. Copy B to A. Get the next B.
119 2 Copy B to A. Get the next B. (Delete A).
120 3 Get the next B. (Delete B).
121 action treats a string as a single character. Wow!
122 action recognizes a regular expression if it is preceded by ( or , or =.
123 """
124 if action <= 1:
125 self._outA()
126
127 if action <= 2:
128 self.theA = self.theB
129 if self.theA == "'" or self.theA == '"':
130 while 1:
131 self._outA()
132 self.theA = self._get()
133 if self.theA == self.theB:
134 break
135 if self.theA <= '\n':
136 raise UnterminatedStringLiteral()
137 if self.theA == '\\':
138 self._outA()
139 self.theA = self._get()
140
141
142 if action <= 3:
143 self.theB = self._next()
144 if self.theB == '/' and (self.theA == '(' or self.theA == ',' or
145 self.theA == '=' or self.theA == ':' or
146 self.theA == '[' or self.theA == '?' or
147 self.theA == '!' or self.theA == '&' or
148 self.theA == '|' or self.theA == ';' or
149 self.theA == '{' or self.theA == '}' or
150 self.theA == '\n'):
151 self._outA()
152 self._outB()
153 while 1:
154 self.theA = self._get()
155 if self.theA == '/':
156 break
157 elif self.theA == '\\':
158 self._outA()
159 self.theA = self._get()
160 elif self.theA <= '\n':
161 raise UnterminatedRegularExpression()
162 self._outA()
163 self.theB = self._next()
164
165
166 def _jsmin(self):
167 """Copy the input to the output, deleting the characters which are
168 insignificant to JavaScript. Comments will be removed. Tabs will be
169 replaced with spaces. Carriage returns will be replaced with linefeeds.
170 Most spaces and linefeeds will be removed.
171 """
172 self.theA = '\n'
173 self._action(3)
174
175 while self.theA != '\000':
176 if self.theA == ' ':
177 if isAlphanum(self.theB):
178 self._action(1)
179 else:
180 self._action(2)
181 elif self.theA == '\n':
182 if self.theB in ['{', '[', '(', '+', '-']:
183 self._action(1)
184 elif self.theB == ' ':
185 self._action(3)
186 else:
187 if isAlphanum(self.theB):
188 self._action(1)
189 else:
190 self._action(2)
191 else:
192 if self.theB == ' ':
193 if isAlphanum(self.theA):
194 self._action(1)
195 else:
196 self._action(3)
197 elif self.theB == '\n':
198 if self.theA in ['}', ']', ')', '+', '-', '"', '\'']:
199 self._action(1)
200 else:
201 if isAlphanum(self.theA):
202 self._action(1)
203 else:
204 self._action(3)
205 else:
206 self._action(1)
207
208 def minify(self, instream, outstream):
209 self.instream = instream
210 self.outstream = outstream
211 self.theA = '\n'
212 self.theB = None
213 self.theLookahead = None
214
215 self._jsmin()
216 self.instream.close()
217
218def compress(in_files, out_file, in_type='js', verbose=False, temp_file='.temp'):
219 temp = open(temp_file, 'w')
220 for f in in_files:
221 fh = open(f)
222 data = fh.read() + '\n'
223 fh.close()
224
225 temp.write(data)
226
227 print ' + %s' % f
228 temp.close()
229
230 out = open(out_file, 'w')
231
232 jsm = JavascriptMinify()
233 jsm.minify(open(temp_file,'r'), out)
234
235 out.close()
236
237 org_size = os.path.getsize(temp_file)
238 new_size = os.path.getsize(out_file)
239
240 print '=> %s' % out_file
241 print 'Original: %.2f kB' % (org_size / 1024.0)
242 print 'Compressed: %.2f kB' % (new_size / 1024.0)
243 print 'Reduction: %.1f%%' % (float(org_size - new_size) / org_size * 100)
244 print ''
245
246 os.remove(temp_file) \ No newline at end of file
diff --git a/scripts/builder/main.py b/scripts/builder/main.py
new file mode 100755
index 0000000..ba0c72a
--- a/dev/null
+++ b/scripts/builder/main.py
@@ -0,0 +1,166 @@
1#!/usr/bin/python
2# -*- coding: UTF-8 -*-
3
4import sys, os, json
5import shutil
6import pprint
7import frontendBuilder
8import codecs
9import itertools
10
11from collections import deque
12from phpBuilder import PhpBuilder
13from pythonBuilder import PythonBuilder
14
15pp = pprint.PrettyPrinter(indent=4, depth=4)
16
17#--------------------------------------------------------------------
18
19def scriptDir ():
20 return os.path.dirname(sys.argv[0])
21
22def projectBaseDir ():
23 return os.path.abspath(scriptDir() + '/../..')
24
25def projectTargetDir():
26 return projectBaseDir() + '/target/'
27
28#--------------------------------------------------------------------
29
30def createFolder (path):
31 if not os.path.exists(path):
32 os.makedirs(path)
33
34#--------------------------------------------------------------------
35
36def loadSettings (component, module):
37 print "MODULE: " + module
38
39 if '.' in module:
40 moduleComponents = module.split('.')
41 module = moduleComponents[0]
42 submodule = moduleComponents[1]
43 else:
44 submodule = module
45
46 settings = codecs.open(projectBaseDir() + '/' + component + '/' + module + '/properties/' + submodule + '.properties.json', 'r', 'utf-8')
47 result = json.load(settings)
48 settings.close
49
50 return result
51
52#====================================================================
53#
54# def assembleFrontend (frontend, versions):
55 # result = {}
56 # settings = loadSettings('frontend', frontend)
57 # builder = frontendBuilder.FrontendBuilder(frontend, settings, projectBaseDir())
58 #
59 # for version in versions:
60 # if version == 'install':
61 # result[version] = builder.assembleInstallVersion()
62 # elif version == 'debug':
63 # result[version] = builder.assembleDebugVersion()
64 # else:
65 # raise Exception('unrecognized version: ' + version)
66 #
67 # return result
68#
69#====================================================================
70
71def assembleBackend (backend, frontends, versions):
72 settings = loadSettings('backend', backend)
73
74 if backend == 'php':
75 backendBuilder = PhpBuilder(projectTargetDir(), frontends, versions, settings)
76 elif backend == 'python':
77 backendBuilder = PythonBuilder(projectTargetDir(), frontends, versions, settings)
78 #elif backend == 'java':
79 #buildJavaBackend (frontends, versions, settings)
80 else:
81 raise Exception('unrecognized backend: ' + backend)
82
83 backendBuilder.run()
84
85#====================================================================
86
87def build (settings):
88 frontends = []
89
90 for frontend in settings['frontends']:
91 frontends.append(frontendBuilder.FrontendBuilder(frontend, loadSettings('frontend', frontend)))
92
93 for backend in settings['backends']:
94 assembleBackend(backend, frontends, settings['versions'])
95
96#--------------------------------------------------------------------
97
98def clean ():
99 print "cleaning up …"
100 if os.path.exists(projectTargetDir()):
101 shutil.rmtree(projectTargetDir())
102
103#--------------------------------------------------------------------
104
105def usage (message):
106 if message != None:
107 print "ERROR: " + message
108
109 print
110 print "build.py clean"
111 print "build.py clean install"
112 print "build.py install --ALL"
113 print "build.py install debug --ALL"
114 print "build.py clean install debug --ALL"
115 print "build.ph install, debug --backends php java --frontends beta gamma"
116 print "build.ph install, debug --backends php java --frontends beta gamma gamma.mobile"
117 exit(1)
118
119#--------------------------------------------------------------------
120
121def main ():
122 settings = {}
123 parameters = list(itertools.islice(sys.argv, 1, None))
124
125 shouldClean = len(filter(lambda x: x == 'clean', parameters)) > 0
126 if (shouldClean):
127 clean ()
128
129 parameters = filter(lambda x: x != 'clean', parameters)
130 versions = list(itertools.takewhile(lambda x: not x.startswith('--'), parameters))
131 settings['versions'] = versions; #['debug', 'install']
132 parameters = deque(itertools.dropwhile(lambda x: not x.startswith('--'), parameters))
133
134 if len(parameters) > 0:
135 parameter = parameters.popleft()
136 if parameter == "--ALL":
137 settings['frontends'] = ['beta', 'gamma', 'mobile']
138 settings['backends'] = ['php', 'python', 'java']
139 else:
140 while parameter != None:
141 values = list(itertools.takewhile(lambda x: not x.startswith('--'), parameters))
142
143 if parameter == "--backends":
144 settings['backends'] = values
145 elif parameter == "--frontends":
146 settings['frontends'] = values
147
148 parameters = deque(itertools.dropwhile(lambda x: not x.startswith('--'), parameters))
149 if parameters:
150 parameter = parameters.popleft()
151 else:
152 parameter = None
153
154 if (not settings.has_key('versions')):
155 usage("missing 'versions'")
156 if (not settings.has_key('frontends')):
157 usage("missing 'frontends'")
158 if (not settings.has_key('backends')):
159 usage("missing 'backends'")
160
161 build (settings)
162
163
164
165if __name__ == "__main__":
166 main() \ No newline at end of file
diff --git a/scripts/builder/phpBuilder.py b/scripts/builder/phpBuilder.py
new file mode 100644
index 0000000..9512192
--- a/dev/null
+++ b/scripts/builder/phpBuilder.py
@@ -0,0 +1,14 @@
1#!/usr/bin/python
2# -*- coding: UTF-8 -*-
3
4from backendBuilder import BackendBuilder
5
6class PhpBuilder(BackendBuilder):
7
8 def name(self):
9 return "PHP builder"
10
11 def relativePath(self):
12 return 'php'
13
14
diff --git a/scripts/builder/pythonBuilder.py b/scripts/builder/pythonBuilder.py
new file mode 100644
index 0000000..44c62a8
--- a/dev/null
+++ b/scripts/builder/pythonBuilder.py
@@ -0,0 +1,14 @@
1#!/usr/bin/python
2# -*- coding: UTF-8 -*-
3
4from backendBuilder import BackendBuilder
5
6class PythonBuilder(BackendBuilder):
7
8 def name(self):
9 return "Python builder"
10
11 def relativePath(self):
12 return 'python'
13
14