#!/usr/bin/env python2.7
#
# Note: python2.6 is specified because that is what the skulpt parser
# used as a reference. This is only important when you are doing
# things like regenerating tests and/or regenerating symtabs
# If you do not have python 2.6 and you ARE NOT creating new tests
# then all should be well for you to use 2.7 or whatever you have around
from optparse import OptionParser
from subprocess import Popen, PIPE
import subprocess
import os
import sys
import glob
import py_compile
import symtable
import shutil
import re
import pprint
import json
import shutil
import time
# Assume that the GitPython module is available until proven otherwise.
GIT_MODULE_AVAILABLE = False
try:
from git import *
except:
GIT_MODULE_AVAILABLE = False
def bowerFileName():
file = open(".bowerrc")
data = json.load(file)
fileName = data["json"]
file.close()
return fileName
def bowerProperty(name):
file = open(bowerFileName())
data = json.load(file)
value = data[name]
file.close()
return value
# Symbolic constants for the project structure.
DIST_DIR = 'dist'
TEST_DIR = 'test'
RUN_DIR = 'support/tmp'
# Symbolic constants for the naming of distribution files.
STANDARD_NAMING = True
PRODUCT_NAME = bowerProperty("name")
OUTFILE_REG = "{0}.js".format(PRODUCT_NAME) if STANDARD_NAMING else "skulpt-uncomp.js"
OUTFILE_MIN = "{0}.min.js".format(PRODUCT_NAME) if STANDARD_NAMING else "skulpt.js"
OUTFILE_LIB = "{0}-stdlib.js".format(PRODUCT_NAME) if STANDARD_NAMING else "builtin.js"
OUTFILE_MAP = "{0}-linemap.txt".format(PRODUCT_NAME) if STANDARD_NAMING else "linemap.txt"
OUTFILE_DEBUGGER = "debugger.js"
# Symbolic constants for file types.
FILE_TYPE_DIST = 'dist'
FILE_TYPE_TEST = 'test'
# Order is important!
Files = [
'support/closure-library/closure/goog/base.js',
'support/closure-library/closure/goog/deps.js',
('support/closure-library/closure/goog/string/string.js', FILE_TYPE_DIST),
('support/closure-library/closure/goog/debug/error.js', FILE_TYPE_DIST),
('support/closure-library/closure/goog/asserts/asserts.js', FILE_TYPE_DIST),
('support/es6-promise-polyfill/promise-1.0.0.hacked.js', FILE_TYPE_DIST),
'src/env.js',
'src/type.js',
'src/abstract.js',
'src/object.js',
'src/function.js',
'src/builtin.js',
'src/fromcodepoint.js', # should become unnecessary, eventually
'src/errors.js',
'src/native.js',
'src/method.js',
'src/misceval.js',
'src/seqtype.js',
'src/list.js',
'src/str.js',
'src/formatting.js',
'src/tuple.js',
'src/dict.js',
'src/numtype.js',
'src/biginteger.js',
'src/int.js',
'src/bool.js',
'src/float.js',
'src/number.js',
'src/long.js',
'src/complex.js',
'src/slice.js',
'src/set.js',
'src/print.js',
'src/module.js',
'src/structseq.js',
'src/generator.js',
'src/file.js',
'src/ffi.js',
'src/iterator.js',
'src/enumerate.js',
'src/tokenize.js',
'gen/parse_tables.js',
'src/parser.js',
'gen/astnodes.js',
'src/ast.js',
'src/symtable.js',
'src/compile.js',
'src/import.js',
'src/timsort.js',
'src/sorted.js',
'src/builtindict.js',
'src/constants.js',
'src/internalpython.js',
("support/jsbeautify/beautify.js", FILE_TYPE_TEST),
]
ExtLibs = [
'support/time-helpers/strftime-min.js',
'support/time-helpers/strptime.min.js'
]
TestFiles = [
'support/closure-library/closure/goog/base.js',
'support/closure-library/closure/goog/deps.js',
'support/closure-library/closure/goog/math/math.js',
'support/closure-library/closure/goog/math/coordinate.js',
'support/closure-library/closure/goog/math/vec2.js',
'support/closure-library/closure/goog/json/json.js',
'support/jsbeautify/beautify.js',
"{0}/namedtests.js".format(TEST_DIR),
"{0}/sprintf.js".format(TEST_DIR),
"{0}/json2.js".format(TEST_DIR),
"{0}/test.js".format(TEST_DIR)
]
def buildNamedTestsFile():
testFiles = ['test/run/'+f.replace(".py","") for f in os.listdir('test/run') if re.match(r"test_.*\.py$",f)]
nt = open("{0}/namedtests.js".format(TEST_DIR),'w')
nt.write("namedtfiles = [")
for f in testFiles:
nt.write("'%s',\n" % f)
nt.write("];")
nt.close()
def isClean():
repo = Repo(".")
return not repo.is_dirty()
def getTip():
repo = Repo(".")
return repo.head.commit.hexsha
def getFileList(type, include_ext_libs=True):
ret = list(ExtLibs) if include_ext_libs else []
for f in Files:
if isinstance(f, tuple):
if f[1] == type:
ret.append(f[0])
else:
if "*" in f:
for g in glob.glob(f):
ret.append(f)
else:
ret.append(f)
return ret
def is64bit():
return sys.maxsize > 2**32
if sys.platform == "win32":
winbase = ".\\support\\d8"
os.environ["D8_PATH"] = winbase
jsengine = winbase + "\\d8.exe --debugger --harmony"
nul = "nul"
crlfprog = os.path.join(os.path.split(sys.executable)[0], "Tools/Scripts/crlf.py")
elif sys.platform == "darwin":
os.environ["D8_PATH"] = "./support/d8/mac"
jsengine = "./support/d8/mac/d8 --debugger"
nul = "/dev/null"
crlfprog = None
elif sys.platform == "linux2":
if is64bit():
os.environ["D8_PATH"] = "support/d8/x64"
jsengine = "support/d8/x64/d8 --debugger --harmony_promises"
else:
os.environ["D8_PATH"] = "support/d8/x32"
jsengine = "support/d8/x32/d8 --debugger --harmony_promises"
nul = "/dev/null"
crlfprog = None
else:
# You're on your own...
os.environ["D8_PATH"] = "support/d8/x32"
jsengine = "support/d8/x32/d8 --debugger --harmony_promises"
nul = "/dev/null"
crlfprog = None
if os.environ.get("CI",False):
os.environ["D8_PATH"] = "support/d8/x64"
jsengine = "support/d8/x64/d8 --harmony_promises"
nul = "/dev/null"
#jsengine = "rhino"
def test(debug_mode=False):
"""runs the unit tests."""
if debug_mode:
debugon = "--debug-mode"
else:
debugon = ""
buildNamedTestsFile()
ret1 = os.system("{0} {1} {2} -- {3}".format(jsengine, ' '.join(getFileList(FILE_TYPE_TEST)), ' '.join(TestFiles), debugon))
ret2 = 0
ret3 = 0
ret4 = 0
if ret1 == 0:
print("Running jshint")
base_dirs = ["src", "debugger"]
for base_dir in base_dirs:
if sys.platform == "win32":
jshintcmd = "{0} {1}".format("jshint", ' '.join(f for f in glob.glob(base_dir + "/*.js")))
jscscmd = "{0} {1} --reporter=inline".format("jscs", ' '.join(f for f in glob.glob(base_dir + "/*.js")))
else:
jshintcmd = "jshint " + base_dir + "/*.js"
jscscmd = "jscs " + base_dir + "/*.js --reporter=inline"
ret2 = os.system(jshintcmd)
print("Running JSCS")
ret3 = os.system(jscscmd)
#ret3 = os.system(jscscmd)
print("Now running new unit tests")
ret4 = rununits()
return ret1 | ret2 | ret3 | ret4
def parse_time_args(argv):
usageString = """
{program} time [filename.py] [iter=1]
Computes the average runtime of a Python file (or test suite, if none specified)
over iter number of trials.
""".format(program=argv[0])
fn = ""
iter = 0
if len(sys.argv) > 4:
print(usageString)
sys.exit(2)
for arg in argv[2:]:
if arg.isdigit():
if iter:
print(usageString)
sys.exit(2)
else:
iter = int(arg)
if iter <= 0:
print("Number of trials must be 1 or greater.")
sys.exit(2)
elif ".py" in arg:
if fn:
print(usageString)
sys.exit(2)
else:
fn = arg
else:
print(usageString)
sys.exit(2)
iter = iter if iter else 1
time_suite(iter=iter, fn=fn)
def time_suite(iter=1, fn=""):
jsprofengine = jsengine.replace('--debugger', '--prof --log-internal-timer-events')
if not os.path.exists("support/tmp"):
os.mkdir("support/tmp")
f = open("support/tmp/run.js", "w")
additional_files = ""
# Profile single file
if fn:
if not os.path.exists(fn):
print("%s doesn't exist" % fn)
raise SystemExit()
modname = os.path.splitext(os.path.basename(fn))[0]
f.write("""
var input = read('%s');
print("-----");
print(input);
print("-----");
Sk.configure({syspath:["%s"], read:read, python3:false, debugging:false});
Sk.misceval.asyncToPromise(function() {
return Sk.importMain("%s", true, true);
}).then(function () {
print("-----");
}, function(e) {
print("UNCAUGHT EXCEPTION: " + e);
print(e.stack);
});
""" % (fn, os.path.split(fn)[0], modname))
# Profile test suite
else:
# Prepare named tests
buildNamedTestsFile()
# Prepare unit tests
testFiles = ['test/unit/'+fn for fn in os.listdir('test/unit') if '.py' in fn]
if not os.path.exists("support/tmp"):
os.mkdir("support/tmp")
f.write("var input;\n")
for fn in testFiles:
modname = os.path.splitext(os.path.basename(fn))[0]
p3on = 'false'
f.write("""
input = read('%s');
print('%s');
Sk.configure({syspath:["%s"], read:read, python3:%s});
Sk.importMain("%s", false);
""" % (fn, fn, os.path.split(fn)[0], p3on, modname))
fn = "test suite"
additional_files = ' '.join(TestFiles)
f.close()
print("Timing %s...\n" % fn)
times = []
# Run profile
for i in range(iter):
if iter > 1:
print("Iteration %d of %d..." % (i + 1, iter))
startTime = time.time()
p = Popen("{0} {1} {2} support/tmp/run.js".format(jsprofengine,
' '.join(getFileList(FILE_TYPE_TEST)),
additional_files),
shell=True, stdout=PIPE, stderr=PIPE)
outs, errs = p.communicate()
if p.returncode != 0:
print("\n\nWARNING: Scripts returned with error code. Timing data may be inaccurate.\n\n")
endTime = time.time()
times.append(endTime - startTime)
avg = sum(times) / len(times)
if iter > 1:
print("\nAverage time over %s iterations: %s seconds" % (iter, avg))
else:
print("%s seconds" % avg)
def parse_profile_args(argv):
usageString = """
{program} profile [filename.py] [output]
Runs profile on Python file (or test suite, if none specified)
and outputs processed results to output file (or stdout if none specified)
""".format(program=argv[0])
fn = ""
out = ""
numArgs = len(sys.argv)
if len(sys.argv) > 4:
print(usageString)
sys.exit(2)
for arg in argv[2:]:
if ".py" in arg:
if fn:
print(usageString)
sys.exit(2)
else:
fn = arg
else:
if out:
print(usageString)
sys.exit(2)
else:
out = arg
profile(fn=fn, output=out)
def profile(fn="", process=True, output=""):
"""
Runs v8 profiler, which outputs tick information to v8.log Use
https://v8.googlecode.com/svn/branches/bleeding_edge/tools/profviz/profviz.html
to analyze log.
"""
jsprofengine = jsengine.replace('--debugger', '--prof --log-internal-timer-events')
if not os.path.exists("support/tmp"):
os.mkdir("support/tmp")
f = open("support/tmp/run.js", "w")
additional_files = ""
# Profile single file
if fn:
if not os.path.exists(fn):
print("%s doesn't exist" % fn)
raise SystemExit()
modname = os.path.splitext(os.path.basename(fn))[0]
f.write("""
var input = read('%s');
print("-----");
print(input);
print("-----");
Sk.configure({syspath:["%s"], read:read, python3:false, debugging:false});
Sk.misceval.asyncToPromise(function() {
return Sk.importMain("%s", true, true);
}).then(function () {
print("-----");
}, function(e) {
print("UNCAUGHT EXCEPTION: " + e);
print(e.stack);
});
""" % (fn, os.path.split(fn)[0], modname))
# Profile test suite
else:
# Prepare named tests
buildNamedTestsFile()
# Prepare unit tests
testFiles = ['test/unit/'+fn for fn in os.listdir('test/unit') if '.py' in fn]
if not os.path.exists("support/tmp"):
os.mkdir("support/tmp")
f.write("var input;\n")
for fn in testFiles:
modname = os.path.splitext(os.path.basename(fn))[0]
p3on = 'false'
f.write("""
input = read('%s');
print('%s');
Sk.configure({syspath:["%s"], read:read, python3:%s});
Sk.importMain("%s", false);
""" % (fn, fn, os.path.split(fn)[0], p3on, modname))
fn = "test suite"
additional_files = ' '.join(TestFiles)
f.close()
# Run profile
print("Running profile on %s..." % fn)
startTime = time.time()
p = Popen("{0} {1} {2} support/tmp/run.js".format(jsprofengine,
' '.join(getFileList(FILE_TYPE_TEST)),
additional_files),
shell=True, stdout=PIPE, stderr=PIPE)
outs, errs = p.communicate()
if p.returncode != 0:
print("\n\nWARNING: Scripts returned with error code. Timing data may be inaccurate.\n\n")
endTime = time.time()
if errs:
print(errs)
print("\n\nRunning time: ", (endTime - startTime), " seconds\n\n")
# Process and display results
if process:
if output:
out_msg = " and saving in %s" % output
output = " > " + output
else:
out_msg = ""
print("Processing profile using d8 processor%s..." % out_msg)
if sys.platform == "win32":
os.system(".\\support\\d8\\tools\\windows-tick-processor.bat v8.log {0}".format(output))
elif sys.platform == "darwin":
os.system("./support/d8/tools/mac-tick-processor {0}".format(output))
elif sys.platform == "linux2":
os.system("./support/d8/tools/linux-tick-processor v8.log {0}".format(output))
else:
print("""d8 processor is unsupported on this platform.
Try using https://v8.googlecode.com/svn/branches/bleeding_edge/tools/profviz/profviz.html.""")
def debugbrowser():
tmpl = """
Skulpt test
%s
"""
if not os.path.exists("support/tmp"):
os.mkdir("support/tmp")
buildVFS()
scripts = []
for f in getFileList(FILE_TYPE_TEST) + ["{0}/browser-stubs.js".format(TEST_DIR), "support/tmp/vfs.js" ] + TestFiles:
scripts.append('' %
os.path.join('../..', f))
with open("support/tmp/test.html", "w") as f:
print(tmpl % '\n'.join(scripts), file=f)
if sys.platform == "win32":
os.system("start support/tmp/test.html")
elif sys.platform == "darwin":
os.system("open support/tmp/test.html")
else:
os.system("gnome-open support/tmp/test.html")
def buildVFS():
""" build a silly virtual file system to support 'read'"""
print(". Slurping test data")
with open("support/tmp/vfs.js", "w") as out:
print("VFSData = {", file=out)
all = []
for root in (TEST_DIR, "src/builtin", "src/lib"):
for dirpath, dirnames, filenames in os.walk(root):
for filename in filenames:
f = os.path.join(dirpath, filename)
if ".svn" in f: continue
if ".swp" in f: continue
if ".pyc" in f: continue
data = open(f, "rb").read()
data = data.replace("\r\n", "\n")
all.append("'%s': '%s'" % (f.replace("\\", "/"), data.encode("hex")))
print(",\n".join(all), file=out)
print("};", file=out)
print("""
function readFromVFS(fn)
{
var hexToStr = function(str)
{
var ret = "";
for (var i = 0; i < str.length; i += 2)
ret += unescape("%" + str.substr(i, 2));
return ret;
}
if (VFSData[fn] === undefined) throw "file not found: " + fn;
return hexToStr(VFSData[fn]);
}
""", file=out)
def buildBrowserTests():
"""combine all the tests data into something we can run from a browser
page (so that it can be tested in the various crappy engines)
we want to use the same code that the command line version of the tests
uses so we stub the d8 functions to push to the browser."""
outfn = "doc/static/browser-test.js"
out = open(outfn, "w")
print("""
window.addevent('onload', function(){
""", file=out)
# stub the d8 functions we use
print("""
function read(fn)
{
var hexToStr = function(str)
{
var ret = "";
for (var i = 0; i < str.length; i += 2)
ret += unescape("%%" + str.substr(i, 2));
return ret;
}
if (VFSData[fn] === undefined) throw "file not found: " + fn;
return hexToStr(VFSData[fn]);
}
var SkulptTestRunOutput = '';
function print()
{
var out = document.getElementById("output");
for (var i = 0; i < arguments.length; ++i)
{
out.innerHTML += arguments[i];
SkulptTestRunOutput += arguments[i];
out.innerHTML += " ";
SkulptTestRunOutput += " ";
}
out.innerHTML += " "
SkulptTestRunOutput += "\\n";
}
function quit(rc)
{
var out = document.getElementById("output");
if (rc === 0)
{
out.innerHTML += "OK";
}
else
{
out.innerHTML += "FAILED";
}
out.innerHTML += " Saving results...";
var sendData = JSON.encode({
browsername: BrowserDetect.browser,
browserversion: BrowserDetect.version,
browseros: BrowserDetect.OS,
version: '%s',
rc: rc,
results: SkulptTestRunOutput
});
var results = new Request.JSON({
url: '/testresults',
method: 'post',
onSuccess: function() { out.innerHTML += " Results saved."; },
onFailure: function() { out.innerHTML += " Couldn't save results."; }
});
results.send(sendData);
}
""" % getTip(), file=out)
for f in ["{0}/browser-detect.js".format(TEST_DIR)] + getFileList(FILE_TYPE_TEST) + TestFiles:
print(open(f).read(), file=out)
print("""
});
""", file=out)
out.close()
print(". Built %s" % outfn)
def getInternalCodeAsJson():
ret = {}
ret['files'] = {}
for f in ["src/" + x for x in os.listdir("src") if os.path.splitext(x)[1] == ".py" if os.path.isfile("src/" + x)]:
ext = os.path.splitext(f)[1]
if ext == ".py":
f = f.replace("\\", "/")
ret['files'][f] = open(f).read()
return "Sk.internalPy=" + json.dumps(ret)
def getBuiltinsAsJson(options):
ret = {}
ret['files'] = {}
for root in ["src/builtin", "src/lib"]:
for dirpath, dirnames, filenames in os.walk(root):
for filename in filenames:
f = os.path.join(dirpath, filename)
ext = os.path.splitext(f)[1]
if ext == ".py" or ext == ".js":
if options.verbose:
print("reading", f)
f = f.replace("\\", "/")
ret['files'][f] = open(f).read()
return "Sk.builtinFiles=" + json.dumps(ret)
def dist(options):
"""builds a 'shippable' version of Skulpt.
this is all combined into one file, tests run, jslint'd, compressed.
"""
if GIT_MODULE_AVAILABLE:
if not isClean():
print("WARNING: working directory not clean (according to 'git status')")
else:
print("Working directory is clean (according to 'git status')")
else:
'''
# We don't really use GitPython
print("+----------------------------------------------------------------------------+")
print("GitPython is not installed for Python 2.6")
print("The 'dist' command will not work without it. Get it using pip or easy_install")
print("or see: http://packages.python.org/GitPython/0.3.1/intro.html#getting-started")
print("+----------------------------------------------------------------------------+")
'''
if options.verbose:
print(". Removing distribution directory, '{0}/'.".format(DIST_DIR))
shutil.rmtree(DIST_DIR, ignore_errors=True)
if not os.path.exists(DIST_DIR): os.mkdir(DIST_DIR)
if options.uncompressed:
make_skulpt_js(options,DIST_DIR)
# Make the compressed distribution.
compfn = os.path.join(DIST_DIR, OUTFILE_MIN)
builtinfn = os.path.join(DIST_DIR, OUTFILE_LIB)
debuggerfn = os.path.join(DIST_DIR, OUTFILE_DEBUGGER)
# Run tests on uncompressed.
if options.verbose:
print(". Running tests on uncompressed...")
ret = 0 #test()
if ret != 0:
print("Tests failed on uncompressed version.")
#sys.exit(1);
# compress
uncompfiles = ' '.join(['--js ' + x for x in getFileList(FILE_TYPE_DIST, include_ext_libs=False)])
if options.verbose:
print(". Compressing...")
ret = os.system("java -jar support/closure-compiler/compiler.jar --define goog.DEBUG=false --output_wrapper \"(function(){%%output%%}());\" --compilation_level SIMPLE_OPTIMIZATIONS --jscomp_error accessControls --jscomp_error checkRegExp --jscomp_error checkTypes --jscomp_error checkVars --jscomp_error deprecated --jscomp_off fileoverviewTags --jscomp_error invalidCasts --jscomp_error missingProperties --jscomp_error nonStandardJsDocs --jscomp_error strictModuleDepCheck --jscomp_error undefinedVars --jscomp_error unknownDefines --jscomp_error visibility %s --externs support/es6-promise-polyfill/externs.js --js_output_file tmp.js" % (uncompfiles))
# to disable asserts
# --define goog.DEBUG=false
#
# to make a file that for ff plugin, not sure of format
# --create_source_map /srcmap.txt
#
# --jscomp_error accessControls --jscomp_error checkRegExp --jscomp_error checkTypes --jscomp_error checkVars --jscomp_error deprecated --jscomp_error fileoverviewTags --jscomp_error invalidCasts --jscomp_error missingProperties --jscomp_error nonStandardJsDocs --jscomp_error strictModuleDepCheck --jscomp_error undefinedVars --jscomp_error unknownDefines --jscomp_error visibility
#
if ret != 0:
print("closure-compiler failed.")
sys.exit(1)
# Copy the debugger file to the output dir
if options.verbose:
print(". Bundling external libraries...")
bundle = ""
for fn in ExtLibs + ["tmp.js"]:
with open(fn, "r") as f:
bundle += f.read()
with open(compfn, "w") as f:
f.write(bundle)
print(". Wrote bundled file")
# Run tests on compressed.
if options.verbose:
print(". Running tests on compressed...")
buildNamedTestsFile()
ret = 0 #os.system("{0} {1} {2}".format(jsengine, compfn, ' '.join(TestFiles)))
if ret != 0:
print("Tests failed on compressed version.")
sys.exit(1)
ret = 0 #rununits(opt=True)
if ret != 0:
print("Tests failed on compressed unit tests")
sys.exit(1)
#doc()
try:
shutil.copy(compfn, os.path.join(DIST_DIR, "tmp.js"))
shutil.copy("debugger/debugger.js", DIST_DIR)
except Exception as e:
print("Couldn't copy debugger to output folder: %s" % e.message)
sys.exit(1)
path_list = os.environ.get('PATH','').split(':')
has_gzip = False
for p in path_list:
has_gzip = os.access(os.path.join(p,"gzip"), os.X_OK)
if has_gzip:
break
if has_gzip:
ret = os.system("gzip -9 {0}/tmp.js".format(DIST_DIR))
if ret != 0:
print("Couldn't gzip to get final size.")
has_gzip = False
os.unlink("{0}/tmp.js".format(DIST_DIR))
size = os.path.getsize("{0}/tmp.js.gz".format(DIST_DIR))
os.unlink("{0}/tmp.js.gz".format(DIST_DIR))
else:
os.unlink("{0}/tmp.js".format(DIST_DIR))
print("No gzip executable, can't get final size")
with open(builtinfn, "w") as f:
f.write(getBuiltinsAsJson(options))
if options.verbose:
print(". Wrote {0}".format(builtinfn))
# Update documentation folder copies of the distribution.
try:
shutil.copy(compfn, os.path.join("doc", "static", OUTFILE_MIN))
shutil.copy(builtinfn, os.path.join("doc", "static", OUTFILE_LIB))
shutil.copy(debuggerfn, os.path.join("doc", "static", "debugger", OUTFILE_DEBUGGER))
except:
print("Couldn't copy to docs dir.")
sys.exit(1)
if options.verbose:
print(". Updated doc dir")
# All good!
if options.verbose:
print(". Wrote {0}.".format(compfn))
if has_gzip:
print(". gzip of compressed: %d bytes" % size)
def make_skulpt_js(options,dest):
if options.verbose:
print(". Writing combined version...")
combined = ''
linemap = open(os.path.join(dest, OUTFILE_MAP), "w")
curline = 1
for file in getFileList(FILE_TYPE_DIST):
curfiledata = open(file).read()
combined += curfiledata
print("%d:%s" % (curline, file), file=linemap)
curline += len(curfiledata.split("\n")) - 1
linemap.close()
uncompfn = os.path.join(dest, OUTFILE_REG)
open(uncompfn, "w").write(combined)
# Prevent accidental editing of the uncompressed distribution file.
if sys.platform != "win32":
os.chmod(os.path.join(dest, OUTFILE_REG), 0o444)
def run_in_browser(fn, options):
shutil.rmtree(RUN_DIR, ignore_errors=True)
if not os.path.exists(RUN_DIR): os.mkdir(RUN_DIR)
docbi(options,RUN_DIR)
scripts = []
for f in getFileList(FILE_TYPE_TEST):
scripts.append('' %
os.path.join('../..', f))
scripts = "\n".join(scripts)
with open (fn,'r') as runfile:
prog = runfile.read()
with open('support/run_template.html') as tpfile:
page = tpfile.read()
page = page % dict(code=prog,scripts=scripts, root='', debug_mode='true')
with open("{0}/run.html".format(RUN_DIR),"w") as htmlfile:
htmlfile.write(page)
if sys.platform == "darwin":
os.system("open {0}/run.html".format(RUN_DIR))
elif sys.platform == "linux2":
os.system("xdg-open {0}/run.html".format(RUN_DIR))
elif sys.platform == "win32":
os.system("start {0}/run.html".format(RUN_DIR))
else:
print("open or refresh {0}/run.html in your browser to test/debug".format(RUN_DIR))
def regenparser():
"""regenerate the parser/ast source code"""
if not os.path.exists("gen"): os.mkdir("gen")
os.chdir("src/pgen/parser")
os.system("python main.py ../../../gen/parse_tables.js")
os.chdir("../ast")
os.system("python asdl_js.py Python.asdl ../../../gen/astnodes.js")
os.chdir("../../..")
# sanity check that they at least parse
#os.system(jsengine + " support/closure-library/closure/goog/base.js src/env.js src/tokenize.js gen/parse_tables.js gen/astnodes.js")
def regenasttests(togen="{0}/run/*.py".format(TEST_DIR)):
"""regenerate the ast test files by running our helper script via real python"""
for f in glob.glob(togen):
transname = f.replace(".py", ".trans")
os.system("python {0}/astppdump.py {1} > {2}".format(TEST_DIR, f, transname))
forcename = f.replace(".py", ".trans.force")
if os.path.exists(forcename):
shutil.copy(forcename, transname)
if crlfprog:
os.system("python {0} {1}".format(crlfprog, transname))
def regenruntests(togen="{0}/run/*.py".format(TEST_DIR)):
"""regenerate the test data by running the tests on real python"""
for f in glob.glob(togen):
os.system("python {0} > {1}.real 2>&1".format(f, f))
forcename = f + ".real.force"
if os.path.exists(forcename):
shutil.copy(forcename, "%s.real" % f)
if crlfprog:
os.system("python %s %s.real" % (crlfprog, f))
for f in glob.glob("{0}/interactive/*.py".format(TEST_DIR)):
p = Popen("python -i > %s.real 2>%s" % (f, nul), shell=True, stdin=PIPE)
p.communicate(open(f).read() + "\004")
forcename = f + ".real.force"
if os.path.exists(forcename):
shutil.copy(forcename, "%s.real" % f)
if crlfprog:
os.system("python %s %s.real" % (crlfprog, f))
def doc():
print("Building Documentation in docs/ProgMan")
ret = os.system("jsdoc -c jsdoc.json HACKING.md")
if ret != 0:
print("Build of docs failed. Is jsdoc installed?")
def symtabdump(fn):
if not os.path.exists(fn):
print("%s doesn't exist" % fn)
raise SystemExit()
text = open(fn).read()
mod = symtable.symtable(text, os.path.split(fn)[1], "exec")
def getidents(obj, indent=""):
ret = ""
ret += """%sSym_type: %s
%sSym_name: %s
%sSym_lineno: %s
%sSym_nested: %s
%sSym_haschildren: %s
""" % (
indent, obj.get_type(),
indent, obj.get_name(),
indent, obj.get_lineno(),
indent, obj.is_nested(),
indent, obj.has_children())
if obj.get_type() == "function":
ret += "%sFunc_params: %s\n%sFunc_locals: %s\n%sFunc_globals: %s\n%sFunc_frees: %s\n" % (
indent, sorted(obj.get_parameters()),
indent, sorted(obj.get_locals()),
indent, sorted(obj.get_globals()),
indent, sorted(obj.get_frees()))
elif obj.get_type() == "class":
ret += "%sClass_methods: %s\n" % (
indent, sorted(obj.get_methods()))
ret += "%s-- Identifiers --\n" % indent
for ident in sorted(obj.get_identifiers()):
info = obj.lookup(ident)
ret += "%sname: %s\n %sis_referenced: %s\n %sis_imported: %s\n %sis_parameter: %s\n %sis_global: %s\n %sis_declared_global: %s\n %sis_local: %s\n %sis_free: %s\n %sis_assigned: %s\n %sis_namespace: %s\n %snamespaces: [\n%s %s]\n" % (
indent, info.get_name(),
indent, info.is_referenced(),
indent, info.is_imported(),
indent, info.is_parameter(),
indent, info.is_global(),
indent, info.is_declared_global(),
indent, info.is_local(),
indent, info.is_free(),
indent, info.is_assigned(),
indent, info.is_namespace(),
indent, '\n'.join([getidents(x, indent + " ") for x in info.get_namespaces()]),
indent
)
return ret
return getidents(mod)
def regensymtabtests(togen="{0}/run/*.py".format(TEST_DIR)):
"""regenerate the test data by running the symtab dump via real python"""
for fn in glob.glob(togen):
outfn = "%s.symtab" % fn
f = open(outfn, "wb")
f.write(symtabdump(fn))
f.close()
def upload():
"""uploads doc to GAE (stub app for static hosting, mostly)"""
ret = os.system("python2.6 ~/Desktop/3rdparty/google_appengine/appcfg.py update doc")
if ret != 0:
print("Couldn't upload.")
raise SystemExit()
def doctest():
ret = os.system("python2.6 ~/Desktop/3rdparty/google_appengine/dev_appserver.py -p 20710 doc")
def docbi(options,dest="doc/static"):
builtinfn = "{0}/{1}".format(dest,OUTFILE_LIB)
with open(builtinfn, "w") as f:
f.write(getBuiltinsAsJson(options))
if options.verbose:
print(". Wrote {fileName}".format(fileName=builtinfn))
def assess(student_code, instructor_code):
student_code = student_code.replace("\\", "/")
instructor_code = instructor_code.replace("\\", "/")
if not os.path.exists(student_code):
print("%s doesn't exist" % student_code)
raise SystemExit()
if not os.path.exists(instructor_code):
print("%s doesn't exist" % instructor_code)
raise SystemExit()
if not os.path.exists("support/tmp"):
os.mkdir("support/tmp")
student_module_name = os.path.splitext(os.path.basename(student_code))[0]
instructor_module_name = os.path.splitext(os.path.basename(instructor_code))[0]
f = open("support/tmp/run.js", "w")
f.write("""
Sk.console = [];
Sk.skip_drawing = true;
var printError = function(error) {{
if (error.constructor == Sk.builtin.NameError
&& error.args.v.length > 0
&& error.args.v[0].v == "name '___' is not defined") {{
print("EXCEPTION: "+error.tp$name);
//print("EXCEPTION: DanglingBlocksError");
}} else {{
print("EXCEPTION: "+error.tp$name);
}}
}}
var student_code = read('{student_code_filename}');
var instructor_code = read('{instructor_code_filename}');
var outputList = [];
Sk.configure({{read:read, python3:true, debugging:false, output: function(text) {{ if (text !== "\\n") {{ outputList.push(text); }} }} }});
Sk.console.printHtml = function(chart, lines) {{
outputList.push(lines);
}};
// Run students' code
Sk.misceval.asyncToPromise(function() {{
return Sk.importMainWithBody("", false, student_code, true);
}}).then(function (data) {{
// Trace table
var traceTable = []; //JSON.stringify(data.$d);
// Run instructor's code
/*Sk.configure({{read:read, python3:true, debugging:false, output: function(text) {{ }} }});
instructor_code += "\\nresult = on_run('''"+student_code+"''', "+
JSON.stringify(outputList)+", "+
JSON.stringify(traceTable)+")";
Sk.misceval.asyncToPromise(function() {{
return Sk.importMainWithBody("", false, instructor_code, true);
}}).then(function (data) {{
var result = data.$d.result.v;
print(result);
}}, function(e) {{
//printError(e);
print("UNCAUGHT EXCEPTION: " + e);
}});*/
}}, function(e) {{
//printError(e);
print(e);
}});""".format(student_code_filename=student_code,
instructor_code_filename=instructor_code))
f.close()
command = jsengine.split(" ")+getFileList(FILE_TYPE_TEST)+["../libs/math.0.19.0.min.js", "../libs/crime_data.js", "support/tmp/run.js"]
try:
p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
out, err = p.communicate()
print(out)
except OSError as e:
print("Execution failed:", e, file=sys.stderr)
def run(fn, shell="", opt=False, p3=True, debug_mode=False, dumpJS='true'):
if not os.path.exists(fn):
print("%s doesn't exist" % fn)
raise SystemExit()
if not os.path.exists("support/tmp"):
os.mkdir("support/tmp")
f = open("support/tmp/run.js", "w")
modname = os.path.splitext(os.path.basename(fn))[0]
if p3:
p3on = 'true'
else:
p3on = 'false'
if debug_mode:
debugon = 'true'
else:
debugon = 'false'
f.write("""
var input = read('%s');
var outputList = [];
//print("-----");
//print(input);
//print("-----");
Sk.configure({syspath:["%s"], read:read, python3:%s, debugging:%s
, output: function(text) {if (text !== "\\n") {outputList.push(text); print(text); }}
});
Sk.misceval.asyncToPromise(function() {
return Sk.importMain("%s", %s, true);
}).then(function (data) {
// Printed
// outputList
// Properties
// JSON.stringify(data.$d);
// Source code
// input
print("-----");
//print(outputList)
}, function(e) {
print("UNCAUGHT EXCEPTION: " + e);
print(e.stack);
});
""" % (fn, os.path.split(fn)[0], p3on, debugon, modname, dumpJS))
f.close()
if opt:
os.system("{0} {1}/{2} support/tmp/run.js".format(jsengine, DIST_DIR, OUTFILE_MIN))
else:
os.system("{0} {1} {2} support/tmp/run.js".format(jsengine, shell, ' '.join(getFileList(FILE_TYPE_TEST))))
def runopt(fn):
run(fn, "", True)
def run3(fn):
run(fn,p3=True)
def rundebug(fn):
run(fn,debug_mode=True)
def shell(fn):
run(fn, "--shell")
def rununits(opt=False, p3=False):
testFiles = ['test/unit/'+f for f in os.listdir('test/unit') if '.py' in f]
jstestengine = jsengine.replace('--debugger', '')
passTot = 0
failTot = 0
for fn in testFiles:
if not os.path.exists("support/tmp"):
os.mkdir("support/tmp")
f = open("support/tmp/run.js", "w")
modname = os.path.splitext(os.path.basename(fn))[0]
if p3:
p3on = 'true'
else:
p3on = 'false'
f.write("""
var input = read('%s');
print('%s');
Sk.configure({syspath:["%s"], read:read, python3:%s});
Sk.misceval.asyncToPromise(function() {
return Sk.importMain("%s", false, true);
}).then(function () {}, function(e) {
print("UNCAUGHT EXCEPTION: " + e);
print(e.stack);
quit(1);
});
""" % (fn, fn, os.path.split(fn)[0], p3on, modname))
f.close()
if opt:
p = Popen("{0} {1}/{2} support/tmp/run.js".format(jstestengine, DIST_DIR,
OUTFILE_MIN),shell=True,
stdout=PIPE, stderr=PIPE)
else:
p = Popen("{0} {1} support/tmp/run.js".format(jstestengine, ' '.join(
getFileList(FILE_TYPE_TEST))), shell=True, stdout=PIPE, stderr=PIPE)
outs, errs = p.communicate()
if p.returncode != 0:
failTot += 1
print("{} exited with error code {}".format(fn,p.returncode))
print(outs)
if errs:
print(errs)
outlines = outs.split('\n')
for ol in outlines:
g = re.match(r'Ran.*passed:\s+(\d+)\s+failed:\s+(\d+)',ol)
if g:
passTot += int(g.group(1))
failTot += int(g.group(2))
print("Summary")
print("Passed: %5d Failed %5d" % (passTot, failTot))
if failTot != 0:
return -1
else:
return 0
def repl():
os.system("{0} {1} repl/repl.js".format(jsengine, ' '.join(getFileList(FILE_TYPE_TEST))))
def nrt(newTest):
"""open a new run test"""
fn = "{0}/run/test_{1}.py".format(TEST_DIR,newTest)
disfn = fn + ".disabled"
if not os.path.exists(fn) and not os.path.exists(disfn):
if 'EDITOR' in os.environ:
editor = os.environ['EDITOR']
else:
editor = 'vim'
os.system(editor + ' ' + fn)
if os.path.exists(fn):
print("Generating tests for %s" % fn)
regensymtabtests(fn)
regenasttests(fn)
regenruntests(fn)
else:
print("Test test_%s.py already exists." % newTest)
print("run ./m regentests test_%s.py" % newTest)
def vmwareregr(names):
"""todo; not working yet.
run unit tests via vmware on a bunch of browsers"""
xp = "/data/VMs/xpsp3/xpsp3.vmx"
ubu = "/data/VMs/ubu910/ubu910.vmx"
# apparently osx isn't very vmware-able. stupid.
class Browser:
def __init__(self, name, vmx, guestloc):
self.name = name
self.vmx = vmx
self.guestloc = guestloc
browsers = [
Browser("ie7-win", xp, "C:\\Program Files\\Internet Explorer\\iexplore.exe"),
Browser("ie8-win", xp, "C:\\Program Files\\Internet Explorer\\iexplore.exe"),
Browser("chrome3-win", xp, "C:\\Documents and Settings\\Administrator\\Local Settings\\Application Data\\Google\\Chrome\\Application\\chrome.exe"),
Browser("chrome4-win", xp, "C:\\Documents and Settings\\Administrator\\Local Settings\\Application Data\\Google\\Chrome\\Application\\chrome.exe"),
Browser("ff3-win", xp, "C:\\Program Files\\Mozilla Firefox\\firefox.exe"),
Browser("ff35-win", xp, "C:\\Program Files\\Mozilla Firefox\\firefox.exe"),
#Browser("safari3-win", xp,
#Browser("safari4-win", xp,
#"ff3-osx": osx,
#"ff35-osx": osx,
#"safari3-osx": osx,
#"safari4-osx": osx,
#"ff3-ubu": ubu,
#"chromed-ubu": ubu,
]
def regengooglocs():
"""scans the closure library and builds an import-everything file to be
used during dev. """
# from calcdeps.py
prov_regex = re.compile('goog\.provide\s*\(\s*[\'\"]([^\)]+)[\'\"]\s*\)')
# walk whole tree, find all the 'provide's in a file, and note the location
root = "support/closure-library/closure"
modToFile = {}
for dirpath, dirnames, filenames in os.walk(root):
for filename in filenames:
f = os.path.join(dirpath, filename)
if ".svn" in f: continue
if os.path.splitext(f)[1] == ".js":
contents = open(f).read()
for prov in prov_regex.findall(contents):
modToFile[prov] = f.lstrip(root)
with open("gen/debug_import_all_closure.js", "w") as glf:
keys = modToFile.keys()
keys.sort()
for m in keys:
if "demos." in m: continue
if not m.startswith("goog."): continue
print("goog.require('%s');" % m, file=glf)
import http.server
from urllib.parse import urlparse
class HttpHandler(http.server.SimpleHTTPRequestHandler):
"""allow grabbing any file for testing, and support /import
which grabs all builtin and lib modules in a json request.
see notes on import for why we can't just grab one at a time.
on real hosting, we'll just prebuild/gzip the stdlib into somewhere on
upload. this is more convenient during dev on localhost though.
"""
def do_GET(self):
prefix = "/import"
if self.path == prefix:
self.send_response(200)
self.send_header("Content-type", "application/json")
self.end_headers()
self.wfile.write(getBuiltinsAsJson(None))
else:
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
def host(PORT = 20710):
"""simple http host from root of dir for testing"""
import SocketServer
httpd = SocketServer.TCPServer(("", PORT), HttpHandler)
print("serving at port", PORT)
httpd.serve_forever()
def usageString(program):
return '''
{program} [] [script.py]
Commands:
run Run a Python file using Skulpt
brun Run a Python file using Skulpt but in your browser
test Run all test cases
rununits Run only the new-style unit tests
dist Build core and library distribution files
docbi Build library distribution file only and copy to doc/static
profile [fn] [out] Profile Skulpt using d8 and show processed results
time [iter] Average runtime of the test suite over [iter] iterations.
regenparser Regenerate parser tests
regenasttests Regen abstract symbol table tests
regenruntests Regenerate runtime unit tests
regensymtabtests Regenerate symbol table tests
regentests Regenerate all of the above
help Display help information about Skulpt
host [PORT] Start a simple HTTP server for testing. Default port: 20710
upload Run appcfg.py to upload doc to live GAE site
doctest Run the GAE development server for doc testing
nrt Generate a file for a new test case
runopt Run a Python file optimized
browser Run all tests in the browser
shell Run a Python program but keep a shell open (like python -i)
vfs Build a virtual file system to support Skulpt read tests
debugbrowser Debug in the browser -- open your javascript console
Options:
-q, --quiet Only output important information
-s, --silent Do not output anything, besides errors
-u, --uncompressed Makes uncompressed core distribution file for debugging
-v, --verbose Make output more verbose [default]
--version Returns the version string in Bower configuration file.
'''.format(program=program)
def main():
parser = OptionParser(usageString("%prog"), version="%prog {0}".format(bowerProperty("version")))
parser.add_option("-q", "--quiet", action="store_false", dest="verbose")
parser.add_option("-s", "--silent", action="store_true", dest="silent", default=False)
parser.add_option("-u", "--uncompressed", action="store_true", dest="uncompressed", default=False)
parser.add_option("-v", "--verbose",
action="store_true",
dest="verbose",
default=False,
help="Make output more verbose [default]")
(options, args) = parser.parse_args()
# This is rather aggressive. Do we really want it?
if options.verbose:
if sys.platform == 'win32':
os.system("cls")
else:
os.system("clear")
if len(sys.argv) < 2:
cmd = "help"
else:
cmd = sys.argv[1]
with open("src/internalpython.js", "w") as f:
f.write(getInternalCodeAsJson() + ";")
if cmd == "test":
test()
elif cmd == "testdebug":
test(True)
elif cmd == "dist":
dist(options)
elif cmd == "regengooglocs":
regengooglocs()
elif cmd == "regentests":
if len(sys.argv) > 2:
togen = "{0}/run/".format(TEST_DIR) + sys.argv[2]
else:
togen = "{0}/run/*.py".format(TEST_DIR)
print("generating tests for ", togen)
regensymtabtests(togen)
regenasttests(togen)
regenruntests(togen)
elif cmd == "regensymtabtests":
regensymtabtests()
elif cmd == "run":
run(sys.argv[2])
elif cmd == "assess":
assess(sys.argv[2], sys.argv[3])
elif cmd == "brun":
run_in_browser(sys.argv[2],options)
elif cmd == 'rununits':
rununits()
elif cmd == "runopt":
runopt(sys.argv[2])
elif cmd == "run3":
run3(sys.argv[2])
elif cmd == "rundebug":
rundebug(sys.argv[2])
elif cmd == "vmwareregr":
vmwareregr()
elif cmd == "regenparser":
regenparser()
elif cmd == "regenasttests":
regenasttests()
elif cmd == "regenruntests":
regenruntests()
elif cmd == "upload":
upload()
elif cmd == "doctest":
doctest()
elif cmd == "docbi":
docbi(options)
elif cmd == "doc":
doc()
elif cmd == "nrt":
print("Warning: nrt is deprectated.")
print("It is preferred that you enhance one of the unit tests in test/unit")
print("Or, create a new unit test file in test/unit using the template in test/unit_tmpl.py")
if len(sys.argv) < 3:
print("Need a name for the new test")
print(usageString(os.path.basename(sys.argv[0])))
sys.exit(2)
nrt(sys.argv[2])
elif cmd == "browser":
buildBrowserTests()
elif cmd == "debugbrowser":
debugbrowser()
elif cmd == "vfs":
buildVFS()
elif cmd == "host":
if len(sys.argv) < 3:
host()
else:
try:
host(int(sys.argv[2]))
except ValueError:
print("Port must be an integer")
sys.exit(2)
elif cmd == "shell":
shell(sys.argv[2]);
elif cmd == "repl":
repl()
elif cmd == "profile":
parse_profile_args(sys.argv)
elif cmd == "time":
parse_time_args(sys.argv)
else:
print(usageString(os.path.basename(sys.argv[0])))
sys.exit(2)
if __name__ == "__main__":
main()