import * as child_process from 'child_process';
import * as fs from 'fs';
import * as zlib from 'zlib';
import * as url from 'url';
import * as http from 'http';
import * as https from 'https';
import * as events from 'events';
import * as crypto from 'crypto';
import * as path from 'path';
import * as os from 'os';
Promise = require("bluebird");
import Util = pxt.Util;
export interface SpawnOptions {
cmd: string;
args: string[];
cwd?: string;
shell?: boolean;
pipe?: boolean;
input?: string;
silent?: boolean;
envOverrides?: pxt.Map;
allowNonZeroExit?: boolean;
}
//This should be correct at startup when running from command line
export let targetDir: string = process.cwd();
export let pxtCoreDir: string = path.join(__dirname, "..");
export let cliFinalizers: (() => Promise)[] = [];
export function addCliFinalizer(f: () => Promise) {
cliFinalizers.push(f)
}
export function runCliFinalizersAsync() {
let fins = cliFinalizers
cliFinalizers = []
return Promise.mapSeries(fins, f => f())
.then(() => { })
}
export function setTargetDir(dir: string) {
targetDir = dir;
(module).paths.push(path.join(targetDir, "node_modules"));
}
export function readResAsync(g: events.EventEmitter) {
return new Promise((resolve, reject) => {
let bufs: Buffer[] = []
g.on('data', (c: any) => {
if (typeof c === "string")
bufs.push(Buffer.from(c, "utf8"))
else
bufs.push(c)
});
g.on("error", (err: any) => reject(err))
g.on('end', () => resolve(Buffer.concat(bufs)))
})
}
export function spawnAsync(opts: SpawnOptions) {
opts.pipe = false
return spawnWithPipeAsync(opts)
.then(() => { })
}
export function spawnWithPipeAsync(opts: SpawnOptions) {
if (opts.pipe === undefined) opts.pipe = true
let info = opts.cmd + " " + opts.args.join(" ")
if (opts.cwd && opts.cwd != ".") info = "cd " + opts.cwd + "; " + info
console.log("[run] " + info)
return new Promise((resolve, reject) => {
let ch = child_process.spawn(opts.cmd, opts.args, {
cwd: opts.cwd,
env: opts.envOverrides ? extendEnv(process.env, opts.envOverrides) : process.env,
stdio: opts.pipe ? [opts.input == null ? process.stdin : "pipe", "pipe", process.stderr] : "inherit",
shell: opts.shell || false
} as any)
let bufs: Buffer[] = []
if (opts.pipe)
ch.stdout.on('data', (buf: Buffer) => {
bufs.push(buf)
if (!opts.silent) {
process.stdout.write(buf)
}
})
ch.on('close', (code: number) => {
if (code != 0 && !opts.allowNonZeroExit)
reject(new Error("Exit code: " + code + " from " + info))
resolve(Buffer.concat(bufs))
});
if (opts.input != null)
ch.stdin.end(opts.input, "utf8")
})
}
function extendEnv(base: any, overrides: any) {
let res: any = {};
Object.keys(base).forEach(key => res[key] = base[key])
Object.keys(overrides).forEach(key => res[key] = overrides[key])
return res;
}
export function addCmd(name: string) {
return name + (/^win/.test(process.platform) ? ".cmd" : "")
}
export function runNpmAsync(...args: string[]) {
return runNpmAsyncWithCwd(".", ...args);
}
export interface NpmRegistry {
_id: string;
_name: string;
"dist-tags": pxt.Map;
"versions": pxt.Map;
}
export function npmRegistryAsync(pkg: string): Promise {
// TODO: use token if available
return Util.httpGetJsonAsync(`https://registry.npmjs.org/${pkg}`);
}
export function runNpmAsyncWithCwd(cwd: string, ...args: string[]) {
return spawnAsync({
cmd: addCmd("npm"),
args: args,
cwd
});
}
export function runGitAsync(...args: string[]) {
return spawnAsync({
cmd: "git",
args: args,
cwd: "."
})
}
export function gitInfoAsync(args: string[], cwd?: string, silent: boolean = false) {
return Promise.resolve()
.then(() => spawnWithPipeAsync({
cmd: "git",
args: args,
cwd,
silent
}))
.then(buf => buf.toString("utf8").trim())
}
export function currGitTagAsync() {
return gitInfoAsync(["describe", "--tags", "--exact-match"])
.then(t => {
if (!t)
Util.userError("no git tag found")
return t
})
}
export function needsGitCleanAsync() {
return Promise.resolve()
.then(() => spawnWithPipeAsync({
cmd: "git",
args: ["status", "--porcelain", "--untracked-files=no"]
}))
.then(buf => {
if (buf.length)
Util.userError("Please commit all files to git before running 'pxt bump'")
})
}
function nodeHttpRequestAsync(options: Util.HttpRequestOptions): Promise {
let isHttps = false
let u = url.parse(options.url)
if (u.protocol == "https:") isHttps = true
/* tslint:disable:no-http-string */
else if (u.protocol == "http:") isHttps = false
/* tslint:enable:no-http-string */
else return Promise.reject("bad protocol: " + u.protocol)
u.headers = Util.clone(options.headers) || {}
let data = options.data
u.method = options.method || (data == null ? "GET" : "POST");
let buf: Buffer = null;
u.headers["accept-encoding"] = "gzip"
u.headers["user-agent"] = "PXT-CLI"
let gzipContent = false
if (data != null) {
if (Buffer.isBuffer(data)) {
buf = data;
} else if (typeof data == "object") {
buf = Buffer.from(JSON.stringify(data), "utf8")
u.headers["content-type"] = "application/json; charset=utf8"
if (options.allowGzipPost) gzipContent = true
} else if (typeof data == "string") {
buf = Buffer.from(data, "utf8")
if (options.allowGzipPost) gzipContent = true
} else {
Util.oops("bad data")
}
}
if (gzipContent) {
buf = zlib.gzipSync(buf)
u.headers['content-encoding'] = "gzip"
}
if (buf)
u.headers['content-length'] = buf.length
return new Promise((resolve, reject) => {
const handleResponse = (res: http.IncomingMessage) => {
let g: events.EventEmitter = res;
if (/gzip/.test(res.headers['content-encoding'])) {
let tmp = zlib.createUnzip();
res.pipe(tmp);
g = tmp;
}
resolve(readResAsync(g).then(buf => {
let text: string = null
try {
text = buf.toString("utf8")
} catch (e) {
}
let resp: Util.HttpResponse = {
statusCode: res.statusCode,
headers: res.headers,
buffer: buf,
text: text
}
return resp;
}))
};
const req = isHttps ? https.request(u, handleResponse) : http.request(u, handleResponse);
req.on('error', (err: any) => reject(err))
req.end(buf)
})
}
function sha256(hashData: string): string {
let sha: string;
let hash = crypto.createHash("sha256");
hash.update(hashData, "utf8");
sha = hash.digest().toString("hex").toLowerCase();
return sha;
}
function init() {
// no, please, I want to handle my errors myself
let async = (Promise)._async
async.fatalError = (e: any) => async.throwLater(e);
Util.isNodeJS = true;
Util.httpRequestCoreAsync = nodeHttpRequestAsync;
Util.sha256 = sha256;
Util.cpuUs = () => {
const p = process.cpuUsage()
return p.system + p.user
}
Util.getRandomBuf = buf => {
let tmp = crypto.randomBytes(buf.length)
for (let i = 0; i < buf.length; ++i)
buf[i] = tmp[i]
}
(global as any).btoa = (str: string) => Buffer.from(str, "binary").toString("base64");
(global as any).atob = (str: string) => Buffer.from(str, "base64").toString("binary");
}
export function sanitizePath(path: string) {
return path.replace(/[^\w@\/]/g, "-").replace(/^\/+/, "")
}
export function readJson(fn: string) {
return JSON.parse(fs.readFileSync(fn, "utf8"))
}
export function readPkgConfig(dir: string) {
//pxt.debug("readPkgConfig in " + dir)
const fn = path.join(dir, pxt.CONFIG_NAME)
const js: pxt.PackageConfig = readJson(fn)
const ap = js.additionalFilePath
if (ap) {
let adddir = path.join(dir, ap);
if (!existsDirSync(adddir))
pxt.U.userError(`additional pxt.json not found: ${adddir} in ${dir} + ${ap}`)
pxt.debug("additional pxt.json: " + adddir)
const js2 = readPkgConfig(adddir)
for (let k of Object.keys(js2)) {
if (!js.hasOwnProperty(k)) {
(js as any)[k] = (js2 as any)[k]
}
}
js.additionalFilePaths = [ap].concat(js2.additionalFilePaths.map(d => path.join(ap, d)))
} else {
js.additionalFilePaths = []
}
// don't inject version number
// as they get serialized later on
// if (!js.targetVersions) js.targetVersions = pxt.appTarget.versions;
return js
}
export function getPxtTarget(): pxt.TargetBundle {
if (fs.existsSync(targetDir + "/built/target.json")) {
let res: pxt.TargetBundle = readJson(targetDir + "/built/target.json")
if (res.id && res.bundledpkgs) return res;
}
let raw: pxt.TargetBundle = readJson(targetDir + "/pxtarget.json")
raw.bundledpkgs = {}
return raw
}
export function pathToPtr(path: string) {
return "ptr-" + sanitizePath(path.replace(/^ptr-/, "")).replace(/[^\w@]/g, "-")
}
export function mkdirP(thePath: string) {
if (thePath == "." || !thePath) return;
if (!fs.existsSync(thePath)) {
mkdirP(path.dirname(thePath))
fs.mkdirSync(thePath)
}
}
export function cpR(src: string, dst: string, maxDepth = 8) {
src = path.resolve(src)
let files = allFiles(src, maxDepth)
let dirs: pxt.Map = {}
for (let f of files) {
let bn = f.slice(src.length)
let dd = path.join(dst, bn)
let dir = path.dirname(dd)
if (!Util.lookup(dirs, dir)) {
mkdirP(dir)
dirs[dir] = true
}
let buf = fs.readFileSync(f)
fs.writeFileSync(dd, buf)
}
}
export function cp(srcFile: string, destDirectory: string) {
mkdirP(destDirectory);
let dest = path.resolve(destDirectory, path.basename(srcFile));
let buf = fs.readFileSync(path.resolve(srcFile));
fs.writeFileSync(dest, buf);
}
export function allFiles(top: string, maxDepth = 8, allowMissing = false, includeDirs = false, ignoredFileMarker: string = undefined): string[] {
let res: string[] = []
if (allowMissing && !existsDirSync(top)) return res
for (const p of fs.readdirSync(top)) {
if (p[0] == ".") continue;
const inner = path.join(top, p)
const st = fs.statSync(inner)
if (st.isDirectory()) {
// check for ingored folder marker
if (ignoredFileMarker && fs.existsSync(path.join(inner, ignoredFileMarker)))
continue;
if (maxDepth > 1)
Util.pushRange(res, allFiles(inner, maxDepth - 1))
if (includeDirs)
res.push(inner);
} else {
res.push(inner)
}
}
return res
}
export function existsDirSync(name: string): boolean {
try {
const stats = fs.lstatSync(name);
return stats && stats.isDirectory();
}
catch (e) {
return false;
}
}
export function writeFileSync(p: string, data: any, options?: { encoding?: string | null; mode?: number | string; flag?: string; } | string | null) {
mkdirP(path.dirname(p));
fs.writeFileSync(p, data, options);
if (pxt.options.debug) {
const stats = fs.statSync(p);
pxt.log(` + ${p} ${stats.size > 1000000 ? (stats.size / 1000000).toFixed(2) + ' m' : stats.size > 1000 ? (stats.size / 1000).toFixed(2) + 'k' : stats.size}b`)
}
}
export function openUrl(startUrl: string, browser: string) {
if (!/^[a-z0-9A-Z#=\.\-\\\/%:\?_&]+$/.test(startUrl)) {
console.error("invalid URL to open: " + startUrl)
return
}
let cmds: pxt.Map = {
darwin: "open",
win32: "start",
linux: "xdg-open"
}
if (/^win/.test(os.platform()) && !/^[a-z0-9]+:\/\//i.test(startUrl))
startUrl = startUrl.replace('/', '\\');
else
startUrl = startUrl.replace('\\', '/');
console.log(`opening ${startUrl}`)
if (browser) {
child_process.spawn(getBrowserLocation(browser), [startUrl], { detached: true });
}
else {
child_process.exec(`${cmds[process.platform]} ${startUrl}`);
}
}
function getBrowserLocation(browser: string) {
let browserPath: string;
const normalizedBrowser = browser.toLowerCase();
if (normalizedBrowser === "chrome") {
switch (os.platform()) {
case "win32":
browserPath = "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe";
break;
case "darwin":
browserPath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
break;
case "linux":
browserPath = "/opt/google/chrome/chrome";
break;
default:
break;
}
}
else if (normalizedBrowser === "firefox") {
browserPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe";
switch (os.platform()) {
case "win32":
browserPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe";
break;
case "darwin":
browserPath = "/Applications/Firefox.app";
break;
case "linux":
default:
break;
}
}
else if (normalizedBrowser === "ie") {
browserPath = "C:/Program Files/Internet Explorer/iexplore.exe";
}
else if (normalizedBrowser === "safari") {
browserPath = "/Applications/Safari.app/Contents/MacOS/Safari";
}
if (browserPath && fs.existsSync(browserPath)) {
return browserPath;
}
return browser;
}
export function fileExistsSync(p: string): boolean {
try {
let stats = fs.lstatSync(p);
return stats && stats.isFile();
}
catch (e) {
return false;
}
}
export let lastResolveMdDirs: string[] = []
// returns undefined if not found
export function resolveMd(root: string, pathname: string, md?: string): string {
const docs = path.join(root, "docs");
const tryRead = (fn: string) => {
if (fileExistsSync(fn + ".md"))
return fs.readFileSync(fn + ".md", "utf8")
if (fileExistsSync(fn + "/index.md"))
return fs.readFileSync(fn + "/index.md", "utf8")
return null
}
const targetMd = md ? md : tryRead(path.join(docs, pathname))
if (targetMd && !/^\s*#+\s+@extends/m.test(targetMd))
return targetMd
const dirs = [
path.join(root, "/node_modules/pxt-core/common-docs"),
...getBundledPackagesDocs()
];
for (const d of dirs) {
const template = tryRead(path.join(d, pathname))
if (template)
return pxt.docs.augmentDocs(template, targetMd)
}
return undefined;
}
export function getBundledPackagesDocs(): string[] {
const handledDirectories = {};
const outputDocFolders: string[] = [];
for (const bundledDir of pxt.appTarget.bundleddirs || []) {
getPackageDocs(bundledDir, outputDocFolders, handledDirectories);
}
return outputDocFolders;
/**
* This needs to produce a topologically sorted array of the docs of `dir` and any required packages,
* such that any package listed as a dependency / additionalFilePath of another
* package is added to `folders` before the one that requires it.
*/
function getPackageDocs(packageDir: string, folders: string[], resolvedDirs: pxt.Map) {
if (resolvedDirs[packageDir])
return;
resolvedDirs[packageDir] = true;
const jsonDir = path.join(packageDir, "pxt.json");
const pxtjson = fs.existsSync(jsonDir) && (readJson(jsonDir) as pxt.PackageConfig);
// before adding this package, include the docs of any package this one depends upon.
if (pxtjson) {
/**
* include the package this extends from first;
* that may have dependencies that overlap with this one or that will later be
* overwritten by this one
**/
if (pxtjson.additionalFilePath) {
getPackageDocs(path.join(packageDir, pxtjson.additionalFilePath), folders, resolvedDirs);
}
if (pxtjson.dependencies) {
Object.keys(pxtjson.dependencies).forEach(dep => {
const parts = /^file:(.+)$/i.exec(pxtjson.dependencies[dep]);
if (parts) {
getPackageDocs(path.join(packageDir, parts[1]), folders, resolvedDirs);
}
});
}
}
const docsDir = path.join(packageDir, "docs");
if (fs.existsSync(docsDir)) {
folders.push(docsDir);
}
}
}
export function lazyDependencies(): pxt.Map {
// find pxt-core package
const deps: pxt.Map = {};
[path.join("node_modules", "pxt-core", "package.json"), "package.json"]
.filter(f => fs.existsSync(f))
.map(f => readJson(f))
.forEach(config => config && config.lazyDependencies && Util.jsonMergeFrom(deps, config.lazyDependencies))
return deps;
}
export function lazyRequire(name: string, install = false): any {
/* tslint:disable:non-literal-require */
let r: any;
try {
r = require(name);
} catch (e) {
pxt.debug(e);
pxt.debug((require.resolve).paths(name));
r = undefined;
}
if (!r && install)
pxt.log(`package "${name}" failed to load, run "pxt npminstallnative" to install native depencencies`)
return r;
/* tslint:enable:non-literal-require */
}
export function stringify(content: any) {
if (process.env["PXT_ENV"] === "production") {
return JSON.stringify(content);
}
return JSON.stringify(content, null, 4);
}
init();