splunk-sdk-javascript/bin/cli.js at develop · splunk/splunk-sdk-javascript

842 lines (712 loc) · 28.8 KB
// Copyright 2011 Splunk, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
(function () {
    var utils = require('../lib/utils');
    var staticResource = require('../contrib/static-resource/index');
    var dox = require('../contrib/dox/dox');
    var doc_builder = require('../contrib/dox/doc_builder');
    var program = require('../contrib/commander');
    var spawn = require('child_process').spawn;
    var path = require('path');
    var fs = require('fs');
    var browserify = require('browserify');
    var http = require('http');
    var url = require('url');
    var needle = require('needle');
     * Constants
    var DEFAULT_PORT = 6969;
    var REFDOC_DIRECTORY = "refs";
    var CLIENT_DIRECTORY = "client";
    var TEST_DIRECTORY = "tests";
    var TEST_PREFIX = "test_";
    var ALL_TESTS = "tests.js";
    var SDK_BROWSER_ENTRY = "./lib/entries/browser.entry.js";
    var TEST_BROWSER_ENTRY = "./lib/entries/browser.test.entry.js";
    var UI_BROWSER_ENTRY = "./lib/entries/browser.ui.entry.js";
    var DOC_FILE = "index.html";
    var BUILD_CACHE_FILE = ".buildcache";
    var SDK_VERSION = JSON.parse(fs.readFileSync(path.resolve(__dirname, "../package.json")).toString("utf-8")).version;
    var DOC_DIRECTORY = "docs-" + SDK_VERSION;
    var DOC_ASSETS_DIRECTORY = "assets";
    var DOC_DIRECTORY_ASSETS = path.join("bin/docs", DOC_ASSETS_DIRECTORY);
    var GENERATED_DOC_ASSETS = path.join(DOC_DIRECTORY, DOC_ASSETS_DIRECTORY);
    var IGNORED_MODULES = [
        "../contrib/nodeunit/test_reporter",
        "../contrib/nodeunit/junit_reporter",
        "../contrib/commander",
        "../../contrib/commander",
        "./platform/node/node_http",
        "./lib/platform/node/node_http",
        "../lib/platform/node/node_http",
        "mocha"
     * UI Component Entry Points (for async loading)
    var UI_COMPONENT_BROWSER_ENTRY = {
        timeline: "./lib/entries/browser.ui.timeline.entry.js",
        charting: "./lib/entries/browser.ui.charting.entry.js"
     * Generated files
    var COMPILED_SDK = path.join(CLIENT_DIRECTORY, "splunk.js");
    var COMPILED_SDK_MIN = path.join(CLIENT_DIRECTORY, "splunk.min.js");
    var COMPILED_TEST = path.join(CLIENT_DIRECTORY, "splunk.test.js");
    var COMPILED_TEST_MIN = path.join(CLIENT_DIRECTORY, "splunk.test.min.js");
    var COMPILED_UI = path.join(CLIENT_DIRECTORY, "splunk.ui.js");
    var COMPILED_UI_MIN = path.join(CLIENT_DIRECTORY, "splunk.ui.min.js");
    var GENERATED_DOCS = path.join(DOC_DIRECTORY, SDK_VERSION, DOC_FILE);
    var GENERATED_REF_DOCS = path.join(DOC_DIRECTORY, SDK_VERSION, REFDOC_DIRECTORY, DOC_FILE);
    var GENERATED_DOCS_DIR = path.join(DOC_DIRECTORY, SDK_VERSION);
     * Helpers
    var serverProxy = function (req, res) {
        var error = { d: { __messages: [{ type: "ERROR", text: "Proxy Error", code: "PROXY" }] } };
        var writeError = function () {
            res.writeHead(500, {});
            res.write(JSON.stringify(error));
            res.end();
        try {
            var body = "";
            req.on('data', function (data) {
                body += data.toString("utf-8");
            req.on('end', function () {
                var destination = req.headers["X-ProxyDestination".toLowerCase()];
                var options = {
                    url: destination,
                    method: req.method,
                    headers: {
                        "Content-Length": req.headers["content-length"] || 0,
                        "Content-Type": req.headers["content-type"] || '',
                        "Authorization": req.headers["authorization"] || '',
                        "User-Agent": "splunk-sdk-javascript/" + SDK_VERSION
                    followAllRedirects: true,
                    body: body || '',
                    jar: false,
                    strictSSL: false,
                    rejectUnauthorized: false,
                    parse_response: false
                    needle(options.method, options.url, options.body, options)
                    .then((response) => {
                        var statusCode = (response ? response.statusCode : 500) || 500;
                        var headers = (response ? response.headers : {}) || {};
                        res.writeHead(statusCode, headers);
                        res.write(response.body);
                        res.end();
                    }).catch((err)=>{
                        res.write(JSON.stringify(err));
                        res.end();
                catch (ex) {
                    writeError();
        catch (ex) {
            writeError();
    var createServer = function (port) {
        // passing where is going to be the document root of resources.
        var handler = staticResource.createHandler(fs.realpathSync(path.resolve(__dirname, "..")));
        var server = http.createServer(function (request, response) {
            var path = url.parse(request.url).pathname;
            if (utils.startsWith(path, "/proxy")) {
                serverProxy(request, response);
                return;
            // handle method returns true if a resource specified with the path
            // has been handled by handler and returns false otherwise.
            if (!handler.handle(path, request, response)) {
                response.writeHead(404);
                response.write('404');
                response.end();
        });
        port = port || DEFAULT_PORT;
        server.listen(port);
        console.log("Running server on port: " + (port) + " -- Hit CTRL+C to exit");
    var makeOption = function (name, value) {
        return ["--" + name, value];
    var makeURL = function (file, port) {
        return "http://localhost:" + (port ? port : DEFAULT_PORT) + "/" + file;
    var temp = {
        _defaultDirectory: '/tmp',
        _environmentVariables: ['TMPDIR', 'TMP', 'TEMP'],
        _findDirectory: function () {
            for (var i = 0; i < temp._environmentVariables.length; i++) {
                var value = process.env[temp._environmentVariables[i]];
                if (value) {
                    return fs.realpathSync(value);
            return fs.realpathSync(temp._defaultDirectory);
        _generateName: function () {
            var now = new Date();
            var name = ["__",
                now.getYear(), now.getMonth(), now.getDay(),
                process.pid,
                (Math.random() * 0x100000000 + 1).toString(36),
                "__"].join('');
            return path.join(temp._findDirectory(), name);
        mkdirSync: function () {
            var tempDirPath = temp._generateName();
            fs.mkdirSync(tempDirPath, "755");
            return tempDirPath;
    // Taken from wrench.js
    var copyDirectoryRecursiveSync = function (sourceDir, newDirLocation, opts) {
        if (!opts || !opts.preserve) {
            try {
                if (fs.statSync(newDirLocation).isDirectory()) {
                    exports.rmdirSyncRecursive(newDirLocation);
            catch (e) { }
        /*  Create the directory where all our junk is moving to; read the mode of the source directory and mirror it */
        var checkDir = fs.statSync(sourceDir);
        try {
            fs.mkdirSync(newDirLocation, checkDir.mode);
        catch (e) {
            //if the directory already exists, that's okay
            if (e.code !== 'EEXIST') {
                throw e;
        var files = fs.readdirSync(sourceDir);
        for (var i = 0; i < files.length; i++) {
            var currFile = fs.lstatSync(sourceDir + "/" + files[i]);
            if (currFile.isDirectory()) {
                /*  recursion this thing right on back. */
                copyDirectoryRecursiveSync(sourceDir + "/" + files[i], newDirLocation + "/" + files[i], opts);
            else if (currFile.isSymbolicLink()) {
                var symlinkFull = fs.readlinkSync(sourceDir + "/" + files[i]);
                fs.symlinkSync(symlinkFull, newDirLocation + "/" + files[i]);
            else {
                /*  At this point, we've hit a file actually worth copying... so copy it on over. */
                var contents = fs.readFileSync(sourceDir + "/" + files[i]);
                fs.writeFileSync(newDirLocation + "/" + files[i], contents);
    var rmdirRecursiveSync = function (path, failSilent) {
        var files;
        try {
            files = fs.readdirSync(path);
        catch (err) {
            if (failSilent) {
                return;
            throw new Error(err.message);
        /*  Loop through and delete everything in the sub-tree after checking it */
        for (var i = 0; i < files.length; i++) {
            var currFile = fs.lstatSync(path + "/" + files[i]);
            if (currFile.isDirectory()) {// Recursive function back to the beginning
                rmdirRecursiveSync(path + "/" + files[i]);
            else if (currFile.isSymbolicLink()) {// Unlink symlinks
                fs.unlinkSync(path + "/" + files[i]);
            else { // Assume it's a file - perhaps a try/catch belongs here?
                fs.unlinkSync(path + "/" + files[i]);
        /*  Now that we know everything in the sub-tree has been deleted, we can delete the main
            directory. Huzzah for the shopkeep. */
        return fs.rmdirSync(path);
    var git = {
        execute: function (args, callback) {
            var program = spawn("git", args);
            process.on("exit", function () {
                program.kill();
            program.stderr.on("data", function (data) {
                process.stderr.write(data);
            return program;
        stash: function (callback) {
            var program = git.execute(["stash"], callback);
            program.on("exit", function (code) {
                if (code) {
                    throw new Error("Stash error");
                else {
                    callback();
        unstash: function (callback) {
            var program = git.execute(["stash", "pop"], callback);
            program.on("exit", function (code) {
                if (code) {
                    throw new Error("Unstash error");
                else {
                    callback();
        switchBranch: function (toBranch, callback) {
            var program = git.execute(["checkout", toBranch], callback);
            program.on("exit", function (code) {
                if (code) {
                    throw new Error("Switch branch error error");
                else {
                    callback();
        currentBranch: function (callback) {
            var program = git.execute(["symbolic-ref", "HEAD"], callback);
            var buffer = "";
            program.stdout.on("data", function (data) {
                buffer = data.toString("utf-8");
            program.on("exit", function (code) {
                if (code) {
                    throw new Error("Couldn't determine current branch name");
                else {
                    var branchName = buffer.replace("refs/heads/", "").trim();
                    callback(null, branchName);
        add: function (filename, callback) {
            var program = git.execute(["add", filename], callback);
            program.on("exit", function (code) {
                if (code) {
                    throw new Error("Add error");
                else {
                    callback(null);
        commit: function (msg, callback) {
            var program = git.execute(["commit", "-m", msg], callback);
            program.on("exit", function (code) {
                if (code) {
                    throw new Error("Commit error");
                else {
                    callback(null);
        push: function (branch, callback) {
            var program = git.execute(["push", "origin", branch], callback);
            program.on("exit", function (code) {
                if (code) {
                    throw new Error("push error");
                else {
                    callback(null);
    var launch = function (file, args, done) {
        done = done || function () { };
        // Add the file to the arguments
        args = args || [];
        args = args.slice();
        args.unshift(file);
        args.unshift("./node_modules/mocha/bin/mocha");
        args.unshift("./node_modules/nyc/bin/nyc")
        args.push("--color=always");
        args = args.filter(arg => arg);
        // Spawn
        var program = spawn("node", args);
        program.stdout.on("data", function (data) {
            var str = data.toString("utf-8");
            process.stdout.write(str);
        });
        program.stderr.on("data", function (data) {
            var str = data.toString("utf-8");
            process.stderr.write(str);
        });
        var exitCode = 0;
        program.on("exit", function (code) {
            if (code) {
                exitCode = code;
                done(code);
            else {
                done();
        });
        process.on("exit", function () {
            program.kill();
            process.reallyExit(exitCode);
        });
        return program;
    var getDependencies = function (entry) {
        var bundle = browserify({
            entry: entry,
            ignore: IGNORED_MODULES,
            cache: BUILD_CACHE_FILE
        });
        var dependencies = [entry];
        for (var file in bundle.files) {
            if (bundle.files.hasOwnProperty(file)) {
                dependencies.push(file);
        return dependencies;
    var compile = function (entry, path, shouldUglify, watch, exportName) {
        exportName = exportName || "splunkjs";
        // Compile/combine all the files into the package
        var bundle = browserify();
        bundle.add(entry);
        bundle.ignore(IGNORED_MODULES);
        bundle.bundle(function (err,src){
            if(err){
                throw err;
            var code = [
                            "(function() {",
                            "var __exportName = '" + exportName + "';",
                            src.toString(),
                        ].join("\n");
            if (shouldUglify){
                var UglifyJS = require("uglify-js");
                code = UglifyJS.minify({"file1.js":code},{toplevel:true}).code;
            fs.writeFileSync(path,code);
            console.log("Compiled " + path);
        });
    var outOfDate = function (dependencies, compiled, compiledMin) {
        if (!fs.existsSync(compiled) || !fs.existsSync(compiledMin)) {
            return true;
        var compiledTime = fs.statSync(compiled).mtime;
        var compiledMinTime = fs.statSync(compiledMin).mtime;
        var latestDependencyTime = Math.max.apply(null, dependencies.map(function (path) {
            return fs.statSync(path).mtime;
        }));
        return latestDependencyTime > compiledTime || latestDependencyTime > compiledMinTime;
    var ensureDirectoryExists = function (dir) {
        if (!fs.existsSync(dir)) {
            fs.mkdirSync(dir, "755");
    var ensureClientDirectory = function () {
        ensureDirectoryExists(CLIENT_DIRECTORY);
     * Tasks
    var compileSDK = function (watch, exportName) {
        ensureClientDirectory();
        var dependencies = getDependencies(SDK_BROWSER_ENTRY);
        if (!outOfDate(dependencies, COMPILED_SDK, COMPILED_SDK_MIN)) {
            console.log("Compiled SDK is not out of date -- skipping...");
            return;
        compile(SDK_BROWSER_ENTRY, COMPILED_SDK, false, watch, exportName);
        compile(SDK_BROWSER_ENTRY, COMPILED_SDK_MIN, true, watch, exportName);
    var compileTests = function (watch, exportName) {
        ensureClientDirectory();
        var dependencies = getDependencies(TEST_BROWSER_ENTRY);
        if (!outOfDate(dependencies, COMPILED_TEST, COMPILED_TEST_MIN)) {
            console.log("Compiled tests are not out of date -- skipping...");
            return;
        compile(TEST_BROWSER_ENTRY, COMPILED_TEST, false, watch);
        compile(TEST_BROWSER_ENTRY, COMPILED_TEST_MIN, true, watch);
    var compileUI = function (watch, exportName) {
        ensureClientDirectory();
        var dependencies = getDependencies(UI_BROWSER_ENTRY);
        if (outOfDate(dependencies, COMPILED_UI, COMPILED_UI_MIN)) {
            compile(UI_BROWSER_ENTRY, COMPILED_UI, false, watch, exportName);
            compile(UI_BROWSER_ENTRY, COMPILED_UI_MIN, true, watch, exportName);
        else {
            console.log("Compiled UI is not out of date -- skipping...");
        for (var component in UI_COMPONENT_BROWSER_ENTRY) {
            if (!UI_COMPONENT_BROWSER_ENTRY.hasOwnProperty(component)) {
                continue;
            var entryPath = UI_COMPONENT_BROWSER_ENTRY[component];
            var generatedPath = path.join(CLIENT_DIRECTORY, "splunk.ui." + component + ".js");
            var generatedMinPath = path.join(CLIENT_DIRECTORY, "splunk.ui." + component + ".min.js");
            dependencies = getDependencies(entryPath);
            if (!outOfDate(dependencies, generatedPath, generatedMinPath)) {
                console.log("Compiled " + component + " is not out of date -- skipping...");
                continue;
            compile(entryPath, generatedPath, false, watch, exportName);
            compile(entryPath, generatedMinPath, true, watch, exportName);
    var compileAll = function (watch, exportName) {
        compileSDK(watch, exportName);
        compileTests(watch);
        compileUI(watch, exportName);
    var runServer = function (port) {
        // TODO: compile doesn't work on Windows, so lets not
        // make runServer depend on it
        createServer(port);
    var launchBrowser = function (file, port) {
        if (!fs.existsSync(file)) {
            throw new Error("File does not exist: " + file);
        if (process.platform === "win32") {
            spawn("cmd.exe", ["/C", "start " + makeURL(file, port)]);
        else {
            spawn("open", [makeURL(file, port)]);
    var launchBrowserTests = function (port) {
        runServer(port);
        launchBrowser("tests/tests.browser.html", port);
    var generateDocs = function (callback) {
        callback = (callback && utils.isFunction(callback)) ? callback : (function () { });
        var files = [
            "lib/log.js",
            "lib/http.js",
            "lib/utils.js",
            "lib/context.js",
            "lib/service.js",
        var comments = [];
        files.forEach(function (file) {
            var contents = fs.readFileSync(file).toString("utf-8");
            var obj = dox.parseComments(contents, file);
            comments = comments.concat(obj);
        });
        doc_builder.generate(comments, SDK_VERSION, function (err, data) {
            if (err) {
                throw err;
            ensureDirectoryExists(DOC_DIRECTORY);
            // copy static assets directory
            copyDirectoryRecursiveSync(DOC_DIRECTORY_ASSETS, GENERATED_DOC_ASSETS);
            for (var name in data) {
                var htmlPath = path.join(DOC_DIRECTORY, name + ".html");
                fs.writeFileSync(htmlPath, data[name]);
            callback(null);
        });
    var uploadDocs = function () {
        var originalBranch = "master";
        var tempPath = "";
        Async.chain([
            function (done) {
                git.currentBranch(done);
            function (branchName, done) {
                originalBranch = branchName;
                generateDocs(done);
            function (done) {
                var tempDirPath = temp.mkdirSync();
                tempPath = tempDirPath;
                copyDirectoryRecursiveSync(GENERATED_DOCS_DIR, tempDirPath);
                done();
            function (done) {
                git.stash(done);
            function (done) {
                git.switchBranch("gh-pages", done);
            function (done) {
                if (fs.existsSync(GENERATED_DOCS_DIR)) {
                    rmdirRecursiveSync(GENERATED_DOCS_DIR);
                ensureDirectoryExists(DOC_DIRECTORY);
                ensureDirectoryExists(path.join(DOC_DIRECTORY, SDK_VERSION));
                copyDirectoryRecursiveSync(tempPath, GENERATED_DOCS_DIR);
                done();
            function (done) {
                git.add(GENERATED_DOCS_DIR, done);
            function (done) {
                git.commit("Updating v" + SDK_VERSION + " docs: " + (new Date()), done);
            function (done) {
                git.push("gh-pages", done);
            function (done) {
                git.switchBranch(originalBranch, done);
            function (done) {
                git.unstash(done);
            function (err) {
                if (err) {
                    console.log(err);
    var runTests = async function (tests, cmdline) {
        cmdline = cmdline || { opts: {} };
        let args = (tests || "").split(",").map(function (arg) { return arg.trim(); });
        let files = args
            .map(arg => {
                if (arg.indexOf('service_tests') >= 0) {
                    return path.join(TEST_DIRECTORY, 'service_tests', arg.split('/')[1] + ".js");
                else {
                    return path.join(TEST_DIRECTORY, TEST_PREFIX + arg + ".js");
            }).filter(file => fs.existsSync(file));
        if (files.length === 0) {
            if (args.length > 0 && args[0].length !== 0) {
                console.log("Unable to find test files for:", args.join(", "));
                process.exit(1);
            files.push(path.join(TEST_DIRECTORY, ALL_TESTS));
        let cmdlineArgs = []
            .concat(cmdline.opts.username ? makeOption("username", cmdline.opts.username) : "")
            .concat(cmdline.opts.scheme ? makeOption("scheme", cmdline.opts.scheme) : "")
            .concat(cmdline.opts.host ? makeOption("host", cmdline.opts.host) : "")
            .concat(cmdline.opts.port ? makeOption("port", cmdline.opts.port) : "")
            .concat(cmdline.opts.app ? makeOption("app", cmdline.opts.app) : "")
            .concat(cmdline.opts.version ? makeOption("version", cmdline.opts.version) : "")
            .concat(cmdline.opts.password ? makeOption("password", cmdline.opts.password) : "")
            .concat(cmdline.opts.reporter ? makeOption("reporter", cmdline.opts.reporter.toLowerCase()) : "")
            .concat(cmdline.opts.ui ? makeOption("ui", cmdline.opts.ui) : ["--ui", "bdd"])
            .concat(cmdline.opts.timeout ? makeOption("timeout", cmdline.opts.timeout) : ["--timeout", "5000"])
            .concat(cmdline.opts.grep ? makeOption("grep", cmdline.opts.grep) : "")
            .concat(cmdline.opts.exit ? "--exit" : "--exit")
            .concat(cmdline.opts.quiet ? "--quiet" : "");
        let testFunctions = files.map(function (file) {
            return function (done) {
                launch(file, cmdlineArgs, done);
        });
        await utils.series(testFunctions);
    var hint = function () {
        var hintRequirePath = path.join(path.resolve(require.resolve('jshint'), './../../../'), 'lib', 'cli');
        var jshint = require(hintRequirePath);
        jshint.interpret(['node', 'jshint', '.']);
    program
        .version('0.0.1');
    program
        .command('compile-sdk [global]')
        .description('Compile all SDK files into a single, browser-includable file.')
        .action(function (globalName) {
            compileSDK(false, globalName);
        });
    program
        .command('compile-test')
        .description('Compile all test files into a single, browser-includable file.')
        .action(compileTests);
    program
        .command('compile-ui')
        .description('Compile all UI files into a single, browser-includable file.')
        .action(compileUI);
    program
        .command('compile [global]')
        .description('Compile all files into several single, browser-includable files.')
        .action(function (globalName) {
            compileAll(false, globalName);
        });
    program
        .command('runserver [port]')
        .description('Run a local server to serve tests.')
        .action(runServer);
    program
        .command('launch-browser <file> [port]')
        .description('Launch the browser to the specified file, running on the local server.')
        .action(launchBrowser);
    program
        .command('tests [files]')
        .description('Run the specified test files (comma-separated), or all of them if no file is specified.')
        .option('--username <username>', 'Splunk username')
        .option('--password <password>', 'Splunk password')
        .option('--scheme <scheme>', 'Splunk scheme')
        .option('--host <host>', 'Splunk host')
        .option('--port <port>', 'Splunk port')
        .option('--version <version>', 'Splunk version')
        .option('--namespace <namespace>', 'Splunk namespace (in the form of owner:app)')
        .option('--reporter <reporter>', '(optional) How to report results')
        .option('--ui <ui>', 'Specify user interface')
        .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds)')
        .option('--grep <grep>', 'Only run tests matching this string or regexp')
        .option('--exit', '(optional) Force Mocha to quit after tests complete')
        .option('--quiet', '(optional) Hides splunkd output.')
        .action(runTests);
    program
        .command('tests-browser [port]')
        .description('Launch the browser test suite.')
        .action(launchBrowserTests);
    program
        .command('hint')
        .description('Run JSHint on the codebase.')
        .action(hint);
    program
        .command('docs')
        .description('Generate reference documentation for the SDK.')
        .action(generateDocs);
    program
        .command('uploaddocs')
        .description('Upload docs to GitHub.')
        .action(uploadDocs);
    program.parse(process.argv);
    if (!program.executedCommand) {
        process.stdout.write(program.helpInformation());
Provide feedback

Saved searches

Use saved searches to filter your results more quickly

FilesExpand file tree

cli.js

Latest commit

History

cli.js

File metadata and controls
