Repository navigation
Expand file tree
/
Copy pathperformanceTest.ts
More file actions
151 lines (125 loc) · 6.27 KB
/
Copy pathperformanceTest.ts
File metadata and controls
151 lines (125 loc) · 6.27 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
'use strict';
/*
Comparing performance metrics is not easy (the metrics can and always get skewed).
One approach is to run the tests multile times and gather multiple sample data.
For Extension activation times, we load both extensions x times, and re-load the window y times in each x load.
I.e. capture averages by giving the extensions sufficient time to warm up.
This block of code merely launches the tests by using either the dev or release version of the extension,
and spawning the tests (mimic user starting tests from command line), this way we can run tests multiple times.
*/
// Must always be on top to setup expected env.
process.env.VSC_PYTHON_PERF_TEST = '1';
import { spawn } from 'child_process';
import * as download from 'download';
import * as fs from '../client/common/platform/fs-paths';
import * as path from 'path';
import * as bent from 'bent';
import { LanguageServerType } from '../client/activation/types';
import { EXTENSION_ROOT_DIR, PVSC_EXTENSION_ID } from '../client/common/constants';
import { unzip } from './common';
import { initializeLogger } from './testLogger';
initializeLogger();
const NamedRegexp = require('named-js-regexp');
const del = require('del');
const tmpFolder = path.join(EXTENSION_ROOT_DIR, 'tmp');
const publishedExtensionPath = path.join(tmpFolder, 'ext', 'testReleaseExtensionsFolder');
const logFilesPath = path.join(tmpFolder, 'test', 'logs');
enum Version {
Dev,
Release,
}
class TestRunner {
public async start() {
await del([path.join(tmpFolder, '**')]);
await this.extractLatestExtension(publishedExtensionPath);
const timesToLoadEachVersion = 2;
const devLogFiles: string[] = [];
const releaseLogFiles: string[] = [];
const languageServerLogFiles: string[] = [];
for (let i = 0; i < timesToLoadEachVersion; i += 1) {
await this.enableLanguageServer();
const devLogFile = path.join(logFilesPath, `dev_loadtimes${i}.txt`);
console.log(`Start Performance Tests: Counter ${i}, for Dev version with Jedi`);
await this.capturePerfTimes(Version.Dev, devLogFile);
devLogFiles.push(devLogFile);
const releaseLogFile = path.join(logFilesPath, `release_loadtimes${i}.txt`);
console.log(`Start Performance Tests: Counter ${i}, for Release version with Jedi`);
await this.capturePerfTimes(Version.Release, releaseLogFile);
releaseLogFiles.push(releaseLogFile);
}
console.log('Compare Performance Results');
await this.runPerfTest(devLogFiles, releaseLogFiles, languageServerLogFiles);
}
private async enableLanguageServer() {
const settings = `{ "python.languageServer": "${LanguageServerType.Jedi}" }`;
await fs.writeFile(path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'performance', 'settings.json'), settings);
}
private async capturePerfTimes(version: Version, logFile: string) {
const releaseVersion = await this.getReleaseVersion();
const devVersion = await this.getDevVersion();
await fs.ensureDir(path.dirname(logFile));
const env: Record<string, {}> = {
ACTIVATION_TIMES_LOG_FILE_PATH: logFile,
ACTIVATION_TIMES_EXT_VERSION: version === Version.Release ? releaseVersion : devVersion,
CODE_EXTENSIONS_PATH: version === Version.Release ? publishedExtensionPath : EXTENSION_ROOT_DIR,
};
await this.launchTest(env);
}
private async runPerfTest(devLogFiles: string[], releaseLogFiles: string[], languageServerLogFiles: string[]) {
const env: Record<string, {}> = {
ACTIVATION_TIMES_DEV_LOG_FILE_PATHS: JSON.stringify(devLogFiles),
ACTIVATION_TIMES_RELEASE_LOG_FILE_PATHS: JSON.stringify(releaseLogFiles),
ACTIVATION_TIMES_DEV_LANGUAGE_SERVER_LOG_FILE_PATHS: JSON.stringify(languageServerLogFiles),
};
await this.launchTest(env);
}
private async launchTest(customEnvVars: Record<string, {}>) {
await new Promise<void>((resolve, reject) => {
const env: Record<string, string> = {
TEST_FILES_SUFFIX: 'perf.test',
CODE_TESTS_WORKSPACE: path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'performance'),
...process.env,
...customEnvVars,
};
const proc = spawn('node', [path.join(__dirname, 'standardTest.js')], { cwd: EXTENSION_ROOT_DIR, env });
proc.stdout.pipe(process.stdout);
proc.stderr.pipe(process.stderr);
proc.on('error', reject);
proc.on('close', (code) => {
if (code === 0) {
resolve();
} else {
reject(`Failed with code ${code}.`);
}
});
});
}
private async extractLatestExtension(targetDir: string): Promise<void> {
const extensionFile = await this.downloadExtension();
await unzip(extensionFile, targetDir);
}
private async getReleaseVersion(): Promise<string> {
const url = `https://marketplace.visualstudio.com/items?itemName=${PVSC_EXTENSION_ID}`;
const request = bent.default('string', 'GET', 200);
const content: string = await request(url);
const re = NamedRegexp('"version"S?:S?"(:<version>\\d{4}\\.\\d{1,2}\\.\\d{1,2})"', 'g');
const matches = re.exec(content);
return matches.groups().version;
}
private async getDevVersion(): Promise<string> {
return require(path.join(EXTENSION_ROOT_DIR, 'package.json')).version;
}
private async downloadExtension(): Promise<string> {
const version = await this.getReleaseVersion();
const url = `https://marketplace.visualstudio.com/_apis/public/gallery/publishers/ms-python/vsextensions/python/${version}/vspackage`;
const destination = path.join(__dirname, `extension${version}.zip`);
if (await fs.pathExists(destination)) {
return destination;
}
await download.default(url, path.dirname(destination), { filename: path.basename(destination) });
return destination;
}
}
new TestRunner().start().catch((ex) => console.error('Error in running Performance Tests', ex));