forked from microsoft/python-language-server
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathModuleDatabase.cs
More file actions
256 lines (222 loc) · 10.4 KB
/
ModuleDatabase.cs
File metadata and controls
256 lines (222 loc) · 10.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
// Copyright(c) Microsoft Corporation
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the License); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABILITY OR NON-INFRINGEMENT.
//
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using LiteDB;
using Microsoft.Python.Analysis.Analyzer;
using Microsoft.Python.Analysis.Caching.Models;
using Microsoft.Python.Analysis.Dependencies;
using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core;
using Microsoft.Python.Core.Collections;
using Microsoft.Python.Core.IO;
using Microsoft.Python.Core.Logging;
using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Caching {
internal sealed class ModuleDatabase : IModuleDatabaseService {
private const int DatabaseFormatVersion = 1;
private readonly Dictionary<string, IDependencyProvider> _dependencies = new Dictionary<string, IDependencyProvider>();
private readonly object _lock = new object();
private readonly IServiceContainer _services;
private readonly ILogger _log;
private readonly IFileSystem _fs;
private readonly string _databaseFolder;
public ModuleDatabase(IServiceContainer services) {
_services = services;
_log = services.GetService<ILogger>();
_fs = services.GetService<IFileSystem>();
var cfs = services.GetService<ICacheFolderService>();
_databaseFolder = Path.Combine(cfs.CacheFolder, $"analysis.v{DatabaseFormatVersion}");
}
/// <summary>
/// Retrieves dependencies from the module persistent state.
/// </summary>
/// <param name="module">Python module to restore analysis for.</param>
/// <param name="dp">Python module dependency provider.</param>
public bool TryRestoreDependencies(IPythonModule module, out IDependencyProvider dp) {
dp = null;
if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) {
return false;
}
lock (_lock) {
if (_dependencies.TryGetValue(module.Name, out dp)) {
return true;
}
if (FindModuleModel(module.Name, module.FilePath, out var model)) {
dp = new DependencyProvider(module, model);
_dependencies[module.Name] = dp;
return true;
}
}
return false;
}
/// <summary>
/// Creates global scope from module persistent state.
/// Global scope is then can be used to construct module analysis.
/// </summary>
/// <param name="module">Python module to restore analysis for.</param>
/// <param name="gs">Python module global scope.</param>
public bool TryRestoreGlobalScope(IPythonModule module, out IRestoredGlobalScope gs) {
gs = null;
if (GetCachingLevel() == AnalysisCachingLevel.None || !module.ModuleType.CanBeCached()) {
return false;
}
lock (_lock) {
if (FindModuleModel(module.Name, module.FilePath, out var model)) {
gs = new RestoredGlobalScope(model, module);
}
}
return gs != null;
}
/// <summary>
/// Writes module data to the database.
/// </summary>
public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default)
=> Task.Run(() => StoreModuleAnalysis(analysis, cancellationToken), cancellationToken);
/// <summary>
/// Determines if module analysis exists in the storage.
/// </summary>
public bool ModuleExistsInStorage(string moduleName, string filePath) {
if (GetCachingLevel() == AnalysisCachingLevel.None) {
return false;
}
for (var retries = 50; retries > 0; --retries) {
try {
lock (_lock) {
var dbPath = FindDatabaseFile(moduleName, filePath);
return !string.IsNullOrEmpty(dbPath);
}
} catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) {
Thread.Sleep(10);
}
}
return false;
}
public void Clear() {
lock (_lock) {
_dependencies.Clear();
}
}
private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) {
var cachingLevel = GetCachingLevel();
if (cachingLevel == AnalysisCachingLevel.None) {
return;
}
var model = ModuleModel.FromAnalysis(analysis, _services, cachingLevel);
if (model == null) {
// Caching level setting does not permit this module to be persisted.
return;
}
Exception ex = null;
for (var retries = 50; retries > 0; --retries) {
lock (_lock) {
cancellationToken.ThrowIfCancellationRequested();
try {
if (!_fs.DirectoryExists(_databaseFolder)) {
_fs.CreateDirectory(_databaseFolder);
}
cancellationToken.ThrowIfCancellationRequested();
using (var db = new LiteDatabase(Path.Combine(_databaseFolder, $"{model.UniqueId}.db"))) {
var modules = db.GetCollection<ModuleModel>("modules");
modules.Upsert(model);
return;
}
} catch (Exception ex1) when (ex1 is IOException || ex1 is UnauthorizedAccessException) {
ex = ex1;
Thread.Sleep(10);
} catch (Exception ex2) {
ex = ex2;
break;
}
}
}
if (ex != null) {
_log?.Log(System.Diagnostics.TraceEventType.Warning, $"Unable to write analysis of {model.Name} to database. Exception {ex.Message}");
if (ex.IsCriticalException()) {
throw ex;
}
}
}
/// <summary>
/// Locates database file based on module information. Module is identified
/// by name, version, current Python interpreter version and/or hash of the
/// module content (typically file sizes).
/// </summary>
private string FindDatabaseFile(string moduleName, string filePath) {
var interpreter = _services.GetService<IPythonInterpreter>();
var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, ModuleType.Specialized, _services, GetCachingLevel());
if (string.IsNullOrEmpty(uniqueId)) {
return null;
}
// Try module name as is.
var dbPath = Path.Combine(_databaseFolder, $"{uniqueId}.db");
if (_fs.FileExists(dbPath)) {
return dbPath;
}
// TODO: resolving to a different version can be an option
// Try with the major.minor Python version.
var pythonVersion = interpreter.Configuration.Version;
dbPath = Path.Combine(_databaseFolder, $"{uniqueId}({pythonVersion.Major}.{pythonVersion.Minor}).db");
if (_fs.FileExists(dbPath)) {
return dbPath;
}
// Try with just the major Python version.
dbPath = Path.Combine(_databaseFolder, $"{uniqueId}({pythonVersion.Major}).db");
return _fs.FileExists(dbPath) ? dbPath : null;
}
private bool FindModuleModel(string moduleName, string filePath, out ModuleModel model) {
model = null;
// We don't cache results here. Module resolution service decides when to call in here
// and it is responsible of overall management of the loaded Python modules.
for (var retries = 50; retries > 0; --retries) {
try {
// TODO: make combined db rather than per module?
var dbPath = FindDatabaseFile(moduleName, filePath);
if (string.IsNullOrEmpty(dbPath)) {
return false;
}
using (var db = new LiteDatabase(dbPath)) {
if (!db.CollectionExists("modules")) {
return false;
}
var modules = db.GetCollection<ModuleModel>("modules");
model = modules.Find(m => m.Name == moduleName).FirstOrDefault();
return model != null;
}
} catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException) {
Thread.Sleep(10);
}
}
return false;
}
private AnalysisCachingLevel GetCachingLevel()
=> _services.GetService<IAnalysisOptionsProvider>()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None;
private sealed class DependencyProvider : IDependencyProvider {
private readonly ISet<AnalysisModuleKey> _dependencies;
public DependencyProvider(IPythonModule module, ModuleModel model) {
var dc = new DependencyCollector(module);
dc.AddImports(model.Imports);
dc.AddFromImports(model.FromImports);
_dependencies = dc.Dependencies;
}
public ISet<AnalysisModuleKey> GetDependencies(PythonAst ast) => _dependencies;
}
}
}