Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Merges from db #1446

Merged
Alexander Sher (AlexanderSher) merged 1 commit into from
Aug 16, 2019
Merged

Merges from db #1446

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,18 @@ private async Task LoadBuiltinTypesAsync(string root, IServiceManager sm, Cancel

sm.AddService(this);
_moduleResolution = new MainModuleResolution(root, sm);
_stubResolution = new TypeshedResolution(sm);

await _moduleResolution.InitializeAsync(cancellationToken);
await _stubResolution.InitializeAsync(cancellationToken);

lock (_lock) {
var builtinModule = _moduleResolution.CreateBuiltinsModule();
_builtinTypes[BuiltinTypeId.NoneType]
= new PythonType("NoneType", new Location(builtinModule), string.Empty, BuiltinTypeId.NoneType);
_builtinTypes[BuiltinTypeId.Unknown]
= UnknownType = new PythonType("Unknown", new Location(builtinModule), string.Empty);
}
await _moduleResolution.InitializeAsync(cancellationToken);

_stubResolution = new TypeshedResolution(sm);
await _stubResolution.InitializeAsync(cancellationToken);

await _moduleResolution.LoadBuiltinTypesAsync(cancellationToken);
}
Expand Down
2 changes: 1 addition & 1 deletion src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ private bool TryAddModulePath(ModuleCreationOptions mco) {
throw new InvalidOperationException("Can't create document with no file path or URI specified");
}

if (!ModuleManagement.TryAddModulePath(filePath, true, out var fullName)) {
if (!ModuleManagement.TryAddModulePath(filePath, 0, true, out var fullName)) {
return false;
}

Expand Down
7 changes: 3 additions & 4 deletions src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using Microsoft.Python.Analysis.Caching;
using Microsoft.Python.Analysis.Core.Interpreter;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core.Collections;

namespace Microsoft.Python.Analysis.Modules {
/// <summary>
Expand All @@ -32,14 +33,12 @@ public interface IModuleManagement: IModuleResolution {
/// <returns></returns>
ModulePath FindModule(string filePath);

IReadOnlyCollection<string> GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken = default);

/// <summary>
/// Cache of module stubs generated from compiled modules.
/// </summary>
IStubCache StubCache { get; }

bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullName);
bool TryAddModulePath(in string path, in long fileSize, in bool allowNonRooted, out string fullName);

/// <summary>
/// Provides ability to specialize module by replacing module import by
Expand All @@ -65,6 +64,6 @@ public interface IModuleManagement: IModuleResolution {
/// <summary>
/// Set of interpreter paths.
/// </summary>
IEnumerable<string> InterpreterPaths { get; }
ImmutableArray<string> InterpreterPaths { get; }
}
}
25 changes: 14 additions & 11 deletions src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Analysis.Values;
using Microsoft.Python.Core;
using Microsoft.Python.Core.Collections;
using Microsoft.Python.Core.Diagnostics;
using Microsoft.Python.Core.IO;
using Microsoft.Python.Core.OS;
Expand All @@ -38,7 +39,7 @@ internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManage
private readonly ConcurrentDictionary<string, IPythonModule> _specialized = new ConcurrentDictionary<string, IPythonModule>();
private IRunningDocumentTable _rdt;

private IEnumerable<string> _userPaths = Enumerable.Empty<string>();
private ImmutableArray<string> _userPaths = ImmutableArray<string>.Empty;

public MainModuleResolution(string root, IServiceContainer services)
: base(root, services) { }
Expand Down Expand Up @@ -200,22 +201,24 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) {
}

// Preserve builtins, they don't need to be reloaded since interpreter does not change.
var builtins = Modules[BuiltinModuleName];
Modules.Clear();
Modules[BuiltinModuleName] = builtins;

PathResolver = new PathResolver(_interpreter.LanguageVersion);

var addedRoots = new HashSet<string>();
addedRoots.UnionWith(PathResolver.SetRoot(Root));
if (Modules.TryGetValue(BuiltinModuleName, out var builtins)) {
Modules.Clear();
Modules[BuiltinModuleName] = builtins;
}

await ReloadSearchPaths(cancellationToken);

addedRoots.UnionWith(PathResolver.SetInterpreterSearchPaths(InterpreterPaths));
addedRoots.UnionWith(PathResolver.SetUserSearchPaths(_userPaths));
PathResolver = new PathResolver(_interpreter.LanguageVersion, Root, InterpreterPaths, _userPaths);

var addedRoots = new HashSet<string> { Root };
addedRoots.UnionWith(InterpreterPaths);
addedRoots.UnionWith(_userPaths);
ReloadModulePaths(addedRoots);
}

public bool TryAddModulePath(in string path, in long fileSize, in bool allowNonRooted, out string fullModuleName)
=> PathResolver.TryAddModulePath(path, fileSize, allowNonRooted, out fullModuleName);

// For tests
internal void AddUnimportableModule(string moduleName)
=> Modules[moduleName] = new ModuleRef(new SentinelModule(moduleName, _services));
Expand Down
22 changes: 5 additions & 17 deletions src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
using Microsoft.Python.Analysis.Documents;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core;
using Microsoft.Python.Core.Collections;
using Microsoft.Python.Core.IO;
using Microsoft.Python.Core.Logging;
using Microsoft.Python.Core.Services;
Expand Down Expand Up @@ -55,9 +56,8 @@ protected ModuleResolutionBase(string root, IServiceContainer services) {
}

public string Root { get; protected set; }
public IEnumerable<string> InterpreterPaths { get; protected set; } = Enumerable.Empty<string>();
public ImmutableArray<string> InterpreterPaths { get; protected set; } = ImmutableArray<string>.Empty;

public IModuleCache ModuleCache { get; protected set; }
public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion);

/// <summary>
Expand All @@ -72,15 +72,6 @@ protected ModuleResolutionBase(string root, IServiceContainer services) {

protected abstract IPythonModule CreateModule(string name);

public IReadOnlyCollection<string> GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken) {
return ModulePath.GetModulesInPath(
searchPath,
recurse: false,
includePackages: true,
requireInitPy: _requireInitPy
).Select(mp => mp.ModuleName).Where(n => !string.IsNullOrEmpty(n)).TakeWhile(_ => !cancellationToken.IsCancellationRequested).ToList();
}

public IStubCache StubCache { get; protected set; }

public IPythonModule GetImportedModule(string name)
Expand All @@ -99,10 +90,7 @@ public IPythonModule GetOrLoadModule(string name) {
moduleRef = Modules.GetOrAdd(name, new ModuleRef());
return moduleRef.GetOrCreate(name, this);
}

public bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullModuleName)
=> PathResolver.TryAddModulePath(path, allowNonRooted, out fullModuleName);


public ModulePath FindModule(string filePath) {
var bestLibraryPath = string.Empty;

Expand All @@ -117,8 +105,8 @@ public ModulePath FindModule(string filePath) {
}

protected void ReloadModulePaths(in IEnumerable<string> rootPaths) {
foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(p))) {
PathResolver.TryAddModulePath(modulePath, false, out _);
foreach (var moduleFile in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(_fs, p))) {
PathResolver.TryAddModulePath(moduleFile.FullName, moduleFile.Length, false, out _);
}
}

Expand Down
61 changes: 10 additions & 51 deletions src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,12 @@
using Microsoft.Python.Analysis.Core.Interpreter;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core;
using Microsoft.Python.Core.Collections;
using Microsoft.Python.Core.IO;

namespace Microsoft.Python.Analysis.Modules.Resolution {
internal sealed class TypeshedResolution : ModuleResolutionBase, IModuleResolution {
private readonly IReadOnlyList<string> _typeStubPaths;
private readonly ImmutableArray<string> _typeStubPaths;

public TypeshedResolution(IServiceContainer services) : base(null, services) {
BuiltinsModule = _interpreter.ModuleResolution.BuiltinsModule;
Expand All @@ -41,7 +42,7 @@ public TypeshedResolution(IServiceContainer services) : base(null, services) {
_typeStubPaths = GetTypeShedPaths(Root)
.Concat(GetTypeShedPaths(stubs))
.Where(services.GetService<IFileSystem>().DirectoryExists)
.ToArray();
.ToImmutableArray();

_log?.Log(TraceEventType.Verbose, @"Typeshed paths:");
foreach (var p in _typeStubPaths) {
Expand All @@ -53,13 +54,13 @@ internal Task InitializeAsync(CancellationToken cancellationToken = default)
=> ReloadAsync(cancellationToken);

protected override IPythonModule CreateModule(string name) {
var mp = FindModuleInSearchPath(_typeStubPaths, null, name);
if (mp != null) {
if (mp.Value.IsCompiled) {
_log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", mp.Value.FullName, mp.Value.SourceFile);
var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name);
if (moduleImport != default) {
if (moduleImport.IsCompiled) {
_log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", moduleImport.FullName, moduleImport.ModulePath);
return null;
}
return new StubPythonModule(mp.Value.FullName, mp.Value.SourceFile, true, _services);
return new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, _services);
}

var i = name.IndexOf('.');
Expand All @@ -74,14 +75,8 @@ protected override IPythonModule CreateModule(string name) {

public Task ReloadAsync(CancellationToken cancellationToken = default) {
Modules.Clear();
PathResolver = new PathResolver(_interpreter.LanguageVersion);

var addedRoots = PathResolver.SetRoot(Root);
ReloadModulePaths(addedRoots);

addedRoots = PathResolver.SetInterpreterSearchPaths(_typeStubPaths);
ReloadModulePaths(addedRoots);

PathResolver = new PathResolver(_interpreter.LanguageVersion, Root, _typeStubPaths, ImmutableArray<string>.Empty);
ReloadModulePaths(_typeStubPaths.Prepend(Root));
cancellationToken.ThrowIfCancellationRequested();
return Task.CompletedTask;
}
Expand Down Expand Up @@ -109,41 +104,5 @@ private IEnumerable<string> GetTypeShedPaths(string typeshedRootPath) {
yield return Path.Combine(thirdParty, subdir);
}
}

private ModulePath? FindModuleInSearchPath(IReadOnlyList<string> searchPaths, IReadOnlyDictionary<string, string> packages, string name) {
if (searchPaths == null || searchPaths.Count == 0) {
return null;
}

var i = name.IndexOf('.');
var firstBit = i < 0 ? name : name.Remove(i);

ModulePath mp;
Func<string, bool> isPackage = IsPackage;
if (firstBit.EndsWithOrdinal("-stubs", ignoreCase: true)) {
isPackage = _fs.DirectoryExists;
}

var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version);
if (packages != null && packages.TryGetValue(firstBit, out var searchPath) && !string.IsNullOrEmpty(searchPath)) {
if (ModulePath.FromBasePathAndName_NoThrow(searchPath, name, isPackage, null, requireInitPy, out mp, out _, out _, out _)) {
return mp;
}
}

if (searchPaths.MaybeEnumerate()
.Any(sp => ModulePath.FromBasePathAndName_NoThrow(sp, name, isPackage, null, requireInitPy, out mp, out _, out _, out _))) {
return mp;
}
return null;
}

/// <summary>
/// Determines whether the specified directory is an importable package.
/// </summary>
private bool IsPackage(string directory)
=> ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ?
!string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) :
_fs.DirectoryExists(directory);
}
}
26 changes: 16 additions & 10 deletions src/Analysis/Ast/Test/ScrapeTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
using Microsoft.Python.Analysis.Modules.Resolution;
using Microsoft.Python.Analysis.Tests.FluentAssertions;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core;
using Microsoft.Python.Core.IO;
using Microsoft.Python.Parsing;
using Microsoft.Python.Parsing.Ast;
Expand Down Expand Up @@ -78,6 +79,7 @@ private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configura

var services = await CreateServicesAsync(moduleDirectory, configuration);
var interpreter = services.GetService<IPythonInterpreter>();
var fs = services.GetService<IFileSystem>();

// TODO: this is Windows only
var dllsDir = Path.Combine(Path.GetDirectoryName(interpreter.Configuration.LibraryPath), "DLLs");
Expand All @@ -92,7 +94,7 @@ private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configura

var modules = new List<IPythonModule>();

foreach (var pyd in PathUtils.EnumerateFiles(dllsDir, "*", recurse: false).Where(ModulePath.IsPythonFile)) {
foreach (var pyd in PathUtils.EnumerateFiles(fs, dllsDir, "*", recurse: false).Select(f => f.FullName).Where(ModulePath.IsPythonFile)) {
var mp = ModulePath.FromFullPath(pyd);
if (mp.IsDebug) {
continue;
Expand Down Expand Up @@ -270,7 +272,11 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params
var services = await CreateServicesAsync(moduleDirectory, configuration);
var interpreter = services.GetService<IPythonInterpreter>();

var modules = ModulePath.GetModulesInLib(configuration.LibraryPath, configuration.SitePackagesPath).ToList();
var pathResolver = interpreter.ModuleResolution.CurrentPathResolver;
var modules = pathResolver.GetAllModuleNames()
.Select(n => pathResolver.GetModuleImportFromModuleName(n))
.Where(i => i.RootPath.PathEquals(configuration.SitePackagesPath) || i.RootPath.PathEquals(configuration.LibraryPath))
.ToList();

var skip = new HashSet<string>(skipModules);
skip.UnionWith(new[] {
Expand All @@ -294,12 +300,12 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params
}

var set = modules
.Where(m => !skip.Contains(m.ModuleName))
.Where(m => !skip.Contains(m.FullName))
.GroupBy(m => {
var i = m.FullName.IndexOf('.');
return i <= 0 ? m.FullName : m.FullName.Remove(i);
})
.SelectMany(g => g.Select(m => Tuple.Create(m, m.ModuleName)))
.SelectMany(g => g.Select(m => Tuple.Create(m, m.FullName)))
.ToArray();
set = set.Where(x => x.Item2 != null && x.Item2.Contains("grammar")).ToArray();

Expand All @@ -317,22 +323,22 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params
var modName = r.Item1;
var mod = interpreter.ModuleResolution.GetOrLoadModule(r.Item2);

anyExtensionSeen |= modName.IsNativeExtension;
anyExtensionSeen |= modName.IsCompiled;
switch (mod) {
case null:
Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile);
Trace.TraceWarning("failed to import {0} from {1}", modName.FullName, modName.ModulePath);
break;
case CompiledPythonModule compiledPythonModule:
var errors = compiledPythonModule.GetParseErrors().ToArray();
if (errors.Any()) {
anyParseError = true;
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
Trace.TraceError("Parse errors in {0}", modName.ModulePath);
foreach (var e in errors) {
Trace.TraceError(e.Message);
}
} else {
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
anyExtensionSuccess |= modName.IsCompiled;
}

break;
Expand All @@ -343,13 +349,13 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params
if (!mod.FilePath.Contains("site-packages")) {
anyParseError = true;
}
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
Trace.TraceError("Parse errors in {0}", modName.ModulePath);
foreach (var e in filteredErrors) {
Trace.TraceError(e.Message);
}
} else {
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
anyExtensionSuccess |= modName.IsCompiled;
}

break;
Expand Down
2 changes: 1 addition & 1 deletion src/Analysis/Core/Impl/DependencyResolution/IImportSearchResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
namespace Microsoft.Python.Analysis.Core.DependencyResolution {
public interface IImportSearchResult {}

public interface IImportChildrenSource {
public interface IImportChildrenSource : IImportSearchResult {
ImmutableArray<string> GetChildrenNames();
bool TryGetChildImport(string name, out IImportSearchResult child);
}
Expand Down
3 changes: 1 addition & 2 deletions src/Analysis/Core/Impl/DependencyResolution/ImplicitPackageImport.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,10 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.

using System.Collections.Generic;
using Microsoft.Python.Core.Collections;

namespace Microsoft.Python.Analysis.Core.DependencyResolution {
public class ImplicitPackageImport : IImportSearchResult, IImportChildrenSource {
public class ImplicitPackageImport : IImportChildrenSource {
private readonly IImportChildrenSource _childrenSource;

public string Name { get; }
Expand Down
Loading