Mikhail Arkhipov (MikhailArkhipov) · Feb 20, 2019 · Feb 16, 2019 · Feb 16, 2019 · Feb 16, 2019 · Feb 18, 2019
diff --git a/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs b/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs
@@ -108,7 +108,9 @@ private bool Save(Node node, bool baseWalk, bool ifTrue) {
                 }
 
                 if (baseWalk && ifTrue) {
-                    Expression = (node is ModuleName m && m.Names != null) ? m.Names.FirstOrDefault() : node;
+                    Expression = node is ModuleName m && m.Names != null
+                        ? m.Names.FirstOrDefault(n => n.StartIndex <= _endLocation && _endLocation <= n.EndIndex)
+                        : node;
                 }
                 return baseWalk;
             }
@@ -264,7 +266,7 @@ public override bool Walk(Arg node) {
             public override bool Walk(ImportStatement node) {
                 if (!base.Walk(node)) {
                     return false;
-                    
+
                 }
 
                 SaveStmt(node, true);
@@ -476,7 +478,7 @@ public override bool Walk(FunctionDefinition node) {
             }
 
 
-            public override bool Walk(IfStatement node) 
+            public override bool Walk(IfStatement node)
                 => base.Walk(node) && Save(node, true, "if");
 
             public override bool Walk(UnaryExpression node) {
@@ -488,7 +490,7 @@ public override bool Walk(UnaryExpression node) {
                 return false;
             }
 
-            public override bool Walk(TryStatement node) 
+            public override bool Walk(TryStatement node)
                 => base.Walk(node) ? Save(node, true, "try") : base.Walk(node);
 
             public override bool Walk(WhileStatement node) {
@@ -506,7 +508,7 @@ public override bool Walk(WithStatement node) {
                     if (node.IsAsync && !Save(node, true, "async")) {
                         return false;
                     }
-                    return Save(node.GetIndexOfWith(_ast), true, "with") && 
+                    return Save(node.GetIndexOfWith(_ast), true, "with") &&
                            node.Items.MaybeEnumerate().All(item => Save(item.AsIndex, true, "as"));
                 }
                 return false;

diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs
@@ -15,57 +15,59 @@
 
 using System;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Core.OS;
 using Microsoft.Python.Parsing;
 using Microsoft.Python.Parsing.Ast;
+using OSPlatform = System.Runtime.InteropServices.OSPlatform;
 
 namespace Microsoft.Python.Analysis.Analyzer.Handlers {
-    internal sealed class ConditionalHandler: StatementHandler {
-        public ConditionalHandler(AnalysisWalker walker) : base(walker) { }
+    internal sealed class ConditionalHandler : StatementHandler {
+        private readonly IOSPlatform _platformService;
+
+        private enum ConditionTestResult {
+            Unrecognized,
+            DontWalkBody,
+            WalkBody
+        }
+
+        public ConditionalHandler(AnalysisWalker walker) : base(walker) {
+            _platformService = Eval.Services.GetService<IOSPlatform>();
+        }
 
         public async Task<bool> HandleIfAsync(IfStatement node, CancellationToken cancellationToken = default) {
-            var allValidComparisons = true;
+            // System version, platform and os.path specializations
+            var someRecognized = false;
             foreach (var test in node.Tests) {
-                if (test.Test is BinaryExpression cmp &&
-                    cmp.Left is MemberExpression me && (me.Target as NameExpression)?.Name == "sys" && me.Name == "version_info" &&
-                    cmp.Right is TupleExpression te && te.Items.All(i => (i as ConstantExpression)?.Value is int)) {
-                    Version v;
-                    try {
-                        v = new Version(
-                            (int)((te.Items.ElementAtOrDefault(0) as ConstantExpression)?.Value ?? 0),
-                            (int)((te.Items.ElementAtOrDefault(1) as ConstantExpression)?.Value ?? 0)
-                        );
-                    } catch (ArgumentException) {
-                        // Unsupported comparison, so walk all children
-                        return true;
+                var result = TryHandleSysVersionInfoAsync(test);
+                if (result != ConditionTestResult.Unrecognized) {
+                    if (result == ConditionTestResult.WalkBody) {
+                        await test.WalkAsync(Walker, cancellationToken);
                     }
+                    someRecognized = true;
+                    continue;
+                }
 
-                    var shouldWalk = false;
-                    switch (cmp.Operator) {
-                        case PythonOperator.LessThan:
-                            shouldWalk = Ast.LanguageVersion.ToVersion() < v;
-                            break;
-                        case PythonOperator.LessThanOrEqual:
-                            shouldWalk = Ast.LanguageVersion.ToVersion() <= v;
-                            break;
-                        case PythonOperator.GreaterThan:
-                            shouldWalk = Ast.LanguageVersion.ToVersion() > v;
-                            break;
-                        case PythonOperator.GreaterThanOrEqual:
-                            shouldWalk = Ast.LanguageVersion.ToVersion() >= v;
-                            break;
+                result = TryHandleSysPlatform(test);
+                if (result != ConditionTestResult.Unrecognized) {
+                    if (result == ConditionTestResult.WalkBody) {
+                        await test.WalkAsync(Walker, cancellationToken);
                     }
+                    someRecognized = true;
+                    continue;
+                }
 
-                    if (shouldWalk) {
-                        // Supported comparison, so only walk the one block
+                result = TryHandleOsPath(test);
+                if (result != ConditionTestResult.Unrecognized) {
+                    if (result == ConditionTestResult.WalkBody) {
                         await test.WalkAsync(Walker, cancellationToken);
-                        return false;
+                        return false; // Execute only one condition.
                     }
-                } else {
-                    allValidComparisons = false;
+                    someRecognized = true;
                 }
             }
 
@@ -86,7 +88,72 @@ public async Task<bool> HandleIfAsync(IfStatement node, CancellationToken cancel
                     }
                 }
             }
-            return !allValidComparisons;
+            return !someRecognized;
+        }
+
+        private ConditionTestResult TryHandleSysVersionInfoAsync(IfStatementTest test) {
+            if (test.Test is BinaryExpression cmp &&
+                cmp.Left is MemberExpression me && (me.Target as NameExpression)?.Name == "sys" && me.Name == "version_info" &&
+                cmp.Right is TupleExpression te && te.Items.All(i => (i as ConstantExpression)?.Value is int)) {
+                Version v;
+                try {
+                    v = new Version(
+                        (int)((te.Items.ElementAtOrDefault(0) as ConstantExpression)?.Value ?? 0),
+                        (int)((te.Items.ElementAtOrDefault(1) as ConstantExpression)?.Value ?? 0)
+                    );
+                } catch (ArgumentException) {
+                    // Unsupported comparison, so walk all children
+                    return ConditionTestResult.WalkBody;
+                }
+
+                var shouldWalk = false;
+                switch (cmp.Operator) {
+                    case PythonOperator.LessThan:
+                        shouldWalk = Ast.LanguageVersion.ToVersion() < v;
+                        break;
+                    case PythonOperator.LessThanOrEqual:
+                        shouldWalk = Ast.LanguageVersion.ToVersion() <= v;
+                        break;
+                    case PythonOperator.GreaterThan:
+                        shouldWalk = Ast.LanguageVersion.ToVersion() > v;
+                        break;
+                    case PythonOperator.GreaterThanOrEqual:
+                        shouldWalk = Ast.LanguageVersion.ToVersion() >= v;
+                        break;
+                }
+                return shouldWalk ? ConditionTestResult.WalkBody : ConditionTestResult.DontWalkBody;
+            }
+            return ConditionTestResult.Unrecognized;
+        }
+
+        private ConditionTestResult TryHandleSysPlatform(IfStatementTest test) {
+            if (test.Test is BinaryExpression cmp &&
+                cmp.Left is MemberExpression me && (me.Target as NameExpression)?.Name == "sys" && me.Name == "platform" &&
+                cmp.Right is ConstantExpression cex && cex.GetStringValue() is string s) {
+                switch (cmp.Operator) {
+                    case PythonOperator.Equals:
+                        return s == "win32" && _platformService.IsWindows ? ConditionTestResult.WalkBody : ConditionTestResult.DontWalkBody;
+                    case PythonOperator.NotEquals:
+                        return s == "win32" && _platformService.IsWindows ? ConditionTestResult.DontWalkBody : ConditionTestResult.WalkBody;
+                }
+                return ConditionTestResult.DontWalkBody;
+            }
+            return ConditionTestResult.Unrecognized;
+        }
+
+        private ConditionTestResult TryHandleOsPath(IfStatementTest test) {
+            if (test.Test is BinaryExpression cmp &&
+                cmp.Left is ConstantExpression cex && cex.GetStringValue() is string s &&
+                cmp.Right is NameExpression nex && nex.Name == "_names") {
+                switch (cmp.Operator) {
+                    case PythonOperator.In when s == "nt":
+                        return _platformService.IsWindows ? ConditionTestResult.WalkBody : ConditionTestResult.DontWalkBody;
+                    case PythonOperator.In when s == "posix":
+                        return _platformService.IsWindows ? ConditionTestResult.DontWalkBody : ConditionTestResult.WalkBody;
+                }
+                return ConditionTestResult.DontWalkBody;
+            }
+            return ConditionTestResult.Unrecognized;
         }
     }
 }
diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs
@@ -46,10 +46,13 @@ public bool HandleFromImport(FromImportStatement node, CancellationToken cancell
                     ImportMembersFromSelf(node);
                     break;
                 case ModuleImport moduleImport:
-                    ImportMembersFromModule(node, moduleImport.FullName, cancellationToken);
+                    ImportMembersFromModule(node, moduleImport.FullName);
                     break;
                 case PossibleModuleImport possibleModuleImport:
-                    HandlePossibleImport(possibleModuleImport, possibleModuleImport.PossibleModuleFullName, Eval.GetLoc(node.Root));
+                    var module = HandlePossibleImport(possibleModuleImport, possibleModuleImport.PossibleModuleFullName, Eval.GetLoc(node.Root));
+                    if (module != null) {
+                        ImportMembersFromModule(node, module.Name);
+                    }
                     break;
                 case PackageImport packageImports:
                     ImportMembersFromPackage(node, packageImports);
@@ -92,27 +95,24 @@ private void ImportMembersFromSelf(FromImportStatement node) {
             }
         }
 
-        private void ImportMembersFromModule(FromImportStatement node, string moduleName, CancellationToken cancellationToken = default) {
+        private void ImportMembersFromModule(FromImportStatement node, string moduleName) {
             var names = node.Names;
             var asNames = node.AsNames;
             var module = ModuleResolution.GetOrLoadModule(moduleName);
             if (module == null) {
                 return;
             }
-
             if (names.Count == 1 && names[0].Name == "*") {
                 // TODO: warn this is not a good style per
                 // TODO: https://docs.python.org/3/faq/programming.html#what-are-the-best-practices-for-using-import-in-a-module
                 // TODO: warn this is invalid if not in the global scope.
-                HandleModuleImportStar(module, cancellationToken);
+                HandleModuleImportStar(module);
                 return;
             }
 
             Eval.DeclareVariable(module.Name, module, VariableSource.Import, node);
 
             for (var i = 0; i < names.Count; i++) {
-                cancellationToken.ThrowIfCancellationRequested();
-
                 var memberName = names[i].Name;
                 if (!string.IsNullOrEmpty(memberName)) {
                     var variableName = asNames[i]?.Name ?? memberName;

diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs
@@ -105,7 +105,6 @@ private IPythonModule HandlePossibleImport(PossibleModuleImport possibleModuleIm
                 }
                 module = childModule;
             }
-
             return module;
         }
 

diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs
@@ -114,8 +114,10 @@ private void MergeStub() {
                         cls.AddMember(name, stubMember, overwrite: true);
                     }
                 } else {
-                    // Re-declare variable with the data from the stub.
-                    if (!stubType.IsUnknown()) {
+                    // Re-declare variable with the data from the stub unless member is a module.
+                    // Modules members that are modules should remain as they are, i.e. os.path
+                    // should remain library with its own stub attached.
+                    if (!stubType.IsUnknown() && !(stubType is IPythonModule)) {
                         sourceType.TransferDocumentation(stubType);
                         // TODO: choose best type between the scrape and the stub. Stub probably should always win.
                         var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration;

diff --git a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs
@@ -18,6 +18,7 @@
 using System.Linq;
 using Microsoft.Python.Analysis.Analyzer.Expressions;
 using Microsoft.Python.Core.Text;
+using Microsoft.Python.Parsing;
 using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis {
@@ -77,5 +78,15 @@ public static bool IsInParameter(this FunctionDefinition fd, PythonAst tree, Sou
                 return index >= p.StartIndex && index <= p.StartIndex + paramName.Length;
             });
         }
+
+        public static string GetStringValue(this ConstantExpression cex) {
+            switch (cex.Value) {
+                case AsciiString asc:
+                    return asc.String;
+                case string s:
+                    return s;
+            }
+            return null;
+        }
     }
 }
diff --git a/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs b/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs
@@ -23,22 +23,32 @@ public static bool IsUnknown(this IMember m) {
             switch (m) {
                 case IPythonType pt when pt.IsUnknown():
                 case IPythonInstance pi when pi.IsUnknown():
+                case IVariable v when v.Value == null || v.Value.IsUnknown():
                 case null:
                     return true;
                 default:
                     return m.MemberType == PythonMemberType.Unknown;
             }
         }
 
-        public static IPythonType GetPythonType(this IMember m)
-            => m is IPythonType pt ? pt : (m as IPythonInstance)?.Type;
+        public static IPythonType GetPythonType(this IMember m) {
+            switch (m) {
+                case IPythonType pt:
+                    return pt;
+                case IPythonInstance pi:
+                    return pi.Type;
+                case IVariable v when v.Value != null:
+                    return v.Value.GetPythonType();
+            }
+            return null;
+        }
 
         public static T GetPythonType<T>(this IMember m) where T : class, IPythonType
-            => m is IPythonType pt ? pt as T : (m as IPythonInstance)?.Type as T;
+            => m.GetPythonType() as T;
 
         public static bool IsGeneric(this IMember m) {
             var t = m.GetPythonType();
-            if(t is IGenericType || t is IGenericTypeParameter) {
+            if (t is IGenericType || t is IGenericTypeParameter) {
                 return true;
             }
             if (t is IPythonClassType c && c.IsGeneric()) {
@@ -47,7 +57,7 @@ public static bool IsGeneric(this IMember m) {
             if (m?.MemberType == PythonMemberType.Generic) {
                 return true;
             }
-            return m is IVariable v && v.Value.MemberType == PythonMemberType.Generic;
+            return m is IVariable v && v.Value?.MemberType == PythonMemberType.Generic;
         }
 
         public static bool TryGetConstant<T>(this IMember m, out T value) {

diff --git a/src/Analysis/Ast/Impl/Modules/ModuleCache.cs b/src/Analysis/Ast/Impl/Modules/ModuleCache.cs
@@ -42,7 +42,7 @@ public ModuleCache(IPythonInterpreter interpreter, IServiceContainer services) {
             _fs = services.GetService<IFileSystem>();
             _log = services.GetService<ILogger>();
             _skipCache = string.IsNullOrEmpty(_interpreter.Configuration.DatabasePath);
-            SearchPathCachePath = Path.Combine(_interpreter.Configuration.DatabasePath, "database.path");
+            SearchPathCachePath = Path.Combine(_interpreter.Configuration.DatabasePath, $"database{_interpreter.Configuration.Version}.path");
         }
 
         public string SearchPathCachePath { get; }

diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs
@@ -369,7 +369,7 @@ private void Parse(CancellationToken cancellationToken) {
 
                 // Do not report issues with libraries or stubs
                 if (sink != null) {
-                    _diagnosticsService?.Replace(Uri, _parseErrors.Concat(Analysis.Diagnostics));
+                    _diagnosticsService?.Replace(Uri, _parseErrors);
                 }
 
                 ContentState = State.Parsed;

diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs
@@ -27,8 +27,8 @@
 using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Core;
-using Microsoft.Python.Core.IO;
 using Microsoft.Python.Core.Diagnostics;
+using Microsoft.Python.Core.IO;
 
 namespace Microsoft.Python.Analysis.Modules.Resolution {
     internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement {
@@ -211,8 +211,5 @@ private bool TryCreateModuleStub(string name, string modulePath, out IPythonModu
             module = !string.IsNullOrEmpty(stubPath) ? new StubPythonModule(name, stubPath, false, _services) : null;
             return module != null;
         }
-
-        protected override void ReportModuleNotFound(string name)
-            => _log?.Log(TraceEventType.Information, $"Import not found: {name}");
     }
 }
diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs
@@ -159,7 +159,5 @@ public IPythonModule GetOrCreate(string name, ModuleResolutionBase mrb) {
                 }
             }
         }
-
-        protected abstract void ReportModuleNotFound(string name);
     }
-}
+}
-Original file line number
+Diff line change
@@ Expand Up @@
                     }
                     module = childModule;
                 }
                 return module;
             }
@@ Expand Down @@