Mikhail Arkhipov (MikhailArkhipov) · Mar 5, 2019 · Mar 1, 2019 · Mar 1, 2019 · Mar 1, 2019 · Mar 2, 2019
diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IProgressReporter.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IProgressReporter.cs
@@ -0,0 +1,20 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    internal interface IProgressReporter {
+        void ReportRemaining(int count = -1);
+    }
+}
diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs
@@ -15,7 +15,6 @@
 
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Core.Collections;
 using Microsoft.Python.Parsing.Ast;

diff --git a/src/Analysis/Ast/Impl/Analyzer/ProgressReporter.cs b/src/Analysis/Ast/Impl/Analyzer/ProgressReporter.cs
@@ -0,0 +1,95 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Threading;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.Services;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    internal sealed class ProgressReporter : IProgressReporter, IDisposable {
+        private const int _initialDelay = 100;
+        private const int _reportingInterval = 200;
+        private const int _disposeInterval = 2000;
+
+        private readonly IProgressService _progressService;
+        private readonly object _lock = new object();
+        private int _lastReportedCount;
+        private int _lastReportedToClient;
+        private IProgress _progress;
+        private Timer _reportTimer;
+        private Timer _disposeTimer;
+        private bool _running;
+        private int _disposedId;
+
+        public ProgressReporter(IProgressService progressService) {
+            _progressService = progressService;
+        }
+
+        public void Dispose() {
+            lock (_lock) {
+                EndReport();
+            }
+        }
+
+        public void ReportRemaining(int count = -1) {
+            lock (_lock) {
+                if (!_running) {
+                    // Delay reporting a bit in case the analysis is short in order to reduce UI flicker.
+                    _running = true;
+                    _reportTimer?.Dispose();
+                    _reportTimer = new Timer(OnReportTimer, null, _initialDelay, _reportingInterval);
+                }
+
+                _disposeTimer?.Dispose();
+                _disposeTimer = new Timer(OnDisposeTimer, ++_disposedId, _disposeInterval, Timeout.Infinite);
+                if (count >= 0) {
+                    _lastReportedCount = count;
+                }
+            }
+        }
+
+        private void OnReportTimer(object o) {
+            lock (_lock) {
+                if (_running && _lastReportedToClient != _lastReportedCount) {
+                    _lastReportedToClient = _lastReportedCount;
+                    _progress = _progress ?? _progressService.BeginProgress();
+                    if (_lastReportedCount > 0) {
+                        _progress?.Report(Resources.AnalysisProgress.FormatUI(_lastReportedCount)).DoNotWait();
+                    }
+                }
+            }
+        }
+
+        private void OnDisposeTimer(object o) {
+            lock (_lock) {
+                if (_disposedId == (int)o) {
+                    EndReport();
+                }
+            }
+        }
+
+        private void EndReport() {
+            if (_running) {
+                _running = false;
+                _progress?.Dispose();
+                _progress = null;
+
+                _reportTimer?.Dispose();
+                _disposeTimer?.Dispose();
+            }
+        }
+    }
+}
diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs
@@ -16,7 +16,6 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Core.DependencyResolution;
@@ -40,6 +39,7 @@ public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable {
         private readonly object _syncObj = new object();
         private readonly AsyncManualResetEvent _analysisCompleteEvent = new AsyncManualResetEvent();
         private readonly AsyncAutoResetEvent _analysisRunningEvent = new AsyncAutoResetEvent();
+        private readonly ProgressReporter _progress;
         private readonly ILogger _log;
         private readonly int _maxTaskRunning = 8;
         private int _runningTasks;
@@ -48,12 +48,16 @@ public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable {
         public PythonAnalyzer(IServiceManager services) {
             _services = services;
             _log = services.GetService<ILogger>();
+            _progress = new ProgressReporter(services.GetService<IProgressService>());
             _dependencyResolver = new DependencyResolver<ModuleKey, PythonAnalyzerEntry>(new ModuleDependencyFinder());
             _analysisCompleteEvent.Set();
             _analysisRunningEvent.Set();
         }
 
-        public void Dispose() => _disposeToken.TryMarkDisposed();
+        public void Dispose() {
+            _progress.Dispose();
+            _disposeToken.TryMarkDisposed();
+        }
 
         public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default)
             => _analysisCompleteEvent.WaitAsync(cancellationToken);
@@ -72,8 +76,8 @@ public async Task<IDocumentAnalysis> GetAnalysisAsync(IPythonModule module, int
             if (waitTime < 0 || Debugger.IsAttached) {
                 return await GetAnalysisAsync(entry, default, cancellationToken);
             }
-            
-            using (var timeoutCts = new CancellationTokenSource(waitTime)) 
+
+            using (var timeoutCts = new CancellationTokenSource(waitTime))
             using (var cts = CancellationTokenSource.CreateLinkedTokenSource(timeoutCts.Token, cancellationToken)) {
                 cts.CancelAfter(waitTime);
                 var timeoutToken = timeoutCts.Token;
@@ -118,17 +122,17 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray<IPyt
                 if (significantDependencies.Count == 0) {
                     return;
                 }
-                
+
                 entry.Invalidate(_version + 1);
                 entry.AddAnalysisDependencies(significantDependencies);
             }
 
             AnalyzeDocumentAsync(key, entry, default).DoNotWait();
 
-            bool IsSignificant(IPythonModule m) 
+            bool IsSignificant(IPythonModule m)
                 => m != entry.Module && ((m.ModuleType == ModuleType.User && m.Analysis.Version < entry.AnalysisVersion) || m.Analysis is EmptyAnalysis);
         }
-        
+
         public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int bufferVersion, CancellationToken cancellationToken) {
             var key = new ModuleKey(module);
             PythonAnalyzerEntry entry;
@@ -137,7 +141,7 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int
                     if (entry.BufferVersion >= bufferVersion) {
                         return;
                     }
-                    
+
                     entry.Invalidate(ast, bufferVersion, _version + 1);
                 } else {
                     entry = new PythonAnalyzerEntry(module, ast, new EmptyAnalysis(_services, (IDocument)module), bufferVersion, _version + 1);
@@ -151,11 +155,14 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int
         private async Task AnalyzeDocumentAsync(ModuleKey key, PythonAnalyzerEntry entry, CancellationToken cancellationToken) {
             _analysisCompleteEvent.Reset();
             _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued");
+            _progress?.ReportRemaining();
 
             using (var cts = CancellationTokenSource.CreateLinkedTokenSource(_disposeToken.CancellationToken, cancellationToken)) {
                 var analysisToken = cts.Token;
- 
+
                 var walker = await _dependencyResolver.AddChangesAsync(key, entry, cts.Token);
+                _progress?.ReportRemaining(walker.AffectedValues.Count);
+
                 var stopOnVersionChange = true;
                 lock (_syncObj) {
                     if (_version > walker.Version) {
@@ -169,10 +176,12 @@ private async Task AnalyzeDocumentAsync(ModuleKey key, PythonAnalyzerEntry entry
                     LoadMissingDocuments(entry.Module.Interpreter, walker.MissingKeys);
                 }
 
+                _progress?.ReportRemaining(walker.AffectedValues.Count);
                 await _analysisRunningEvent.WaitAsync(cancellationToken);
                 var stopWatch = Stopwatch.StartNew();
 
                 try {
+                    _progress?.ReportRemaining(walker.AffectedValues.Count);
                     lock (_syncObj) {
                         foreach (var affectedEntry in walker.AffectedValues) {
                             affectedEntry.Invalidate(_version);
@@ -187,10 +196,13 @@ private async Task AnalyzeDocumentAsync(ModuleKey key, PythonAnalyzerEntry entry
                     }
 
                     _log?.Log(TraceEventType.Verbose, $"Analysis version {walker.Version} of {walker.AffectedValues.Count} entries has started.");
+                    _progress?.ReportRemaining(walker.Remaining);
                     await AnalyzeAffectedEntriesAsync(walker, stopOnVersionChange, stopWatch, analysisToken);
                 } finally {
                     _analysisRunningEvent.Set();
                     stopWatch.Stop();
+                    _progress?.ReportRemaining(walker.Remaining);
+
                     if (_log != null) {
                         if (walker.Remaining == 0) {
                             _log?.Log(TraceEventType.Verbose, $"Analysis version {walker.Version} has been completed in {stopWatch.Elapsed.TotalMilliseconds} ms.");
@@ -227,21 +239,22 @@ private async Task AnalyzeAffectedEntriesAsync(IDependencyChainWalker<ModuleKey,
                     StartAnalysis(node, walker.Version, stopWatch, cancellationToken);
                 }
             }
-            
+
             if (walker.MissingKeys.Where(k => !k.IsTypeshed).Count == 0) {
                 Interlocked.Exchange(ref _runningTasks, 0);
                 _analysisCompleteEvent.Set();
             }
         }
 
-        private static void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray<ModuleKey> missingKeys) {
+        private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray<ModuleKey> missingKeys) {
             foreach (var (moduleName, _, isTypeshed) in missingKeys) {
                 var moduleResolution = isTypeshed ? interpreter.TypeshedResolution : interpreter.ModuleResolution;
                 moduleResolution.GetOrLoadModule(moduleName);
+                _progress?.ReportRemaining();
             }
         }
 
-        private void StartAnalysis(IDependencyChainNode<PythonAnalyzerEntry> node, int version, Stopwatch stopWatch, CancellationToken cancellationToken) 
+        private void StartAnalysis(IDependencyChainNode<PythonAnalyzerEntry> node, int version, Stopwatch stopWatch, CancellationToken cancellationToken)
             => Task.Run(() => Analyze(node, version, stopWatch, cancellationToken), cancellationToken);
 
         /// <summary>
@@ -299,7 +312,7 @@ public ModuleKey(string name, string filePath, bool isTypeshed) {
                 IsTypeshed = isTypeshed;
             }
 
-            public bool Equals(ModuleKey other) 
+            public bool Equals(ModuleKey other)
                 => Name.EqualsOrdinal(other.Name) && FilePath.PathEquals(other.FilePath) && IsTypeshed == other.IsTypeshed;
 
             public override bool Equals(object obj) => obj is ModuleKey other && Equals(other);
@@ -378,7 +391,7 @@ private static void HandleSearchResults(bool isTypeshed, HashSet<ModuleKey> depe
                 }
             }
 
-            private static bool Ignore(IModuleManagement moduleResolution, string name) 
+            private static bool Ignore(IModuleManagement moduleResolution, string name)
                 => moduleResolution.BuiltinModuleName.EqualsOrdinal(name) || moduleResolution.GetSpecializedModule(name) != null;
         }
     }

diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs
@@ -19,6 +19,7 @@
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Python.Analysis.Analyzer;
 using Microsoft.Python.Core.Collections;
 using Microsoft.Python.Core.Threading;
 
@@ -28,7 +29,6 @@ internal sealed class DependencyResolver<TKey, TValue> : IDependencyResolver<TKe
         private readonly DependencyGraph<TKey, TValue> _vertices = new DependencyGraph<TKey, TValue>();
         private readonly Dictionary<TKey, DependencyVertex<TKey, TValue>> _changedVertices = new Dictionary<TKey, DependencyVertex<TKey, TValue>>();
         private readonly object _syncObj = new object();
-
         public ImmutableArray<TKey> MissingKeys { get; private set; }
 
         public DependencyResolver(IDependencyFinder<TKey, TValue> dependencyFinder) {
@@ -69,6 +69,7 @@ public async Task<IDependencyChainWalker<TKey, TValue>> AddChangesAsync(TKey key
 
             var walkingGraph = CreateWalkingGraph(snapshot, changedVertices);
             var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value);
+
             var loopsCount = FindLoops(walkingGraph);
             var (startingVertices, totalNodesCount) = ResolveLoops(walkingGraph, loopsCount);
             foreach (var vertex in walkingGraph) {
@@ -84,7 +85,7 @@ public async Task<IDependencyChainWalker<TKey, TValue>> AddChangesAsync(TKey key
 
                 MissingKeys = missingKeys;
             }
-            
+
             return new DependencyChainWalker(this, startingVertices, affectedValues, missingKeys, totalNodesCount, version);
         }
 
@@ -274,7 +275,14 @@ public int Remaining {
                 }
             }
 
-            public DependencyChainWalker(in DependencyResolver<TKey, TValue> dependencyResolver, in ImmutableArray<WalkingVertex<TKey, TValue>> startingVertices, in ImmutableArray<TValue> affectedValues, in ImmutableArray<TKey> missingKeys, in int totalNodesCount, in int version) {
+            public DependencyChainWalker(
+                in DependencyResolver<TKey, TValue> dependencyResolver,
+                in ImmutableArray<WalkingVertex<TKey, TValue>> startingVertices,
+                in ImmutableArray<TValue> affectedValues,
+                in ImmutableArray<TKey> missingKeys,
+                in int totalNodesCount,
+                in int version) {
+
                 _syncObj = new object();
                 _dependencyResolver = dependencyResolver;
                 _ppc = new PriorityProducerConsumer<IDependencyChainNode<TValue>>();
@@ -318,7 +326,7 @@ public void MarkCompleted(WalkingVertex<TKey, TValue> vertex, bool commitChanges
                 if (commitChanges && vertex.SecondPass == null) {
                     _dependencyResolver.CommitChanges(vertex.DependencyVertex);
                 }
-                
+
                 if (isCompleted) {
                     _ppc.Produce(null);
                     _ppc.Dispose();

diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs
@@ -15,7 +15,7 @@
 
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Analyzer;
 using Microsoft.Python.Core.Collections;
 
 namespace Microsoft.Python.Analysis.Dependencies {

diff --git a/src/Analysis/Ast/Impl/Resources.Designer.cs b/src/Analysis/Ast/Impl/Resources.Designer.cs
diff --git a/src/Analysis/Ast/Impl/Resources.resx b/src/Analysis/Ast/Impl/Resources.resx
@@ -339,4 +339,7 @@
   <data name="Analysis_UnknownParameterName" xml:space="preserve">
     <value>Unknown parameter name.</value>
   </data>
+  <data name="AnalysisProgress" xml:space="preserve">
+    <value>Analyzing in background, {0} items left...</value>
+  </data>
 </root>
diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs
@@ -328,9 +328,10 @@ public async Task<WorkspaceEdit> Rename(JToken token, CancellationToken cancella
         #endregion
 
         #region Extensions
+
         [JsonRpcMethod("python/loadExtension")]
-        public Task LoadExtension(JToken token, CancellationToken cancellationToken)
-            => _server.LoadExtensionAsync(ToObject<PythonAnalysisExtensionParams>(token), _services, cancellationToken);
+        public Task LoadExtension(JToken token, CancellationToken cancellationToken) => Task.CompletedTask;
+            //=> _server.LoadExtensionAsync(ToObject<PythonAnalysisExtensionParams>(token), _services, cancellationToken);
 
         [JsonRpcMethod("python/extensionCommand")]
         public Task ExtensionCommand(JToken token, CancellationToken cancellationToken)