maxim-lobanov · Apr 24, 2020 · Apr 23, 2020 · Apr 23, 2020 · Apr 23, 2020 · Apr 23, 2020
diff --git a/azure-pipelines/templates/build-job.yml b/azure-pipelines/templates/build-job.yml
@@ -13,17 +13,10 @@ jobs:
     inputs:
       targetType: filePath
       filePath: './builders/build-python.ps1'
-      arguments: '-Version $(VERSION) -Architecture $(Architecture) -Platform $(Platform)'
-
-  - task: ArchiveFiles@2
-    inputs:
-      rootFolderOrFile: '$(Build.BinariesDirectory)'
-      archiveType: zip
-      includeRootFolder: false
-      archiveFile: '$(Build.ArtifactStagingDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).zip'
+      arguments: '-Version $(VERSION) -Platform $(Platform) -Architecture $(Architecture)'
 
   - task: PublishPipelineArtifact@1
     displayName: 'Publish Artifact: Python $(VERSION)'
     inputs:
-      targetPath: '$(Build.ArtifactStagingDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).zip' 
+      targetPath: '$(Build.ArtifactStagingDirectory)' 
       artifactName: 'python-$(VERSION)-$(Platform)-$(Architecture)'
diff --git a/azure-pipelines/templates/test-job.yml b/azure-pipelines/templates/test-job.yml
@@ -7,4 +7,74 @@ jobs:
   - checkout: self
     submodules: true
 
-  - template: test-steps.yml
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: 'current'
+      artifact: 'python-$(VERSION)-$(Platform)-$(Architecture)'
+      path: $(Build.BinariesDirectory)
+
+  - task: ExtractFiles@1
+    inputs:
+      archiveFilePatterns: '$(Build.BinariesDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).*'
+      destinationFolder: $(Build.BinariesDirectory)
+      cleanDestinationFolder: false
+
+  - task: PowerShell@2
+    displayName: 'Apply build artifact to the local machines'
+    inputs:
+      TargetType: inline
+      script: |
+        if ($env:PLATFORM -match 'windows') { powershell ./setup.ps1 } else { sh ./setup.sh }
+      workingDirectory: '$(Build.BinariesDirectory)'
+
+  - task: UsePythonVersion@0
+    displayName: 'Use Python $(VERSION)'
+    inputs:
+      versionSpec: '$(VERSION)'
+      architecture: '$(Architecture)'
+
+  - task: PowerShell@2
+    displayName: 'Verbose sysconfig dump'
+    inputs:
+      TargetType: inline
+      script: |
+        Invoke-Expression "python ./sources/python-config-output.py"
+      workingDirectory: '$(Build.SourcesDirectory)/tests'
+    condition: ne(variables['Platform'], 'windows-2016')
+
+  - task: PowerShell@2
+    displayName: 'Verbose python binary links'
+    inputs:
+      TargetType: inline
+      script: |
+        $pythonLocation = which python
+        if ($env:PLATFORM -match 'macos') { otool -L $pythonLocation } else { ldd $pythonLocation }
+      workingDirectory: '$(Build.BinariesDirectory)'
+    condition: ne(variables['Platform'], 'windows-2016')
+
+  - task: PowerShell@2
+    displayName: 'Run tests'
+    inputs:
+      TargetType: inline
+      script: |
+        Install-Module Pester -Force -Scope CurrentUser
+        Import-Module Pester
+        $pesterParams = @{
+          Path="./python-tests.ps1";
+          Parameters=@{
+            Version="$(VERSION)";
+            Platform="$(Platform)";
+          }
+        }
+        Invoke-Pester -Script $pesterParams -OutputFile "test_results.xml" -OutputFormat NUnitXml
+      workingDirectory: '$(Build.SourcesDirectory)/tests'
+
+  - task: PublishTestResults@2
+    displayName: 'Publish test results'
+    inputs:
+      testResultsFiles: '*.xml'
+      testResultsFormat: NUnit
+      searchFolder: 'tests'
+      failTaskOnFailedTests: true
+      testRunTitle: "$(TestRunTitle)"
+    condition: always()
diff --git a/azure-pipelines/templates/test-steps.yml b/azure-pipelines/templates/test-steps.yml
diff --git a/builders/build-python.ps1 b/builders/build-python.ps1
@@ -21,15 +21,14 @@ Required parameter. The platform for which Python will be built.
 #>
 
 param(
-    [Parameter (Mandatory=$true)]
-    [version] $Version,
-    [string] $Architecture = "x64",
-    [Parameter (Mandatory=$true)]
-    [string] $Platform
+    [Parameter (Mandatory=$true)][Version] $Version,
+    [Parameter (Mandatory=$true)][string] $Platform,
+    [string] $Architecture = "x64"
 )
 
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "common-helpers.psm1") -DisableNameChecking
 Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "nix-helpers.psm1") -DisableNameChecking
+Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "win-helpers.psm1") -DisableNameChecking
 
 function Get-PythonBuilder {
     <#
@@ -58,11 +57,11 @@ function Get-PythonBuilder {
 
     $Platform = $Platform.ToLower()  
     if ($Platform -match 'windows') {
-        $builder = [WinPythonBuilder]::New($Version, $Architecture)
+        $builder = [WinPythonBuilder]::New($Version, $Architecture, $Platform)
     } elseif ($Platform -match 'ubuntu') {
-        $builder = [UbuntuPythonBuilder]::New($Platform, $Version)
+        $builder = [UbuntuPythonBuilder]::New($Version, $Architecture, $Platform)
     } elseif ($Platform -match 'macos') {
-        $builder = [macOSPythonBuilder]::New($Platform, $Version)
+        $builder = [macOSPythonBuilder]::New($Version, $Architecture, $Platform)
     } else {
         Write-Host "##vso[task.logissue type=error;] Invalid platform: $Platform"
         exit 1
@@ -72,5 +71,5 @@ function Get-PythonBuilder {
 }
 
 ### Create Python builder instance, and build artifact
-$Builder = Get-PythonBuilder -Version $Version -Platform $Platform -Architecture $Architecture
+$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture  -Platform $Platform 
 $Builder.Build()
diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1
@@ -17,11 +17,10 @@ class macOSPythonBuilder : NixPythonBuilder {
     #>
 
     macOSPythonBuilder(
-        [string] $platform,
-        [version] $version
-    ) : Base($platform, $version) {
-
-    }
+        [version] $version,
+        [string] $architecture,
+        [string] $platform
+    ) : Base($version, $architecture, $platform) { }
 
     [void] Configure() {
         <#

diff --git a/builders/nix-python-builder.psm1 b/builders/nix-python-builder.psm1
@@ -28,22 +28,19 @@ class NixPythonBuilder : PythonBuilder {
 
     #>
 
-    [string] $Platform
-    [string] $PlatformVersion
     [string] $InstallationTemplateName
     [string] $InstallationScriptName
     [string] $OutputArtifactName
 
     NixPythonBuilder(
-        [string] $platform,
-        [version] $version
-    ) : Base($version, "x64") {
-        $this.Platform = $platform.Split("-")[0]
-        $this.PlatformVersion = $platform.Split("-")[1]
+        [version] $version,
+        [string] $architecture,
+        [string] $platform
+    ) : Base($version, $architecture, $platform) {
         $this.InstallationTemplateName = "nix-setup-template.sh"	
         $this.InstallationScriptName = "setup.sh"
 
-        $this.OutputArtifactName = "tool.zip"
+        $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz"
     }
 
     [uri] GetSourceUri() {
@@ -64,7 +61,6 @@ class NixPythonBuilder : PythonBuilder {
         #>
 
         if ($this.Version.Major -eq 2) { $pythonBinary = "python" } else { $pythonBinary = "python3" }
-
         return $pythonBinary
     }
 
@@ -77,36 +73,23 @@ class NixPythonBuilder : PythonBuilder {
         $sourceUri = $this.GetSourceUri()
         Write-Host "Sources URI: $sourceUri"
 
-        $archiveFilepath = Download-File -Uri $sourceUri -OutputFolder $this.ArtifactLocation
-        Unpack-TarArchive -ArchivePath $archiveFilepath -OutputDirectory $this.TempFolderLocation
-        $expandedSourceLocation = Join-Path -Path $this.TempFolderLocation -ChildPath "Python-$($this.Version)"
+        $archiveFilepath = Download-File -Uri $sourceUri -OutputFolder $this.WorkFolderLocation
+        $expandedSourceLocation = Join-Path -Path $this.TempFolderLocation -ChildPath "SourceCode"
+        New-Item -Path $expandedSourceLocation -ItemType Directory
+
+        Extract-TarArchive -ArchivePath $archiveFilepath -OutputDirectory $expandedSourceLocation
         Write-Debug "Done; Sources location: $expandedSourceLocation"
 
         return $expandedSourceLocation
     }
 
-    [void] ArchiveArtifact([string] $pythonToolLocation) {
-        <#
-        .SYNOPSIS
-        Create .zip archive with Python binaries.
-
-        .PARAMETER pythonToolLocation
-        The location of Python binaries that's need to be archived.
-
-        #>
-
-        $artifact = Join-Path -Path $this.ArtifactLocation -ChildPath $this.OutputArtifactName
-        Pack-Zip -PathToArchive $pythonToolLocation -ToolZipFile $artifact 
-        Write-Debug "Done; Artifact location: $artifact"
-    }
-
     [void] CreateInstallationScript() {
         <#
         .SYNOPSIS
         Create Python artifact installation script based on template specified in InstallationTemplateName property.
         #>
 
-        $installationScriptLocation = New-Item -Path $this.ArtifactLocation -Name $this.InstallationScriptName -ItemType File
+        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File
         $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName
 
         $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
@@ -122,20 +105,30 @@ class NixPythonBuilder : PythonBuilder {
         Executes "make" and "make install" commands for configured build sources. Make output will be writen in build_output.txt located in artifact location folder.
         #>
 
-        Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)-$($this.PlatformVersion)"
-        $buildOutputLocation = New-Item -Path $this.ArtifactLocation -Name "build_output.txt" -ItemType File
+        Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)"
+        $buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File
 
         # Fix error "find: build": build dir not exist before first compilation
         New-Item -ItemType Directory -Path ./build
 
         # execute "make" with error action = continue for python 2.* for ubuntu, because it throws errors with some modules
-        $makeErrorAction = if ($this.Version.Major -eq 2 -and $this.Platform -eq "ubuntu") { "Continue" } else { "Stop" }
+        $makeErrorAction = if ($this.Version.Major -eq 2 -and $this.Platform -match "ubuntu") { "Continue" } else { "Stop" }
         Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction $makeErrorAction
         Execute-Command -Command "make install" -ErrorAction $makeErrorAction
 
         Write-Debug "Done; Make log location: $buildOutputLocation"
     }
 
+    [void] CopyBuildResults() {
+        $buildFolder = $this.GetFullPythonToolcacheLocation()
+        Move-Item -Path "$buildFolder/*" -Destination $this.WorkFolderLocation
+    }
+
+    [void] ArchiveArtifact() {
+        $OutputPath = Join-Path $this.ArtifactFolderLocation $this.OutputArtifactName
+        Create-TarArchive -SourceFolder $this.WorkFolderLocation -ArchivePath $OutputPath
+    }
+
     [void] Build() {
         <#
         .SYNOPSIS
@@ -152,19 +145,23 @@ class NixPythonBuilder : PythonBuilder {
         $sourcesLocation = $this.Download()
 
         Push-Location -Path $sourcesLocation
-        Write-Host "Configure for $($this.Platform)-$($this.PlatformVersion)..."
+        Write-Host "Configure for $($this.Platform)..."
         $this.Configure()
 
-        Write-Host "Make for $($this.Platform)-$($this.PlatformVersion)..."
+        Write-Host "Make for $($this.Platform)..."
         $this.Make()
         Pop-Location
 
-        New-ToolStructureDump -ToolPath $this.GetFullPythonToolcacheLocation() -OutputFolder $this.ArtifactLocation
+        Write-Host "Generate structure dump"
+        New-ToolStructureDump -ToolPath $this.GetFullPythonToolcacheLocation() -OutputFolder $this.WorkFolderLocation
 
-        Write-Host "Archive generated artifact..."
-        $this.ArchiveArtifact($this.GetFullPythonToolcacheLocation())
+        Write-Host "Copying build results to destination location"
+        $this.CopyBuildResults()
 
         Write-Host "Create installation script..."
         $this.CreateInstallationScript()
+
+        Write-Host "Archive artifact..."
+        $this.ArchiveArtifact()
     }
 }