Repository navigation
166 lines (160 loc) · 6.96 KB
/
Copy pathdocker.yml
File metadata and controls
166 lines (160 loc) · 6.96 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
name: Build Docker & cloud images
on:
workflow_dispatch:
inputs:
image_version:
description: "Docker image version"
required: true
staging:
description: "Staging build"
type: boolean
default: false
build_docker:
description: "Build docker images"
type: boolean
default: true
build_aws:
description: "Build AWS images"
type: boolean
default: true
build_azure:
description: "Build Azure images"
type: boolean
default: true
build_gcp:
description: "Build GCP images"
type: boolean
default: true
env:
PACKER_VERSION: "1.9.2"
BUILD_PREFIX: ${{ inputs.staging && format('stgn-{0}-', github.run_number) || '' }} # staging ? prefix : ''
jobs:
build-docker:
if: inputs.build_docker
defaults:
run:
working-directory: docker
runs-on: ubuntu-latest
strategy:
matrix:
python: [ "3.8", "3.9", "3.10", "3.11" ]
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Build and upload to DockerHub
run: |
docker buildx build --platform linux/arm64/v8 --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-arm64 -f cpu/Dockerfile .
docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-amd64 -f cpu/Dockerfile .
docker manifest create dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }} --amend dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-arm64 --amend dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-amd64
docker manifest push dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}
docker buildx build --platform linux/amd64 --build-arg PYTHON=${{ matrix.python }} --push --provenance=false --tag dstackai/base:py${{ matrix.python }}-${{ inputs.image_version }}-cuda-11.8 -f cuda/Dockerfile .
build-aws-images:
needs: build-docker
if: always() && inputs.build_aws && (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped')
defaults:
run:
working-directory: runner
runs-on: ubuntu-latest
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
strategy:
matrix:
variant: [ "", "-cuda" ]
steps:
- uses: actions/checkout@v3
- name: Download packer
run: |
wget https://releases.hashicorp.com/packer/${{ env.PACKER_VERSION }}/packer_${{ env.PACKER_VERSION }}_linux_amd64.zip
unzip packer_${{ env.PACKER_VERSION }}_linux_amd64.zip
chmod +x packer
cp -R ami/packer/* .
- name: Run packer
run: |
./packer build -var-file=versions.json $PROD_VARS -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX aws-image${{ matrix.variant }}.json
env:
PROD_VARS: ${{ !inputs.staging && '-var-file=aws-vars-prod.json' || '' }} # production ? var-file : ''
build-azure-images:
needs: build-docker
if: always() && inputs.build_azure && (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped')
defaults:
run:
working-directory: runner
runs-on: ubuntu-latest
env:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
VERSION: ${{ github.run_number }}
strategy:
matrix:
variant: [ "", "-cuda" ]
steps:
- uses: actions/checkout@v3
- uses: Azure/login@v1
name: Log in to az
with:
creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}'
- name: Download packer
run: |
wget https://releases.hashicorp.com/packer/${{ env.PACKER_VERSION }}/packer_${{ env.PACKER_VERSION }}_linux_amd64.zip
unzip packer_${{ env.PACKER_VERSION }}_linux_amd64.zip
chmod +x packer
cp -R ami/packer/* .
- name: Run packer
run: |
./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX azure-image${{ matrix.variant }}.json
- name: Publish azure image
if: ${{ !inputs.staging }}
run: |
IMAGE_DEFINITION=${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
IMAGE_NAME=${BUILD_PREFIX}dstack${{ matrix.variant }}-${{ inputs.image_version }}
../scripts/publish_azure_image.sh $IMAGE_DEFINITION $IMAGE_NAME
build-gcp-images:
needs: build-docker
if: always() && inputs.build_gcp && (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped')
defaults:
run:
working-directory: runner
runs-on: ubuntu-latest
strategy:
matrix:
variant: [ "", "-cuda" ]
permissions:
contents: 'read'
id-token: 'write'
steps:
- uses: actions/checkout@v3
- name: 'Authenticate to Google Cloud'
uses: 'google-github-actions/auth@v1'
with:
workload_identity_provider: 'projects/531508670106/locations/global/workloadIdentityPools/github-identity-pool/providers/github-id-provider'
service_account: '[email protected]'
create_credentials_file: true
- name: 'Set up Cloud SDK'
uses: 'google-github-actions/setup-gcloud@v1'
- name: Download packer
run: |
wget https://releases.hashicorp.com/packer/${{ env.PACKER_VERSION }}/packer_${{ env.PACKER_VERSION }}_linux_amd64.zip
unzip packer_${{ env.PACKER_VERSION }}_linux_amd64.zip
chmod +x packer
cp -R ami/packer/* .
- name: Run packer
run: |
./packer build -var-file=versions.json -var image_version=${{ inputs.image_version }} -var build_prefix=$BUILD_PREFIX gcp-image${{ matrix.variant }}.json
- name: Publish images
run: |
IMAGE_VERSION=${IMAGE_VERSION//./-}
gcloud compute images add-iam-policy-binding ${BUILD_PREFIX}dstack${{ matrix.variant }}-$IMAGE_VERSION --member='allAuthenticatedUsers' --role='roles/compute.imageUser'
env:
IMAGE_VERSION: ${{ inputs.image_version }}