github · thejandroman · Dec 16, 2019 · Dec 16, 2019 · Dec 16, 2019 · Dec 18, 2019
diff --git a/bin/ghe-backup b/bin/ghe-backup
@@ -221,8 +221,13 @@ ghe-backup-audit-log || failures="$failures audit-log"
 echo "Backing up hookshot logs ..."
 ghe-backup-es-hookshot || failures="$failures hookshot"
 
-echo "Backing up Git repositories ..."
-ghe-backup-repositories || failures="$failures repositories"
+if should_use_gitbackups_for_repositories && can_use_gitbackups_for_repositories; then
+  echo "Backing up Git repositories using gitbackups ..."
+  ghe-backup-repositories-gitbackups || failures="$failures repositories"
+else
+  echo "Backing up Git repositories using rsync ..."
+  ghe-backup-repositories-rsync || failures="$failures repositories"
+fi
 
 echo "Backing up GitHub Pages ..."
 ghe-backup-pages || failures="$failures pages"

diff --git a/bin/ghe-host-check b/bin/ghe-host-check
@@ -113,7 +113,7 @@ fi
 
 # backup-utils 2.13 onwards limits support to the current and previous two releases
 # of GitHub Enterprise Server.
-supported_minimum_version="2.17.0"
+supported_minimum_version="2.18.0"
 
 if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then
   supported=1

diff --git a/debian/changelog b/debian/changelog
@@ -4,6 +4,13 @@ github-backup-utils (2.19.4) UNRELEASED; urgency=medium
 
  -- Hao Jiang <[email protected]>  Tue, 18 Feb 2020 17:54:31 +0000
 
+github-backup-utils (2.20.0) UNRELEASED; urgency=medium
+
+  * Fix `ghe-backup-repositories` performance for large instances #541
+  * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551
+
+ -- Alejandro Figueroa <[email protected]>  Tue, 11 Feb 2020 20:47:17 +0000
+
 github-backup-utils (2.19.3) UNRELEASED; urgency=medium
 
   * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551

diff --git a/script/cibuild b/script/cibuild
@@ -6,7 +6,7 @@ set -e
 # place with this version at the beginning of each test and many commands have
 # conditional logic based on the remote version. Running the suite against
 # different major versions ensures we're covering these conditional paths.
-REMOTE_VERSIONS="2.17.0 2.19.0"
+REMOTE_VERSIONS="2.18.0 2.20.0"
 
 # Enable verbose logging of ssh commands
 export GHE_VERBOSE_SSH=true

diff --git a/script/release b/script/release
@@ -31,6 +31,7 @@ GH_REPO = ENV['GH_REPO'] || 'backup-utils'
 GH_OWNER = ENV['GH_OWNER'] || 'github'
 GH_AUTHOR = ENV['GH_AUTHOR']
 DEB_PKG_NAME = 'github-backup-utils'
+GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master'
 
 CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium
 
@@ -136,7 +137,7 @@ def beautify_changes(changes)
 end
 
 def changelog
-  changes = `git log --pretty=oneline origin/stable...origin/master --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip)
+  changes = `git log --pretty=oneline origin/stable...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip)
   raise 'Building the changelog failed' if $CHILD_STATUS != 0
 
   changes
@@ -232,8 +233,8 @@ end
 
 def update_stable_branch
   `git checkout --quiet stable`
-  unless (out = `git merge --quiet --ff-only origin/master`)
-    warn "Merging master into stable failed:\n\n#{out}"
+  unless (out = `git merge --quiet --ff-only origin/#{GH_BASE_BRANCH}`)
+    warn "Merging #{GH_BASE_BRANCH} into stable failed:\n\n#{out}"
   end
   unless (out = `git push --quiet origin stable`)
     warn "Failed pushing the stable branch:\n\n#{out}"
@@ -245,7 +246,7 @@ def create_release_pr(version, release_body)
     'title': "Bump version: #{version}",
     'body': release_body,
     'head': "release-#{version}",
-    'base': 'master'
+    'base': GH_BASE_BRANCH
   }.to_json
   res = post("/repos/#{GH_OWNER}/#{GH_REPO}/pulls", body)
   raise "Creating release PR failed (#{res.code})" unless res.is_a? Net::HTTPSuccess
@@ -328,9 +329,9 @@ def attach_assets_to_release(upload_url, release_id, files)
 end
 
 def clean_up(version)
-  `git checkout --quiet master`
+  `git checkout --quiet #{GH_BASE_BRANCH}`
   `git fetch --quiet origin --prune`
-  `git pull --quiet origin master --prune`
+  `git pull --quiet origin #{GH_BASE_BRANCH} --prune`
   `git branch --quiet -D release-#{version} >/dev/null 2>&1`
   `git push --quiet origin :release-#{version} >/dev/null 2>&1`
   `git branch --quiet -D tmp-packging >/dev/null 2>&1`
@@ -382,6 +383,7 @@ if $PROGRAM_NAME == __FILE__
       puts "Repo: #{GH_REPO}"
       puts "Author: #{GH_AUTHOR}"
       puts "Token: #{ENV['GH_RELEASE_TOKEN'] && 'set' || 'unset'}"
+      puts "Base branch: #{GH_BASE_BRANCH}"
       puts 'Changelog:'
       if release_changes.empty?
         puts ' => No new bug fixes, enhancements or features.'
@@ -429,7 +431,7 @@ if $PROGRAM_NAME == __FILE__
 
     puts 'Creating release...'
     release_title = "GitHub Enterprise Server Backup Utilities v#{version}"
-    res = create_release "v#{version}", 'master', release_title, release_body, true
+    res = create_release "v#{version}", GH_BASE_BRANCH, release_title, release_body, true
 
     # Tidy up before building tarball and deb pkg
     clean_up version

diff --git a/share/github-backup-utils/ghe-backup-audit-log b/share/github-backup-utils/ghe-backup-audit-log
@@ -29,6 +29,11 @@ is_skip_truncate_enabled(){
   ghe-ssh "$host" test -e "$GHE_REMOTE_DATA_USER_DIR/common/audit-log-import/skip_truncate"
 }
 
+# Check whether the audit_entries table is deleted
+audit_entries_deleted(){
+   [ -z "$(ghe-ssh "$host" -- "/usr/local/share/enterprise/github-mysql 'SHOW TABLES LIKE \"audit_entries\"'")" ]
+}
+
 is_binary_backup(){
   ghe-ssh "$host" ghe-config --true "mysql.backup.binary"
 }
@@ -38,7 +43,10 @@ backup_mysql(){
     ghe_verbose "Skip backup audit_entries for Mysql since it is using binary backup"
     return
   fi
-  if is_skip_truncate_enabled; then
+
+  if audit_entries_deleted; then
+    ghe_verbose "audit_entries table does not exist"
+  elif is_skip_truncate_enabled; then
     # As skip_truncate exists, we need to also backup the audit entries
     # in MySQL because Elasticsearch may not be fully synced.
     "${base_path}/ghe-backup-mysql-audit-log"

diff --git a/share/github-backup-utils/ghe-backup-repositories-gitbackups b/share/github-backup-utils/ghe-backup-repositories-gitbackups
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-repositories-gitbackups
+#/ Take an online, incremental snapshot of all Git repository, wiki, and gist data using gitbackups.
+#/
+#/ Note: This command typically isn't called directly. It's invoked by
+#/ ghe-backup.
+set -e
+
+# Bring in the backup configuration
+# shellcheck source=share/github-backup-utils/ghe-backup-config
+. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config"
+
+bm_start "$(basename $0)"
+
+echo "github-env ./bin/ghe-backup-repositories" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
+
+bm_end "$(basename $0)"
diff --git a/...thub-backup-utils/ghe-backup-repositories → ...ackup-utils/ghe-backup-repositories-rsync b/...thub-backup-utils/ghe-backup-repositories → ...ackup-utils/ghe-backup-repositories-rsync
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
-#/ Usage: ghe-backup-repositories
-#/ Take an online, incremental snapshot of all Git repository data.
+#/ Usage: ghe-backup-repositories-rsync
+#/ Take an online, incremental snapshot of all Git repository data using rsync.
 #/
 #/ Note: This command typically isn't called directly. It's invoked by
 #/ ghe-backup.

diff --git a/share/github-backup-utils/ghe-restore-audit-log b/share/github-backup-utils/ghe-restore-audit-log
@@ -46,8 +46,9 @@ is_mysql_supported(){
      'test \"\$ENTERPRISE_AUDIT_LOG_MYSQL_LOGGER_ENABLED\" = \"1\""' | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
 }
 
+# Check if the backup is binary by looking up the sentinel file 
 is_binary_backup(){
-  ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary"
+  test -f "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"/mysql-binary-backup-sentinel 
 }
 
 # Helper function to set remote flags in `/data/user/common/audit-log-import`
@@ -90,6 +91,11 @@ restore_mysql(){
 
   ghe_verbose "Restoring MySQL audit logs ..."
 
+  if ! mysql_table_schema_available; then
+    ghe_verbose "schema.gz does not exist"
+    return
+  fi
+
   "${base_path}/ghe-restore-mysql-audit-log" "$GHE_HOSTNAME" "$only_schema"
 }
 

diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version
@@ -1 +1 @@
-2.19.4
+2.20.0
diff --git a/test/testlib.sh b/test/testlib.sh
@@ -42,7 +42,7 @@ export GHE_BACKUP_CONFIG GHE_DATA_DIR GHE_REMOTE_DATA_DIR GHE_REMOTE_ROOT_DIR
 
 # The default remote appliance version. This may be set in the environment prior
 # to invoking tests to emulate a different remote vm version.
-: ${GHE_TEST_REMOTE_VERSION:=2.19.0}
+: ${GHE_TEST_REMOTE_VERSION:=2.20.0}
 export GHE_TEST_REMOTE_VERSION
 
 # Source in the backup config and set GHE_REMOTE_XXX variables based on the