github · ewgenius · Jan 31, 2020 · Jan 30, 2020 · Jan 30, 2020 · Jan 30, 2020
diff --git a/.travis.yml b/.travis.yml
@@ -9,6 +9,7 @@ matrix:
         - brew install moreutils
         - brew install shellcheck
         - brew install jq
+        - brew install coreutils
       script: make test
     - os: linux
       dist: trusty
@@ -24,4 +25,5 @@ matrix:
             - moreutils
             - fakeroot
             - jq
+            - coreutils
       script: debuild -uc -us
diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config
@@ -304,3 +304,29 @@ ghe_debug() {
 version() {
   echo "${@#v}" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }';
 }
+
+# The list of gists returned by the source changed in 2.16.23, 2.17.14,
+# 2.18.8, and 2.19.3.  We need to account for this difference here.
+# In older versions, all paths need to be truncated with `dirname`.
+# In newer versions, gist paths are unmodified, and only other repo types
+# are truncated with `dirname`.
+fix_paths_for_ghe_version() {
+  if [[ "$GHE_REMOTE_VERSION" =~ 2.16. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.16.23)" ]] || \
+     [[ "$GHE_REMOTE_VERSION" =~ 2.17. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.17.14)" ]] || \
+     [[ "$GHE_REMOTE_VERSION" =~ 2.18. && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.18.8)" ]] || \
+     [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.19.3)" ]]; then
+    GIST_FILTER="-e /gist/b"
+  else
+    unset GIST_FILTER
+  fi
+
+  # This sed expression is equivalent to running `dirname` on each line,
+  # but without all the fork+exec overhead of calling `dirname` that many
+  # times:
+  #   1. strip off trailing slashes
+  #   2. if the result has no slashes in it, the dirname is "."
+  #   3. truncate from the final slash (if any) to the end
+  # If the GIST_FILTER was set above (because we're on a modern version of
+  # GHES), then don't modify lines with "gist" in them.
+  sed $GIST_FILTER -e 's/\/$//; s/^[^\/]*$/./; s/\/[^\/]*$//'
+}
diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories
@@ -366,25 +366,8 @@ bm_end "$(basename $0) - Special Data Directories Sync"
 
 if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
   bm_start "$(basename $0) - Verifying Routes"
-
-  # The list of gists returned by the source changed in 2.16.23, 2.17.14, 2.18.8 & 2.19.3
-  # so we need to account for this difference here.
-  parse_paths() {
-    while read -r line; do
-      if [[ "$GHE_REMOTE_VERSION" =~ 2.16 && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.16.23)" ]] || \
-         [[ "$GHE_REMOTE_VERSION" =~ 2.17 && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.17.14)" ]] || \
-         [[ "$GHE_REMOTE_VERSION" =~ 2.18 && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.18.8)" ]] || \
-         [[ "$GHE_REMOTE_VERSION" =~ 2.19 && "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.19.3)" ]] && \
-         (echo "$line" | grep -q "gist"); then
-        echo "$line"
-      else
-        dirname "$line"
-      fi
-    done
-  }
-
-  cat $tempdir/*.rsync | sort | uniq > $tempdir/source_routes
-  (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | parse_paths | sort | uniq) > $tempdir/destination_routes
+  cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes
+  (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes
 
   git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
 

diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage
@@ -141,8 +141,8 @@ bm_end "$(basename $0) - Storage object sync"
 if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
   bm_start "$(basename $0) - Verifying Routes"
 
-  cat $tempdir/*.rsync | sort | uniq > $tempdir/source_routes
-  (cd $backup_dir/ && find * -mindepth 3 -maxdepth 3 -type f -print | sort | uniq) > $tempdir/destination_routes
+  cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes
+  (cd $backup_dir/ && find * -mindepth 3 -maxdepth 3 -type f -print | uniq | sort | uniq) > $tempdir/destination_routes
 
   git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
 

diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh
@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
 # ghe-backup command tests
 
+TESTS_DIR="$PWD/$(dirname "$0")"
 # Bring in testlib
 # shellcheck source=test/testlib.sh
-. "$(dirname "$0")/testlib.sh"
+. "$TESTS_DIR/testlib.sh"
 
 # Create the backup data dir and fake remote repositories dirs
 mkdir -p "$GHE_DATA_DIR" "$GHE_REMOTE_DATA_USER_DIR"
@@ -344,3 +345,68 @@ begin_test "ghe-backup missing directories or files on source appliance"
     verify_all_backedup_data
 )
 end_test
+
+# acceptance criteria is less then 2 seconds for 100,000 lines
+begin_test "ghe-backup fix_paths_for_ghe_version performance tests - gists"
+(
+    set -e
+    timeout 2 bash -c "
+        source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+        GHE_REMOTE_VERSION=2.16.23
+        seq 1 100000 | sed -e 's/$/ gist/' | fix_paths_for_ghe_version | grep -c gist
+    "
+)
+end_test
+
+# acceptance criteria is less then 2 seconds for 100,000 lines
+begin_test "ghe-backup fix_paths_for_ghe_version performance tests - wikis"
+(
+    set -e
+    timeout 2 bash -c "
+        source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+        GHE_REMOTE_VERSION=2.16.23
+        seq 1 100000 | sed -e 's/$/ wiki/' | fix_paths_for_ghe_version | grep -c '^\.$'
+    "
+)
+end_test
+
+# check fix_paths_for_ghe_version version thresholds
+begin_test "ghe-backup fix_paths_for_ghe_version newer/older"
+(
+    set -e
+
+    # modern versions keep foo/gist as foo/gist
+    for ver in 2.16.23 v2.16.23 v2.17.14 v2.18.8 v2.19.3 v2.20.0 v3.0.0; do
+        echo "## $ver, not gist"
+        [ "$(bash -c "
+            source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+            GHE_REMOTE_VERSION=$ver
+            echo foo/bar | fix_paths_for_ghe_version
+        ")" == "foo" ]
+
+        echo "## $ver, gist"
+        [ "$(bash -c "
+            source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+            GHE_REMOTE_VERSION=$ver
+            echo foo/gist | fix_paths_for_ghe_version
+        ")" == "foo/gist" ]
+    done
+
+    # old versions change foo/gist to foo
+    for ver in 1.0.0 bob a.b.c "" 1.2.16 2.0.0 v2.0.0 v2.15.123 v2.16.22 v2.17.13 v2.18.7 v2.19.2; do
+        echo "## $ver, not gist"
+        [ "$(bash -c "
+            source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+            GHE_REMOTE_VERSION=$ver
+            echo foo/bar | fix_paths_for_ghe_version
+        ")" == "foo" ]
+
+        echo "## $ver, gist"
+        [ "$(bash -c "
+            source '$TESTS_DIR/../share/github-backup-utils/ghe-backup-config'
+            GHE_REMOTE_VERSION=$ver
+            echo foo/gist | fix_paths_for_ghe_version
+        ")" == "foo" ]
+    done
+)
+end_test