Split logic into separate scripts.

Refactor for performance when lots of branches. 
Make more robust in various situations where one of the remotes might be missing or disconnected.

.wvs/root-ci.yml

@@ -27,75 +27,93 @@ ProvisionRunner_github:
 githubsync:
   stage: test
   variables:
-    GIT_STRATEGY: none
-    CI_DEBUG_TRACE: "true"
+    #GIT_STRATEGY: none
+    CI_DEBUG_TRACE: "false"
   tags:
     - ubuntu22-p4sync
     - $CI_PROJECT_ID
   script:
     - |
-      set +e
+      set -e
       RET=0
       REPO=${GITHUB_SYNC_SOURCE##*/}
       REPO_DIR=${REPO%.*}
       echo "${REPO}"
+      STARTDIR=`pwd`
+      
 
-      echo "Checking if ${REPO_DIR} exists..."
-      if [ -d /data/${REPO_DIR}/.git -o -f /data/${REPO_DIR}/HEAD ]; then
-        echo "Updating existing repo"
-        cd /data/${REPO_DIR}
-      else
-        echo "New Repo"
-        cd /data
-        git clone  ${GITHUB_SYNC_SOURCE} /data/${REPO_DIR}
-        RET=$(( $RET + $? ))
-        cd /data/${REPO_DIR}
-      fi
-
-      if git remote get-url wvs; then
-        git remote remove wvs
-        RET=$(( $RET + $? ))
-      fi
+      chmod +x ${STARTDIR}/migrateAllLfs.sh
+      chmod +x ${STARTDIR}/checkRemote.sh
 
-      git remote add wvs https://${WVS_PAT_USER}:${WVS_PAT}@wvs.io/${WVS_SYNC_TARGET}
-      RET=$(( $RET + $? ))
+      # Check the status of the repository's origin
+      ${STARTDIR}/checkRemote.sh "/data/${REPO_DIR}" origin
+      ORIGIN_STATUS=$?
 
-      git remote -v
+      # Many of the cases are identical except for the status print, however it is
+      # anticipated that they may differentiate over time.
+      case $ORIGIN_STATUS in
+        0)  echo "Origin exists and is accessible. Updating existing repo."
+            cd "/data/${REPO_DIR}"
+            ;;
+        1)  echo "Specified directory does not exist. Cloning new repo."
+            cd /data
+            git clone ${GITHUB_SYNC_SOURCE} "/data/${REPO_DIR}"
+            RET=$(( $RET + $? ))
+            cd /data/${REPO_DIR}
+            ;;
+        2)  echo "Specified directory is not a git repository. Cloning new repo."
+            cd /data
+            rm -rf "${REPO_DIR}" # Remove the non-git directory
+            git clone ${GITHUB_SYNC_SOURCE} "/data/${REPO_DIR}"
+            RET=$(( $RET + $? ))
+            cd /data/${REPO_DIR}
+            ;;
+        3)  echo "No origin is configured for this repository. Cloning new repo."
+            cd /data
+            rm -rf "${REPO_DIR}" # Remove the existing repository
+            git clone ${GITHUB_SYNC_SOURCE} "/data/${REPO_DIR}"
+            RET=$(( $RET + $? ))
+            cd /data/${REPO_DIR}
+            ;;
+        4)  echo "Origin specified but does not exist or is not accessible. Cloning new repo."
+            cd /data
+            rm -rf "${REPO_DIR}" # Remove the existing repository
+            git clone ${GITHUB_SYNC_SOURCE} "/data/${REPO_DIR}"
+            RET=$(( $RET + $? ))
+            cd /data/${REPO_DIR}
+            ;;
+      esac
 
-      git fetch --all
-      git lfs fetch --all
-      
-      for branch in  $(git branch --list --remote "origin/*" | grep -v --regexp='->'); do
+      # If the previous invocation crashed or was cancelled, it might have left a lock file
+      if [ -d .git ] && [ -f .git/index.lock ]; then
+        echo "Removing stale index.lock"
+        rm .git/index.lock
+      fi  
 
-        if git rev-parse --verify "$branch" &>/dev/null; then
-            # The branch already exists, so just check it out
-            echo "checking out existing local branch $branch"
-            git checkout --force "$branch"
-        else
-            # The branch doesn't exist, so create it and set it to track the remote branch
-            echo "checking out new local branch $branch"
-            git checkout --force -b "$branch" "origin/$branch"
-        fi
+      set +e
+      if git remote get-url wvsrepo; then
+        git remote remove wvsrepo
+        RET=$(( $RET + $? ))
+      fi
+      set -e
 
-        # Check if there are unstaged changes
-        if [[ -n $(git status -s) ]]; then
-            echo "ERROR There are unstaged changes on branch $branch"
-            RET=$(( $RET + 1 ))
-        else
-            echo "No unstaged changes."
-            git pull
-            git lfs fetch --all
-        fi
+      git remote add wvsrepo https://${WVS_PAT_USER}:${WVS_PAT}@wvs.io/${WVS_SYNC_TARGET}
+      RET=$(( $RET + $? ))
 
-        git push wvs "$branch:refs/heads/${branch//origin\/}"
-        RET=$(( $RET + $? ))
-      done
+      # Check the status of wvremote
+      set +e
+      ${STARTDIR}/checkRemote.sh "/data/${REPO_DIR}" wvsrepo
+      if [ $? -ne 0 ]; then
+        echo "wvsrepo does not exist. Creating new repository on wvs"
+        git push -u wvsrepo
+      fi
+      set -e
 
-      echo "pushing all LFS"
-      git lfs push --all
 
+      ${STARTDIR}/migrateAllLfs.sh
+      
       echo "pushing tags"
-      git push wvs 'refs/tags/*:refs/tags/*'
+      git push wvsrepo 'refs/tags/*:refs/tags/*'
       RET=$(( $RET + $? ))
 
       exit $RET

checkRemote.sh

+#!/bin/sh
+
+# When you have a local cache of a wvs repository, you can use this script to
+# check if the remote is accessible. There are 5 possible exit codes:
+# 0 - Remote exists and is accessible
+# 1 - The specified directory does not exist
+# 2 - The specified directory is not a git repository
+# 3 - No remote is configured for this repository
+# 4 - Remote specified but does not exist or is not accessible
+
+set +e
+
+# Set REPO_DIR variable to the directory where your repository is located.
+# It can be passed as an argument: REPO_DIR="$1"
+REPO_DIR="$1"
+REMOTE_TO_CHECK="$2"
+
+# Navigate to the repository directory
+cd "$REPO_DIR" || exit 1
+
+# Check if it's a git repository
+if [ -d .git ] || git rev-parse --git-dir > /dev/null 2>&1; then
+    # Check if an remote is configured
+    git ls-remote $REMOTE_TO_CHECK &> /dev/null
+    if [ $? -eq 0 ]; then
+        # Try fetching from remote and check the exit status
+        git fetch $REMOTE_TO_CHECK &> /dev/null
+        if [ $? -eq 0 ]; then
+            exit 0 # Remote exists and is accessible
+        else
+            exit 4 # Remote does not exist or is not accessible
+        fi
+    else
+        exit 3 # No remote is configured for this repository
+    fi
+else
+    exit 2 # The specified directory is not a git repository
+fi

migrateAllLfs.sh

+#!/bin/sh
+
+# Exit script immediately on any errors
+set -e
+
+# Function to check and normalize modified LFS files
+normalize_lfs_files() {
+    # Flag to indicate whether any LFS files were renormalized
+    renormalized_files=false
+    
+    # Get the list of modified files
+    modified_files=$(git status --porcelain | awk '{if ($1 == "M") print $2}')
+    
+    # Loop over each modified file to check if it is managed by LFS
+    for file in $modified_files; do
+        is_lfs_file=$(git check-attr filter $file | grep "filter: lfs")
+        
+        # If the file is managed by LFS, then renormalize it
+        if [ ! -z "$is_lfs_file" ]; then
+            echo "git add $file --renormalize"
+            git add $file --renormalize
+            
+            # Mark that LFS files have been renormalized
+            renormalized_files=true
+        fi
+    done
+    
+    # If any LFS files were renormalized, commit the changes
+    if $renormalized_files ; then
+        echo "git commit -m 'Renormalized LFS files'"
+        git commit -m "Renormalized LFS files"
+    fi
+}
+
+# Remember the original branch to return to it later
+original_branch=$(git branch --show-current)
+
+# Fetch all branches from origin
+echo "git fetch origin"
+git fetch origin
+
+# set git data necessary to commit
+git config user.email $GITLAB_USER_EMAIL
+git config user.name $GITLAB_USER_NAME
+git config checkout.defaultRemote origin
+
+# Get the list of all branches on origin
+branches=$(git branch -r | grep "origin/" | sed 's/origin\///' | grep -v HEAD)
+
+# Loop over all branches from origin
+for branch in $branches; do
+
+    echo "************************************************************"
+    echo "Checking branch $branch"
+    echo "************************************************************"
+
+
+    # Check if the remote branch is ahead of the local branch
+    local_commit=$(git rev-parse $branch)
+    remote_commit=$(git rev-parse origin/$branch)
+    base_commit=$(git merge-base $branch origin/$branch)
+
+    if [ "$remote_commit" = "$base_commit" ]; then
+        echo "Local branch $branch is up-to-date or ahead of origin/$branch. Skipping."
+        continue
+    fi
+
+    # Checkout each branch locally with --force option
+    echo "git checkout --force $branch"
+    git checkout --force $branch
+
+    # Checkout each branch locally with --force option
+    echo "git pull"
+    git pull
+ 
+    # Fetch all LFS objects for the current branch
+    echo "git lfs fetch --all"
+    git lfs fetch --all
+    
+    # Normalize LFS files if there are any modifications
+    normalize_lfs_files
+    
+    # Push all fetched LFS objects to the wvsrepo remote
+    echo "git lfs push wvsrepo --all"
+    git lfs push wvsrepo --all
+    
+    # Push any changes (including LFS pointer changes) for the current branch to the wvsrepo remote
+    echo "git push wvsrepo $branch"
+    git push wvsrepo $branch
+done
+
+# Return to the original branch
+echo "git checkout --force $original_branch"
+git checkout --force $original_branch
+
+# Print completion message
+echo "LFS objects from all branches on 'origin' have been fetched, renormalized, and pushed to 'wvsrepo'."

wvs-sync.yml

-name: WVSTrigger
-
-# Controls when the workflow will run
-on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
-  push:
-  pull_request:
-    branches: [ "main", "wvs-sync" ]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
-  # This workflow contains a single job called "build"
-  trigger:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Trigger WVS Pipeline
-        env:
-          WVS_TRIGGER_TOKEN: ${{ secrets.WVS_TRIGGER_TOKEN }}
-          WVS_PROJECT_ID: ${{ secrets.WVS_PROJECT_ID }}
-        run: |
-          echo "Triggering pipeline at https://wvs.io/api/v4/projects/${WVS_PROJECT_ID}/trigger/pipeline"
-          curl -D - -XPOST --fail -F token=${WVS_TRIGGER_TOKEN} -F ref=main https://wvs.io/api/v4/projects/${WVS_PROJECT_ID}/trigger/pipeline