feat(mirror): refactor sync-mirror.sh for multi-repo, add infra setup script
Wraps the per-repo body in sync_repo() and loops over MIRROR_REPOS at the bottom. teleo-codex stays bidirectional (full PR roundtrip + pipeline.db linking). teleo-infrastructure runs main_only: branch+tag sync Forgejo→ GitHub, ff-only GitHub→Forgejo on main, divergence alerting per-repo. Steps 2.1 (fork PR refs) and 4 (Forgejo PR auto-create + DB link) gated on MODE=bidirectional. Setup script (deploy/setup-infra-mirror.sh) initializes the bare repo at /opt/teleo-eval/mirror/teleo-infrastructure.git, configures remotes, performs initial Forgejo→GitHub push. Idempotent. Pre-flight checks both GitHub repo (must be created manually first — fine-grained PAT can't create repos in the org) and Forgejo repo are accessible. Per-repo divergence state file (.divergence-count.<repo>) so each repo has independent counter + alert state. Also pulls in the source_channel update from Apr 6 that lived only on VPS (line 215 added 'github'). Not deployed yet — pending Ganymede review and GitHub repo creation. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
1351db70a9
commit
bf647b7abb
2 changed files with 384 additions and 186 deletions
116
deploy/setup-infra-mirror.sh
Executable file
116
deploy/setup-infra-mirror.sh
Executable file
|
|
@ -0,0 +1,116 @@
|
|||
#!/bin/bash
|
||||
# One-time setup: prepare the bare mirror repo for teleo-infrastructure.
|
||||
#
|
||||
# Prerequisites (must happen BEFORE running this):
|
||||
# 1. GitHub repo `living-ip/teleo-infrastructure` created (manual via web or
|
||||
# `gh repo create` — the deploy PAT is fine-grained to teleo-codex only
|
||||
# and cannot create new repos in the org).
|
||||
# 2. GitHub PAT updated to include push access on the new repo (or rotate
|
||||
# to a classic PAT with `repo` scope covering both).
|
||||
#
|
||||
# This script is idempotent — safe to re-run.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MIRROR_BASE="/opt/teleo-eval/mirror"
|
||||
REPO_DIR="$MIRROR_BASE/teleo-infrastructure.git"
|
||||
FORGEJO_URL="http://localhost:3000/teleo/teleo-infrastructure.git"
|
||||
GITHUB_REPO="living-ip/teleo-infrastructure"
|
||||
FORGEJO_TOKEN_FILE="/opt/teleo-eval/secrets/forgejo-admin-token"
|
||||
GITHUB_PAT_FILE="/opt/teleo-eval/secrets/github-pat"
|
||||
|
||||
if [ ! -f "$FORGEJO_TOKEN_FILE" ]; then
|
||||
echo "ERROR: missing $FORGEJO_TOKEN_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$GITHUB_PAT_FILE" ]; then
|
||||
echo "ERROR: missing $GITHUB_PAT_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FORGEJO_TOKEN=$(cat "$FORGEJO_TOKEN_FILE" | tr -d '[:space:]')
|
||||
GITHUB_PAT=$(cat "$GITHUB_PAT_FILE" | tr -d '[:space:]')
|
||||
|
||||
# Sanity check: GitHub repo must exist before we point a remote at it.
|
||||
echo "Verifying GitHub repo $GITHUB_REPO exists..."
|
||||
GH_STATUS=$(curl -sS -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: Bearer $GITHUB_PAT" \
|
||||
"https://api.github.com/repos/$GITHUB_REPO")
|
||||
if [ "$GH_STATUS" != "200" ]; then
|
||||
echo "ERROR: GitHub repo $GITHUB_REPO not accessible (HTTP $GH_STATUS)" >&2
|
||||
echo "Create it first: gh repo create $GITHUB_REPO --public --description 'Pipeline + diagnostics infra for the LivingIP collective'" >&2
|
||||
exit 2
|
||||
fi
|
||||
echo " OK — $GITHUB_REPO accessible"
|
||||
|
||||
# Sanity check: Forgejo repo must exist.
|
||||
echo "Verifying Forgejo repo teleo/teleo-infrastructure exists..."
|
||||
FG_STATUS=$(curl -sS -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $FORGEJO_TOKEN" \
|
||||
"http://localhost:3000/api/v1/repos/teleo/teleo-infrastructure")
|
||||
if [ "$FG_STATUS" != "200" ]; then
|
||||
echo "ERROR: Forgejo repo teleo/teleo-infrastructure not accessible (HTTP $FG_STATUS)" >&2
|
||||
exit 3
|
||||
fi
|
||||
echo " OK — Forgejo repo accessible"
|
||||
|
||||
# Init bare mirror if missing
|
||||
if [ -d "$REPO_DIR" ]; then
|
||||
echo "Bare repo already exists at $REPO_DIR — skipping init"
|
||||
else
|
||||
echo "Creating bare repo at $REPO_DIR..."
|
||||
mkdir -p "$REPO_DIR"
|
||||
cd "$REPO_DIR"
|
||||
git init --bare >/dev/null
|
||||
chown -R teleo:teleo "$REPO_DIR"
|
||||
echo " OK — bare repo initialized"
|
||||
fi
|
||||
|
||||
cd "$REPO_DIR"
|
||||
|
||||
# Configure remotes (idempotent: set-url succeeds whether remote exists or not)
|
||||
# Forgejo remote (origin convention is reversed in this codebase: origin=GitHub,
|
||||
# forgejo=Forgejo, matching the existing teleo-codex.git layout).
|
||||
FORGEJO_REMOTE_URL="http://github-mirror:${FORGEJO_TOKEN}@localhost:3000/teleo/teleo-infrastructure.git"
|
||||
GITHUB_REMOTE_URL="https://m3taversal:${GITHUB_PAT}@github.com/${GITHUB_REPO}.git"
|
||||
|
||||
if git remote get-url forgejo >/dev/null 2>&1; then
|
||||
git remote set-url forgejo "$FORGEJO_REMOTE_URL"
|
||||
echo " Updated forgejo remote URL"
|
||||
else
|
||||
git remote add forgejo "$FORGEJO_REMOTE_URL"
|
||||
echo " Added forgejo remote"
|
||||
fi
|
||||
|
||||
if git remote get-url origin >/dev/null 2>&1; then
|
||||
git remote set-url origin "$GITHUB_REMOTE_URL"
|
||||
echo " Updated origin remote URL"
|
||||
else
|
||||
git remote add origin "$GITHUB_REMOTE_URL"
|
||||
echo " Added origin remote"
|
||||
fi
|
||||
|
||||
# Initial fetch from Forgejo
|
||||
echo "Fetching from Forgejo..."
|
||||
git fetch forgejo --prune 2>&1 | sed 's/^/ /'
|
||||
|
||||
# Initial push to GitHub (will populate the empty repo)
|
||||
echo "Pushing initial state to GitHub..."
|
||||
# Sync local refs from forgejo remote refs first (mirrors what sync-mirror.sh does)
|
||||
while read branch; do
|
||||
[ "$branch" = "HEAD" ] && continue
|
||||
git update-ref "refs/heads/$branch" "refs/remotes/forgejo/$branch" 2>/dev/null || true
|
||||
done < <(git for-each-ref --format="%(refname:lstrip=3)" refs/remotes/forgejo/)
|
||||
|
||||
git push origin --all 2>&1 | sed 's/^/ /' || {
|
||||
echo "WARN: initial push failed — you may need to authorize the PAT for $GITHUB_REPO" >&2
|
||||
}
|
||||
git push origin --tags 2>&1 | sed 's/^/ /' || true
|
||||
|
||||
# Final permissions sweep
|
||||
chown -R teleo:teleo "$REPO_DIR"
|
||||
|
||||
echo
|
||||
echo "Setup complete. Verify with:"
|
||||
echo " ssh teleo@77.42.65.182 ls -la $REPO_DIR/refs/heads"
|
||||
echo " /opt/teleo-eval/sync-mirror.sh && tail -50 /opt/teleo-eval/logs/sync.log"
|
||||
|
|
@ -2,22 +2,35 @@
|
|||
# Bidirectional sync: Forgejo (authoritative) <-> GitHub (public mirror)
|
||||
# Forgejo wins on conflict. Runs every 2 minutes via cron.
|
||||
#
|
||||
# Repos handled (see MIRROR_REPOS below):
|
||||
# - teleo-codex (mode=bidirectional): full PR roundtrip — fork PR refs from
|
||||
# GitHub, auto-create Forgejo PR mirrors, link github_pr in pipeline.db.
|
||||
# - teleo-infrastructure (mode=main_only): one-way sync of branches+tags from
|
||||
# Forgejo to GitHub. No PR roundtrip — pipeline doesn't process infra PRs;
|
||||
# external infra PRs land on GitHub for visibility, get reviewed manually.
|
||||
#
|
||||
# Security note: GitHub->Forgejo path is for external contributor convenience.
|
||||
# Never auto-process branches arriving via this path without a PR.
|
||||
# Eval pipeline and extract cron only act on PRs, not raw branches.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_DIR="/opt/teleo-eval/mirror/teleo-codex.git"
|
||||
LOG="/opt/teleo-eval/logs/sync.log"
|
||||
LOCKFILE="/tmp/sync-mirror.lock"
|
||||
PIPELINE_DB="/opt/teleo-eval/pipeline/pipeline.db"
|
||||
GITHUB_PAT_FILE="/opt/teleo-eval/secrets/github-pat"
|
||||
GITHUB_REPO="living-ip/teleo-codex"
|
||||
|
||||
log() { echo "[$(date -Iseconds)] $1" >> "$LOG"; }
|
||||
# (forgejo_owner_repo, github_owner_repo, bare_path, mode)
|
||||
# mode: bidirectional | main_only
|
||||
MIRROR_REPOS=(
|
||||
"teleo/teleo-codex living-ip/teleo-codex /opt/teleo-eval/mirror/teleo-codex.git bidirectional"
|
||||
"teleo/teleo-infrastructure living-ip/teleo-infrastructure /opt/teleo-eval/mirror/teleo-infrastructure.git main_only"
|
||||
)
|
||||
|
||||
# Lockfile — prevent concurrent runs
|
||||
REPO_TAG="main"
|
||||
log() { echo "[$(date -Iseconds)] [$REPO_TAG] $1" >> "$LOG"; }
|
||||
|
||||
# Lockfile — prevent concurrent runs (single lock for whole script)
|
||||
if [ -f "$LOCKFILE" ]; then
|
||||
pid=$(cat "$LOCKFILE" 2>/dev/null)
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
|
|
@ -28,114 +41,155 @@ fi
|
|||
echo $$ > "$LOCKFILE"
|
||||
trap 'rm -f "$LOCKFILE"' EXIT
|
||||
|
||||
# Pre-flight: fix permissions if another user touched the mirror dir (Rhea)
|
||||
BAD_PERMS=$(find "$REPO_DIR" ! -user teleo 2>/dev/null | head -1 || true)
|
||||
if [ -n "$BAD_PERMS" ]; then
|
||||
log "Fixing mirror permissions (found: $BAD_PERMS)"
|
||||
chown -R teleo:teleo "$REPO_DIR" 2>/dev/null
|
||||
fi
|
||||
cd "$REPO_DIR" || { log "ERROR: cannot cd to $REPO_DIR"; exit 1; }
|
||||
|
||||
# Step 1: Fetch from Forgejo (must succeed — it's authoritative)
|
||||
log "Fetching from Forgejo..."
|
||||
if ! git fetch forgejo --prune >> "$LOG" 2>&1; then
|
||||
log "ERROR: Forgejo fetch failed — aborting"
|
||||
exit 1
|
||||
fi
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# sync_repo: process one mirror entry. Sets module-level FORGEJO_REPO,
|
||||
# GITHUB_REPO, REPO_DIR, MODE, REPO_TAG used by inner steps.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
sync_repo() {
|
||||
FORGEJO_REPO="$1" # e.g. teleo/teleo-codex (path on Forgejo)
|
||||
GITHUB_REPO="$2" # e.g. living-ip/teleo-codex (path on GitHub)
|
||||
REPO_DIR="$3" # bare mirror dir
|
||||
MODE="$4" # bidirectional | main_only
|
||||
REPO_TAG="${FORGEJO_REPO##*/}" # short name for log prefix
|
||||
|
||||
# Step 2: Fetch from GitHub (warn on failure, don't abort)
|
||||
log "Fetching from GitHub..."
|
||||
git fetch origin --prune >> "$LOG" 2>&1 || log "WARN: GitHub fetch failed"
|
||||
# Pre-flight: bare repo must exist
|
||||
if [ ! -d "$REPO_DIR" ]; then
|
||||
log "ERROR: bare repo missing at $REPO_DIR — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Step 2.1: Fetch GitHub fork PR refs
|
||||
# Fork-based PRs don't create branches on origin — they create refs/pull/N/head
|
||||
# Fetch these so we can push them to Forgejo for evaluation
|
||||
GITHUB_PAT_STEP2=$(cat "$GITHUB_PAT_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
if [ -n "$GITHUB_PAT_STEP2" ]; then
|
||||
OPEN_PRS=$(curl -sf "https://api.github.com/repos/$GITHUB_REPO/pulls?state=open&per_page=100" \
|
||||
-H "Authorization: token $GITHUB_PAT_STEP2" 2>/dev/null || echo "[]")
|
||||
echo "$OPEN_PRS" | python3 -c "
|
||||
# Pre-flight: fix permissions if another user touched the mirror dir (Rhea)
|
||||
BAD_PERMS=$(find "$REPO_DIR" ! -user teleo 2>/dev/null | head -1 || true)
|
||||
if [ -n "$BAD_PERMS" ]; then
|
||||
log "Fixing mirror permissions (found: $BAD_PERMS)"
|
||||
chown -R teleo:teleo "$REPO_DIR" 2>/dev/null || true
|
||||
fi
|
||||
cd "$REPO_DIR" || { log "ERROR: cannot cd to $REPO_DIR"; return 0; }
|
||||
|
||||
# Step 1: Fetch from Forgejo (must succeed — it's authoritative)
|
||||
log "Fetching from Forgejo..."
|
||||
if ! git fetch forgejo --prune >> "$LOG" 2>&1; then
|
||||
log "ERROR: Forgejo fetch failed — skipping this repo"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Step 2: Fetch from GitHub (warn on failure, don't abort)
|
||||
log "Fetching from GitHub..."
|
||||
git fetch origin --prune >> "$LOG" 2>&1 || log "WARN: GitHub fetch failed"
|
||||
|
||||
# Step 2.1: Fetch GitHub fork PR refs (bidirectional only)
|
||||
# Fork-based PRs don't create branches on origin — they create refs/pull/N/head.
|
||||
# main_only repos don't accept fork PRs through the mirror path.
|
||||
if [ "$MODE" = "bidirectional" ]; then
|
||||
local PAT
|
||||
PAT=$(cat "$GITHUB_PAT_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
if [ -n "$PAT" ]; then
|
||||
local OPEN_PRS
|
||||
OPEN_PRS=$(curl -sf "https://api.github.com/repos/$GITHUB_REPO/pulls?state=open&per_page=100" \
|
||||
-H "Authorization: token $PAT" 2>/dev/null || echo "[]")
|
||||
echo "$OPEN_PRS" | python3 -c "
|
||||
import sys, json
|
||||
prs = json.load(sys.stdin)
|
||||
for pr in prs:
|
||||
head = pr.get('head', {})
|
||||
# Only process fork PRs (repo differs from base repo)
|
||||
base_repo = pr.get('base', {}).get('repo', {}).get('full_name', '')
|
||||
head_repo = head.get('repo', {}) or {}
|
||||
head_full = head_repo.get('full_name', '')
|
||||
if head_full and head_full != base_repo:
|
||||
print(f\"{pr['number']} {head.get('ref', '')} {head.get('sha', '')}\")
|
||||
" 2>/dev/null | while read pr_num branch_name head_sha; do
|
||||
if [ -z "$pr_num" ] || [ -z "$branch_name" ]; then continue; fi
|
||||
PR_BRANCH="gh-pr-${pr_num}/${branch_name}"
|
||||
# Check if we already have this ref at the right SHA
|
||||
EXISTING=$(git rev-parse "refs/heads/$PR_BRANCH" 2>/dev/null || true)
|
||||
if [ "$EXISTING" = "$head_sha" ]; then continue; fi
|
||||
# Fetch the PR ref and create a local branch
|
||||
git fetch origin "refs/pull/${pr_num}/head:refs/heads/$PR_BRANCH" >> "$LOG" 2>&1 && \
|
||||
log "Fetched fork PR #$pr_num -> $PR_BRANCH" || \
|
||||
log "WARN: Failed to fetch fork PR #$pr_num"
|
||||
done
|
||||
fi
|
||||
|
||||
# Step 2.5: GitHub main -> Forgejo main (ff-only)
|
||||
# If a PR was merged on GitHub, GitHub main is ahead of Forgejo main.
|
||||
# Fast-forward Forgejo main to match — safe because ff-only guarantees no divergence.
|
||||
GITHUB_MAIN_FF=$(git rev-parse refs/remotes/origin/main 2>/dev/null || true)
|
||||
FORGEJO_MAIN_FF=$(git rev-parse refs/remotes/forgejo/main 2>/dev/null || true)
|
||||
if [ -n "$GITHUB_MAIN_FF" ] && [ -n "$FORGEJO_MAIN_FF" ]; then
|
||||
if [ "$GITHUB_MAIN_FF" != "$FORGEJO_MAIN_FF" ]; then
|
||||
if git merge-base --is-ancestor "$FORGEJO_MAIN_FF" "$GITHUB_MAIN_FF"; then
|
||||
log "GitHub main ($GITHUB_MAIN_FF) ahead of Forgejo main ($FORGEJO_MAIN_FF) — fast-forwarding"
|
||||
git push forgejo "refs/remotes/origin/main:refs/heads/main" >> "$LOG" 2>&1 && \
|
||||
log "Forgejo main fast-forwarded to $GITHUB_MAIN_FF" || \
|
||||
log "WARN: Failed to fast-forward Forgejo main"
|
||||
if [ -z "$pr_num" ] || [ -z "$branch_name" ]; then continue; fi
|
||||
local PR_BRANCH="gh-pr-${pr_num}/${branch_name}"
|
||||
local EXISTING
|
||||
EXISTING=$(git rev-parse "refs/heads/$PR_BRANCH" 2>/dev/null || true)
|
||||
if [ "$EXISTING" = "$head_sha" ]; then continue; fi
|
||||
git fetch origin "refs/pull/${pr_num}/head:refs/heads/$PR_BRANCH" >> "$LOG" 2>&1 && \
|
||||
log "Fetched fork PR #$pr_num -> $PR_BRANCH" || \
|
||||
log "WARN: Failed to fetch fork PR #$pr_num"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Step 3: Forgejo -> GitHub (primary direction)
|
||||
# Update local refs from Forgejo remote refs using process substitution (avoids subshell)
|
||||
log "Syncing Forgejo -> GitHub..."
|
||||
while read branch; do
|
||||
[ "$branch" = "HEAD" ] && continue
|
||||
git update-ref "refs/heads/$branch" "refs/remotes/forgejo/$branch" 2>/dev/null || \
|
||||
log "WARN: Failed to update ref $branch"
|
||||
done < <(git for-each-ref --format="%(refname:lstrip=3)" refs/remotes/forgejo/)
|
||||
|
||||
# Safety: verify Forgejo main descends from GitHub main before force-pushing
|
||||
GITHUB_MAIN=$(git rev-parse refs/remotes/origin/main 2>/dev/null || true)
|
||||
FORGEJO_MAIN=$(git rev-parse refs/remotes/forgejo/main 2>/dev/null || true)
|
||||
PUSH_MAIN=true
|
||||
if [ -n "$GITHUB_MAIN" ] && [ -n "$FORGEJO_MAIN" ]; then
|
||||
if ! git merge-base --is-ancestor "$GITHUB_MAIN" "$FORGEJO_MAIN"; then
|
||||
log "CRITICAL: Forgejo main is NOT a descendant of GitHub main — skipping main push"
|
||||
log "CRITICAL: GitHub main: $GITHUB_MAIN, Forgejo main: $FORGEJO_MAIN"
|
||||
PUSH_MAIN=false
|
||||
# Step 2.5: GitHub main -> Forgejo main (ff-only)
|
||||
# If a PR was merged on GitHub, GitHub main is ahead of Forgejo main.
|
||||
# Fast-forward Forgejo main to match — safe because ff-only guarantees no divergence.
|
||||
local GITHUB_MAIN_FF FORGEJO_MAIN_FF
|
||||
GITHUB_MAIN_FF=$(git rev-parse refs/remotes/origin/main 2>/dev/null || true)
|
||||
FORGEJO_MAIN_FF=$(git rev-parse refs/remotes/forgejo/main 2>/dev/null || true)
|
||||
if [ -n "$GITHUB_MAIN_FF" ] && [ -n "$FORGEJO_MAIN_FF" ]; then
|
||||
if [ "$GITHUB_MAIN_FF" != "$FORGEJO_MAIN_FF" ]; then
|
||||
if git merge-base --is-ancestor "$FORGEJO_MAIN_FF" "$GITHUB_MAIN_FF"; then
|
||||
log "GitHub main ($GITHUB_MAIN_FF) ahead of Forgejo main ($FORGEJO_MAIN_FF) — fast-forwarding"
|
||||
git push forgejo "refs/remotes/origin/main:refs/heads/main" >> "$LOG" 2>&1 && \
|
||||
log "Forgejo main fast-forwarded to $GITHUB_MAIN_FF" || \
|
||||
log "WARN: Failed to fast-forward Forgejo main"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$PUSH_MAIN" = true ]; then
|
||||
git push origin --all --force >> "$LOG" 2>&1 || log "WARN: Push to GitHub failed"
|
||||
else
|
||||
# Push all branches except main
|
||||
# Step 3: Forgejo -> GitHub (primary direction)
|
||||
log "Syncing Forgejo -> GitHub..."
|
||||
while read branch; do
|
||||
[ "$branch" = "main" ] && continue
|
||||
[ "$branch" = "HEAD" ] && continue
|
||||
git push origin --force "refs/heads/$branch:refs/heads/$branch" >> "$LOG" 2>&1 || \
|
||||
log "WARN: Failed to push $branch to GitHub"
|
||||
done < <(git for-each-ref --format="%(refname:lstrip=2)" refs/heads/)
|
||||
fi
|
||||
git push origin --tags --force >> "$LOG" 2>&1 || log "WARN: Tag push to GitHub failed"
|
||||
git update-ref "refs/heads/$branch" "refs/remotes/forgejo/$branch" 2>/dev/null || \
|
||||
log "WARN: Failed to update ref $branch"
|
||||
done < <(git for-each-ref --format="%(refname:lstrip=3)" refs/remotes/forgejo/)
|
||||
|
||||
# Step 4: GitHub -> Forgejo (external contributions only)
|
||||
# Only push branches that exist on GitHub but NOT on Forgejo
|
||||
log "Checking GitHub-only branches..."
|
||||
GITHUB_ONLY=$(comm -23 \
|
||||
<(git for-each-ref --format="%(refname:lstrip=3)" refs/remotes/origin/ | grep -v HEAD | sort) \
|
||||
<(git for-each-ref --format="%(refname:lstrip=3)" refs/remotes/forgejo/ | grep -v HEAD | sort))
|
||||
# Safety: verify Forgejo main descends from GitHub main before force-pushing
|
||||
local GITHUB_MAIN FORGEJO_MAIN PUSH_MAIN
|
||||
GITHUB_MAIN=$(git rev-parse refs/remotes/origin/main 2>/dev/null || true)
|
||||
FORGEJO_MAIN=$(git rev-parse refs/remotes/forgejo/main 2>/dev/null || true)
|
||||
PUSH_MAIN=true
|
||||
if [ -n "$GITHUB_MAIN" ] && [ -n "$FORGEJO_MAIN" ]; then
|
||||
if ! git merge-base --is-ancestor "$GITHUB_MAIN" "$FORGEJO_MAIN"; then
|
||||
log "CRITICAL: Forgejo main is NOT a descendant of GitHub main — skipping main push"
|
||||
log "CRITICAL: GitHub main: $GITHUB_MAIN, Forgejo main: $FORGEJO_MAIN"
|
||||
PUSH_MAIN=false
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$GITHUB_ONLY" ]; then
|
||||
if [ "$PUSH_MAIN" = true ]; then
|
||||
git push origin --all --force >> "$LOG" 2>&1 || log "WARN: Push to GitHub failed"
|
||||
else
|
||||
# Push all branches except main
|
||||
while read branch; do
|
||||
[ "$branch" = "main" ] && continue
|
||||
[ "$branch" = "HEAD" ] && continue
|
||||
git push origin --force "refs/heads/$branch:refs/heads/$branch" >> "$LOG" 2>&1 || \
|
||||
log "WARN: Failed to push $branch to GitHub"
|
||||
done < <(git for-each-ref --format="%(refname:lstrip=2)" refs/heads/)
|
||||
fi
|
||||
git push origin --tags --force >> "$LOG" 2>&1 || log "WARN: Tag push to GitHub failed"
|
||||
|
||||
# Step 4: GitHub -> Forgejo + Forgejo PR auto-create (bidirectional only)
|
||||
if [ "$MODE" = "bidirectional" ]; then
|
||||
sync_github_to_forgejo_with_prs
|
||||
fi
|
||||
|
||||
# Step 6: Divergence alerting (applies to both modes)
|
||||
check_divergence
|
||||
}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Step 4 split out: codex-specific GitHub→Forgejo branch push + PR auto-create.
|
||||
# Reads FORGEJO_REPO, GITHUB_REPO, PIPELINE_DB, REPO_TAG from sync_repo scope.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
sync_github_to_forgejo_with_prs() {
|
||||
log "Checking GitHub-only branches..."
|
||||
local FORGEJO_HOST="http://localhost:3000/api/v1/repos/$FORGEJO_REPO"
|
||||
local GITHUB_ONLY
|
||||
GITHUB_ONLY=$(comm -23 \
|
||||
<(git for-each-ref --format="%(refname:lstrip=3)" refs/remotes/origin/ | grep -v HEAD | sort) \
|
||||
<(git for-each-ref --format="%(refname:lstrip=3)" refs/remotes/forgejo/ | grep -v HEAD | sort))
|
||||
|
||||
if [ -z "$GITHUB_ONLY" ]; then
|
||||
log "No new GitHub-only branches"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local FORGEJO_TOKEN
|
||||
FORGEJO_TOKEN=$(cat /opt/teleo-eval/secrets/forgejo-admin-token 2>/dev/null)
|
||||
for branch in $GITHUB_ONLY; do
|
||||
log "New from GitHub: $branch -> Forgejo"
|
||||
|
|
@ -151,22 +205,23 @@ if [ -n "$GITHUB_ONLY" ]; then
|
|||
continue
|
||||
}
|
||||
fi
|
||||
# Auto-create PR on Forgejo for mirrored branches (external contributor path)
|
||||
# Skip pipeline-internal branches
|
||||
# Skip pipeline-internal branch prefixes (no PR creation)
|
||||
case "$branch" in
|
||||
extract/*|ingestion/*) continue ;;
|
||||
esac
|
||||
if [ -n "$FORGEJO_TOKEN" ]; then
|
||||
# Check if PR already exists for this branch (open or closed)
|
||||
# NOTE: Forgejo ?head= filter is broken (ignores head value, returns all PRs).
|
||||
# Workaround: fetch open+closed PRs, pipe to Python, check head.ref.
|
||||
HAS_PR=$( {
|
||||
curl -sf "http://localhost:3000/api/v1/repos/teleo/teleo-codex/pulls?state=open&limit=50" \
|
||||
-H "Authorization: token $FORGEJO_TOKEN" 2>/dev/null || echo "[]"
|
||||
echo ""
|
||||
curl -sf "http://localhost:3000/api/v1/repos/teleo/teleo-codex/pulls?state=closed&sort=created&limit=50" \
|
||||
-H "Authorization: token $FORGEJO_TOKEN" 2>/dev/null || echo "[]"
|
||||
} | python3 -c "
|
||||
if [ -z "$FORGEJO_TOKEN" ]; then continue; fi
|
||||
|
||||
# Check if PR already exists for this branch (open or closed)
|
||||
# NOTE: Forgejo ?head= filter is broken (ignores head value, returns all PRs).
|
||||
# Workaround: fetch open+closed PRs, pipe to Python, check head.ref.
|
||||
local HAS_PR
|
||||
HAS_PR=$( {
|
||||
curl -sf "$FORGEJO_HOST/pulls?state=open&limit=50" \
|
||||
-H "Authorization: token $FORGEJO_TOKEN" 2>/dev/null || echo "[]"
|
||||
echo ""
|
||||
curl -sf "$FORGEJO_HOST/pulls?state=closed&sort=created&limit=50" \
|
||||
-H "Authorization: token $FORGEJO_TOKEN" 2>/dev/null || echo "[]"
|
||||
} | python3 -c "
|
||||
import sys, json
|
||||
branch = sys.argv[1]
|
||||
for line in sys.stdin:
|
||||
|
|
@ -179,104 +234,131 @@ for line in sys.stdin:
|
|||
except: pass
|
||||
print('no')
|
||||
" "$branch" 2>/dev/null || echo "no")
|
||||
if [ "$HAS_PR" = "no" ]; then
|
||||
# Build PR title — for fork PRs, use the GitHub PR title
|
||||
if [[ "$branch" == gh-pr-* ]]; then
|
||||
FORK_GH_NUM=$(echo "$branch" | sed 's|gh-pr-\([0-9]*\)/.*|\1|')
|
||||
GITHUB_PAT_T=$(cat "$GITHUB_PAT_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
PR_TITLE=$(curl -sf "https://api.github.com/repos/$GITHUB_REPO/pulls/$FORK_GH_NUM" \
|
||||
-H "Authorization: token $GITHUB_PAT_T" 2>/dev/null | \
|
||||
python3 -c "import sys,json; print(json.load(sys.stdin).get('title',''))" 2>/dev/null || true)
|
||||
[ -z "$PR_TITLE" ] && PR_TITLE=$(echo "$branch" | sed 's|/|: |;s/-/ /g')
|
||||
else
|
||||
PR_TITLE=$(echo "$branch" | sed 's|/|: |;s/-/ /g')
|
||||
fi
|
||||
PAYLOAD=$(python3 -c "import sys,json; print(json.dumps({'title':sys.argv[1],'head':sys.argv[2],'base':'main'}))" "$PR_TITLE" "$branch")
|
||||
RESULT=$(curl -sf -X POST "http://localhost:3000/api/v1/repos/teleo/teleo-codex/pulls" \
|
||||
-H "Authorization: token $FORGEJO_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD" 2>/dev/null || echo "")
|
||||
PR_NUM=$(echo "$RESULT" | grep -o '"number":[0-9]*' | head -1 | grep -o "[0-9]*" || true)
|
||||
if [ -n "$PR_NUM" ]; then
|
||||
log "Auto-created PR #$PR_NUM on Forgejo for $branch"
|
||||
# Step 4.5: Link GitHub PR to Forgejo PR in pipeline DB
|
||||
if [[ "$branch" == gh-pr-* ]]; then
|
||||
GH_PR_NUM=$(echo "$branch" | sed 's|gh-pr-\([0-9]*\)/.*|\1|')
|
||||
else
|
||||
GITHUB_PAT=$(cat "$GITHUB_PAT_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
GH_PR_NUM=""
|
||||
if [ -n "$GITHUB_PAT" ]; then
|
||||
GH_PR_NUM=$(curl -sf "https://api.github.com/repos/$GITHUB_REPO/pulls?head=living-ip:$branch&state=all" \
|
||||
-H "Authorization: token $GITHUB_PAT" 2>/dev/null | \
|
||||
python3 -c "import sys,json; prs=json.load(sys.stdin); print(prs[0]['number'] if prs else '')" 2>/dev/null || true)
|
||||
fi
|
||||
fi
|
||||
if [[ "$GH_PR_NUM" =~ ^[0-9]+$ ]] && [[ "$PR_NUM" =~ ^[0-9]+$ ]]; then
|
||||
sqlite3 "$PIPELINE_DB" "UPDATE prs SET github_pr = $GH_PR_NUM WHERE number = $PR_NUM;" 2>/dev/null && \
|
||||
log "Linked GitHub PR #$GH_PR_NUM -> Forgejo PR #$PR_NUM" || \
|
||||
log "WARN: Failed to link GitHub PR #$GH_PR_NUM to Forgejo PR #$PR_NUM in DB"
|
||||
fi
|
||||
else
|
||||
log "WARN: Failed to auto-create PR for $branch"
|
||||
fi
|
||||
|
||||
if [ "$HAS_PR" = "yes" ]; then continue; fi
|
||||
|
||||
# Build PR title — for fork PRs, use the GitHub PR title
|
||||
local PR_TITLE PAYLOAD RESULT PR_NUM GH_PR_NUM
|
||||
if [[ "$branch" == gh-pr-* ]]; then
|
||||
local FORK_GH_NUM PAT_T
|
||||
FORK_GH_NUM=$(echo "$branch" | sed 's|gh-pr-\([0-9]*\)/.*|\1|')
|
||||
PAT_T=$(cat "$GITHUB_PAT_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
PR_TITLE=$(curl -sf "https://api.github.com/repos/$GITHUB_REPO/pulls/$FORK_GH_NUM" \
|
||||
-H "Authorization: token $PAT_T" 2>/dev/null | \
|
||||
python3 -c "import sys,json; print(json.load(sys.stdin).get('title',''))" 2>/dev/null || true)
|
||||
[ -z "$PR_TITLE" ] && PR_TITLE=$(echo "$branch" | sed 's|/|: |;s/-/ /g')
|
||||
else
|
||||
PR_TITLE=$(echo "$branch" | sed 's|/|: |;s/-/ /g')
|
||||
fi
|
||||
PAYLOAD=$(python3 -c "import sys,json; print(json.dumps({'title':sys.argv[1],'head':sys.argv[2],'base':'main'}))" "$PR_TITLE" "$branch")
|
||||
RESULT=$(curl -sf -X POST "$FORGEJO_HOST/pulls" \
|
||||
-H "Authorization: token $FORGEJO_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD" 2>/dev/null || echo "")
|
||||
PR_NUM=$(echo "$RESULT" | grep -o '"number":[0-9]*' | head -1 | grep -o "[0-9]*" || true)
|
||||
if [ -z "$PR_NUM" ]; then
|
||||
log "WARN: Failed to auto-create PR for $branch"
|
||||
continue
|
||||
fi
|
||||
log "Auto-created PR #$PR_NUM on Forgejo for $branch"
|
||||
|
||||
# Step 4.5: Link GitHub PR to Forgejo PR in pipeline DB
|
||||
if [[ "$branch" == gh-pr-* ]]; then
|
||||
GH_PR_NUM=$(echo "$branch" | sed 's|gh-pr-\([0-9]*\)/.*|\1|')
|
||||
else
|
||||
local PAT
|
||||
PAT=$(cat "$GITHUB_PAT_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
GH_PR_NUM=""
|
||||
if [ -n "$PAT" ]; then
|
||||
GH_PR_NUM=$(curl -sf "https://api.github.com/repos/$GITHUB_REPO/pulls?head=living-ip:$branch&state=all" \
|
||||
-H "Authorization: token $PAT" 2>/dev/null | \
|
||||
python3 -c "import sys,json; prs=json.load(sys.stdin); print(prs[0]['number'] if prs else '')" 2>/dev/null || true)
|
||||
fi
|
||||
fi
|
||||
if [[ "$GH_PR_NUM" =~ ^[0-9]+$ ]] && [[ "$PR_NUM" =~ ^[0-9]+$ ]]; then
|
||||
sqlite3 "$PIPELINE_DB" "UPDATE prs SET github_pr = $GH_PR_NUM, source_channel = 'github' WHERE number = $PR_NUM;" 2>/dev/null && \
|
||||
log "Linked GitHub PR #$GH_PR_NUM -> Forgejo PR #$PR_NUM" || \
|
||||
log "WARN: Failed to link GitHub PR #$GH_PR_NUM to Forgejo PR #$PR_NUM in DB"
|
||||
fi
|
||||
done
|
||||
else
|
||||
log "No new GitHub-only branches"
|
||||
fi
|
||||
}
|
||||
|
||||
# Step 6: Divergence alerting
|
||||
# After all sync steps, check if GitHub and Forgejo main still differ.
|
||||
# 2 consecutive divergent cycles (4 min) triggers a one-shot Telegram alert.
|
||||
DIVERGENCE_FILE="/opt/teleo-eval/logs/.divergence-count"
|
||||
git fetch forgejo main --quiet 2>/dev/null || true
|
||||
git fetch origin main --quiet 2>/dev/null || true
|
||||
GH_MAIN_FINAL=$(git rev-parse refs/remotes/origin/main 2>/dev/null || true)
|
||||
FG_MAIN_FINAL=$(git rev-parse refs/remotes/forgejo/main 2>/dev/null || true)
|
||||
|
||||
if [ -n "$GH_MAIN_FINAL" ] && [ -n "$FG_MAIN_FINAL" ] && [ "$GH_MAIN_FINAL" != "$FG_MAIN_FINAL" ]; then
|
||||
PREV=$(cat "$DIVERGENCE_FILE" 2>/dev/null || echo "0")
|
||||
if [ "$PREV" = "alerted" ]; then
|
||||
log "DIVERGENCE: still diverged (already alerted)"
|
||||
else
|
||||
COUNT=$((PREV + 1))
|
||||
echo "$COUNT" > "$DIVERGENCE_FILE"
|
||||
log "DIVERGENCE: cycle $COUNT — GitHub=$GH_MAIN_FINAL Forgejo=$FG_MAIN_FINAL"
|
||||
if [ "$COUNT" -ge 2 ]; then
|
||||
BOT_TOKEN=$(cat /opt/teleo-eval/secrets/telegram-bot-token 2>/dev/null || true)
|
||||
ADMIN_CHAT=$(cat /opt/teleo-eval/secrets/admin-chat-id 2>/dev/null || true)
|
||||
if [ -n "$BOT_TOKEN" ] && [ -n "$ADMIN_CHAT" ]; then
|
||||
ALERT_MSG=$(python3 -c "
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Step 6 split out: divergence alerting. Per-repo state file so each repo
|
||||
# has its own divergence counter and alert state.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
check_divergence() {
|
||||
local DIVERGENCE_FILE="/opt/teleo-eval/logs/.divergence-count.${REPO_TAG}"
|
||||
git fetch forgejo main --quiet 2>/dev/null || true
|
||||
git fetch origin main --quiet 2>/dev/null || true
|
||||
local GH_MAIN_FINAL FG_MAIN_FINAL
|
||||
GH_MAIN_FINAL=$(git rev-parse refs/remotes/origin/main 2>/dev/null || true)
|
||||
FG_MAIN_FINAL=$(git rev-parse refs/remotes/forgejo/main 2>/dev/null || true)
|
||||
|
||||
if [ -n "$GH_MAIN_FINAL" ] && [ -n "$FG_MAIN_FINAL" ] && [ "$GH_MAIN_FINAL" != "$FG_MAIN_FINAL" ]; then
|
||||
local PREV
|
||||
PREV=$(cat "$DIVERGENCE_FILE" 2>/dev/null || echo "0")
|
||||
if [ "$PREV" = "alerted" ]; then
|
||||
log "DIVERGENCE: still diverged (already alerted)"
|
||||
else
|
||||
local COUNT=$((PREV + 1))
|
||||
echo "$COUNT" > "$DIVERGENCE_FILE"
|
||||
log "DIVERGENCE: cycle $COUNT — GitHub=$GH_MAIN_FINAL Forgejo=$FG_MAIN_FINAL"
|
||||
if [ "$COUNT" -ge 2 ]; then
|
||||
local BOT_TOKEN ADMIN_CHAT
|
||||
BOT_TOKEN=$(cat /opt/teleo-eval/secrets/telegram-bot-token 2>/dev/null || true)
|
||||
ADMIN_CHAT=$(cat /opt/teleo-eval/secrets/admin-chat-id 2>/dev/null || true)
|
||||
if [ -n "$BOT_TOKEN" ] && [ -n "$ADMIN_CHAT" ]; then
|
||||
local ALERT_MSG
|
||||
ALERT_MSG=$(python3 -c "
|
||||
import json, sys
|
||||
msg = '⚠️ Mirror divergence detected\\n\\n'
|
||||
msg = '⚠️ Mirror divergence detected (' + sys.argv[5] + ')\\n\\n'
|
||||
msg += f'GitHub main: {sys.argv[1][:8]}\\n'
|
||||
msg += f'Forgejo main: {sys.argv[2][:8]}\\n'
|
||||
msg += f'Diverged for {sys.argv[3]} consecutive cycles ({int(sys.argv[3])*2} min)\\n\\n'
|
||||
msg += 'Check sync-mirror.sh logs: /opt/teleo-eval/logs/sync.log'
|
||||
print(json.dumps({'chat_id': sys.argv[4], 'text': msg, 'parse_mode': 'HTML'}))
|
||||
" "$GH_MAIN_FINAL" "$FG_MAIN_FINAL" "$COUNT" "$ADMIN_CHAT")
|
||||
if curl -sf -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$ALERT_MSG" >> "$LOG" 2>&1; then
|
||||
log "DIVERGENCE: alert sent to admin"
|
||||
echo "alerted" > "$DIVERGENCE_FILE"
|
||||
" "$GH_MAIN_FINAL" "$FG_MAIN_FINAL" "$COUNT" "$ADMIN_CHAT" "$REPO_TAG")
|
||||
if curl -sf -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$ALERT_MSG" >> "$LOG" 2>&1; then
|
||||
log "DIVERGENCE: alert sent to admin"
|
||||
echo "alerted" > "$DIVERGENCE_FILE"
|
||||
else
|
||||
log "WARN: Failed to send divergence alert (will retry next cycle)"
|
||||
fi
|
||||
else
|
||||
log "WARN: Failed to send divergence alert (will retry next cycle)"
|
||||
log "WARN: Cannot send divergence alert — missing bot token or admin chat ID"
|
||||
fi
|
||||
else
|
||||
log "WARN: Cannot send divergence alert — missing bot token or admin chat ID"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ -f "$DIVERGENCE_FILE" ]; then
|
||||
PREV=$(cat "$DIVERGENCE_FILE" 2>/dev/null || echo "0")
|
||||
if [ "$PREV" != "0" ]; then
|
||||
log "DIVERGENCE: resolved — repos back in sync"
|
||||
else
|
||||
if [ -f "$DIVERGENCE_FILE" ]; then
|
||||
local PREV
|
||||
PREV=$(cat "$DIVERGENCE_FILE" 2>/dev/null || echo "0")
|
||||
if [ "$PREV" != "0" ]; then
|
||||
log "DIVERGENCE: resolved — repos back in sync"
|
||||
fi
|
||||
rm -f "$DIVERGENCE_FILE"
|
||||
fi
|
||||
rm -f "$DIVERGENCE_FILE"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
log "Sync complete"
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Main: process each configured mirror in sequence.
|
||||
# A failure on one repo doesn't block subsequent repos — sync_repo returns 0
|
||||
# on most error paths to keep the loop going.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
REPO_TAG="main"
|
||||
log "Starting sync cycle"
|
||||
|
||||
for entry in "${MIRROR_REPOS[@]}"; do
|
||||
# Read the 4 fields. `read` splits on $IFS (whitespace) by default.
|
||||
read -r forgejo_repo github_repo bare_path mode <<< "$entry"
|
||||
sync_repo "$forgejo_repo" "$github_repo" "$bare_path" "$mode"
|
||||
done
|
||||
|
||||
REPO_TAG="main"
|
||||
log "Sync cycle complete"
|
||||
|
|
|
|||
Loading…
Reference in a new issue