commit c508efcd93ad51211c30d16eaf7e49d2d97e4443 Author: al Date: Wed Aug 26 10:35:54 2026 +0000 Add xh1-research-vv2 diff --git a/xh1-research-vv2 b/xh1-research-vv2 new file mode 100644 index 0000000..fc53d49 --- /dev/null +++ b/xh1-research-vv2 @@ -0,0 +1,1347 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================ +# XH-1 RESEARCH HARNESS +# +# Multi-provider / multi-model autonomous research system. +# +# Supports any OpenAI-compatible Chat Completions API: +# +# OpenRouter +# Ollama +# LM Studio +# vLLM +# LocalAI +# OpenAI +# Other compatible providers +# +# Pipeline: +# +# RESEARCH +# ↓ +# REVIEW +# ↓ +# PASS ───────────────→ COMMIT +# │ +# FAIL +# ↓ +# REVISION +# ↓ +# REVIEW +# ↓ +# ... +# +# ============================================================ + +ROOT="${XH1_RESEARCH_ROOT:-research}" +CONFIG="${XH1_RESEARCH_CONFIG:-xh1-research.conf}" +STATE="${ROOT}/.xh1" + +mkdir -p \ + "$STATE/logs" \ + "$STATE/runs" \ + "$STATE/responses" + +# ============================================================ +# Defaults +# ============================================================ + +MAX_RESEARCH_ROUNDS="${MAX_RESEARCH_ROUNDS:-3}" +MAX_API_RETRIES="${MAX_API_RETRIES:-3}" +DELAY_SECONDS="${DELAY_SECONDS:-5}" +MAX_ITERATIONS="${MAX_ITERATIONS:-0}" + +REVIEW_ENABLED="${REVIEW_ENABLED:-true}" +AUTO_COMMIT="${AUTO_COMMIT:-false}" + +RUN_ID="" + +# ============================================================ +# Helpers +# ============================================================ + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +log() { + echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*" | + tee -a "$STATE/logs/harness.log" +} + +require() { + command -v "$1" >/dev/null 2>&1 || + die "Required command not found: $1" +} + +require_dependencies() { + require bash + require curl + require jq + require find + require grep + require sed + require awk +} + +timestamp() { + date -u '+%Y%m%dT%H%M%SZ' +} + +load_config() { + if [[ -f "$CONFIG" ]]; then + # shellcheck disable=SC1090 + source "$CONFIG" + fi +} + +# ============================================================ +# Provider abstraction +# ============================================================ + +# +# Provider variables are defined in xh1-research.conf. +# +# Example: +# +# PROVIDER_openrouter_URL="https://openrouter.ai/api/v1" +# PROVIDER_openrouter_KEY_ENV="OPENROUTER_API_KEY" +# +# PROVIDER_ollama_URL="http://127.0.0.1:11434/v1" +# PROVIDER_ollama_KEY_ENV="" +# +# PROVIDER_openai_URL="https://api.openai.com/v1" +# PROVIDER_openai_KEY_ENV="OPENAI_API_KEY" +# + +provider_url() { + local provider="$1" + + local variable="PROVIDER_${provider}_URL" + + printf '%s' "${!variable:-}" +} + +provider_key_env() { + local provider="$1" + + local variable="PROVIDER_${provider}_KEY_ENV" + + printf '%s' "${!variable:-}" +} + +provider_api_key() { + local provider="$1" + + local key_env + key_env="$(provider_key_env "$provider")" + + # Local providers such as Ollama normally require no key. + if [[ -z "$key_env" ]]; then + printf '' + return 0 + fi + + local key="${!key_env:-}" + + [[ -n "$key" ]] || + die "Provider '$provider' requires environment variable '$key_env'." + + printf '%s' "$key" +} + +provider_check() { + local provider="$1" + + local url + url="$(provider_url "$provider")" + + [[ -n "$url" ]] || + die "Provider '$provider' has no URL." + + local key + key="$(provider_api_key "$provider")" + + local -a headers + + headers=( + -H "Content-Type: application/json" + ) + + if [[ -n "$key" ]]; then + headers+=( + -H "Authorization: Bearer $key" + ) + fi + + curl \ + --fail \ + --silent \ + --show-error \ + --connect-timeout 10 \ + --max-time 20 \ + "${headers[@]}" \ + "${url%/}/models" \ + >/dev/null +} + +# ============================================================ +# API call +# ============================================================ + +api_call() { + + local provider="$1" + local model="$2" + local system_prompt="$3" + local user_prompt="$4" + local output="$5" + + local url + url="$(provider_url "$provider")" + + [[ -n "$url" ]] || + die "Unknown provider: $provider" + + local key + key="$(provider_api_key "$provider")" + + local payload + + payload="$( + jq -n \ + --arg model "$model" \ + --arg system "$system_prompt" \ + --arg user "$user_prompt" \ + '{ + model: $model, + messages: [ + { + role: "system", + content: $system + }, + { + role: "user", + content: $user + } + ], + temperature: 0.2 + }' + )" + + local attempt=1 + + while (( attempt <= MAX_API_RETRIES )); do + + local json="${output}.json" + + log "API: provider=$provider model=$model attempt=$attempt" + + local -a headers + + headers=( + -H "Content-Type: application/json" + ) + + if [[ -n "$key" ]]; then + headers+=( + -H "Authorization: Bearer $key" + ) + fi + + # OpenRouter-specific attribution headers. + if [[ "$provider" == "openrouter" ]]; then + headers+=( + -H "HTTP-Referer: https://github.com/riscvcxh1/xh1-research" + -H "X-Title: XH-1 Research Harness" + ) + fi + + if curl \ + --fail \ + --silent \ + --show-error \ + --connect-timeout 30 \ + --max-time 1800 \ + "${headers[@]}" \ + "${url%/}/chat/completions" \ + -d "$payload" \ + > "$json" + then + + if jq -er \ + '.choices[0].message.content' \ + "$json" \ + > "$output" + then + + log "Response received: $(wc -c < "$output" | tr -d ' ') bytes" + + return 0 + fi + + log "Provider returned an invalid response." + else + log "API request failed." + fi + + sleep $((attempt * attempt)) + + ((attempt++)) + done + + return 1 +} + +# ============================================================ +# Run state +# ============================================================ + +start_run() { + + RUN_ID="$(timestamp)" + + mkdir -p "$STATE/runs/$RUN_ID" + + : > "$STATE/runs/$RUN_ID/completed" + : > "$STATE/runs/$RUN_ID/failed" + : > "$STATE/runs/$RUN_ID/attempts" + + log "Started run: $RUN_ID" +} + +run_has_completed() { + grep -Fxq "$1" \ + "$STATE/runs/$RUN_ID/completed" \ + 2>/dev/null +} + +run_has_failed() { + grep -Fxq "$1" \ + "$STATE/runs/$RUN_ID/failed" \ + 2>/dev/null +} + +mark_completed() { + printf '%s\n' "$1" \ + >> "$STATE/runs/$RUN_ID/completed" +} + +mark_failed() { + printf '%s\n' "$1" \ + >> "$STATE/runs/$RUN_ID/failed" +} + +record_attempt() { + + printf '%s\t%s\t%s\t%s\t%s\n' \ + "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ + "$1" \ + "$2" \ + "$3" \ + "$4" \ + >> "$STATE/runs/$RUN_ID/attempts" +} + +# ============================================================ +# Research discovery +# ============================================================ + +is_soon() { + grep -qE '^SOON[[:space:]]*$' "$1" +} + +find_next_task() { + + while IFS= read -r -d '' file; do + + [[ "$file" == "$STATE"/* ]] && continue + + if is_soon "$file" && + ! run_has_completed "$file" && + ! run_has_failed "$file" + then + printf '%s\n' "$file" + return 0 + fi + + done < <( + find "$ROOT" \ + -type f \ + -name '*.md' \ + -not -path "$STATE/*" \ + -print0 + ) + + return 1 +} + +# ============================================================ +# Context +# ============================================================ + +build_context() { + + local file="$1" + + local relative="${file#"$ROOT"/}" + local area="${relative%%/*}" + + cat </dev/null 2>&1 || + return 0 + + git add "$1" + + git commit \ + -m "research: $(basename "$1" .md | tr '-' ' ')" || + true +} + +# ============================================================ +# Research one topic +# ============================================================ + +research_topic() { + + local file="$1" + + log "==================================================" + log "Researching: $file" + log "==================================================" + + local round=1 + + local current="$STATE/runs/$RUN_ID/${file//\//_}.current.md" + local review="$STATE/runs/$RUN_ID/${file//\//_}.review.md" + + while (( round <= MAX_RESEARCH_ROUNDS )); do + + log "Research round $round/$MAX_RESEARCH_ROUNDS" + + local candidate="$STATE/runs/$RUN_ID/${file//\//_}.round${round}.md" + + # ---------------------------------------------------- + # Research + # ---------------------------------------------------- + + if (( round == 1 )); then + + log "Running researcher." + + if ! api_call \ + "$RESEARCH_PROVIDER" \ + "$RESEARCH_MODEL" \ + "$(research_system_prompt)" \ + "$(research_user_prompt "$file")" \ + "$candidate" + then + + record_attempt \ + "$file" "$round" "research" "api-failure" + + mark_failed "$file" + + return 1 + fi + + record_attempt \ + "$file" "$round" "research" "completed" + + # ---------------------------------------------------- + # Revision + # ---------------------------------------------------- + + else + + log "Running revision agent." + + if ! api_call \ + "$REVISION_PROVIDER" \ + "$REVISION_MODEL" \ + "$(revision_system_prompt)" \ + "$(revision_user_prompt "$current" "$review")" \ + "$candidate" + then + + record_attempt \ + "$file" "$round" "revision" "api-failure" + + mark_failed "$file" + + return 1 + fi + + record_attempt \ + "$file" "$round" "revision" "completed" + fi + + cp "$candidate" "$current" + + # ---------------------------------------------------- + # Review + # ---------------------------------------------------- + + if [[ "$REVIEW_ENABLED" != "true" ]]; then + + replace_soon "$file" "$candidate" + + mark_completed "$file" + + git_commit "$file" + + return 0 + fi + + log "Running reviewer." + + if ! api_call \ + "$REVIEW_PROVIDER" \ + "$REVIEW_MODEL" \ + "$(review_system_prompt)" \ + "$(review_user_prompt "$candidate")" \ + "$review" + then + + record_attempt \ + "$file" "$round" "review" "api-failure" + + mark_failed "$file" + + return 1 + fi + + echo + echo "==========================================" + echo "RESEARCH REVIEW" + echo "==========================================" + + cat "$review" + + echo + echo "==========================================" + echo + + local verdict + + verdict="$( + grep -m1 '^VERDICT:' "$review" || true + )" + + record_attempt \ + "$file" "$round" "review" "$verdict" + + # ---------------------------------------------------- + # PASS + # ---------------------------------------------------- + + if [[ "$verdict" == "VERDICT: PASS" ]]; then + + log "Research PASSED review." + + replace_soon "$file" "$candidate" + + mark_completed "$file" + + git_commit "$file" + + log "Completed: $file" + + return 0 + fi + + # ---------------------------------------------------- + # FAIL + # ---------------------------------------------------- + + log "Research rejected by reviewer." + + if (( round >= MAX_RESEARCH_ROUNDS )); then + + log "Maximum research rounds reached." + log "Leaving original document unchanged." + log "Marking topic failed for this run." + + mark_failed "$file" + + return 2 + fi + + log "Preparing revision round $((round + 1))." + + ((round++)) + + sleep "$DELAY_SECONDS" + done + + mark_failed "$file" + + return 2 +} + +# ============================================================ +# Run one +# ============================================================ + +run_single() { + + start_run + + local file="${1:-}" + + if [[ -z "$file" ]]; then + file="$(find_next_task || true)" + fi + + [[ -n "$file" ]] || + die "No research topics remain." + + research_topic "$file" +} + +# ============================================================ +# Run all +# ============================================================ + +run_all() { + + start_run + + local iteration=0 + + while true; do + + if (( MAX_ITERATIONS > 0 && + iteration >= MAX_ITERATIONS )); then + + log "Maximum run iterations reached." + + break + fi + + local file + + file="$(find_next_task || true)" + + if [[ -z "$file" ]]; then + + log "No remaining research topics." + + break + fi + + ((iteration += 1)) + + echo + echo "##################################################" + echo "XH-1 AUTONOMOUS RESEARCH" + echo "Iteration: $iteration" + echo "Topic: $file" + echo "##################################################" + echo + + research_topic "$file" || true + + sleep "$DELAY_SECONDS" + done + + status +} + +# ============================================================ +# Status +# ============================================================ + +status() { + + local total + local soon + + total="$( + find "$ROOT" \ + -type f \ + -name '*.md' \ + -not -path "$STATE/*" | + wc -l | + tr -d ' ' + )" + + soon="$( + grep -RIl \ + '^SOON[[:space:]]*$' \ + "$ROOT" \ + --include='*.md' \ + --exclude-dir='.xh1' \ + 2>/dev/null | + wc -l | + tr -d ' ' + )" + + local completed=$((total - soon)) + + echo + echo "╔══════════════════════════════════════════╗" + echo "║ XH-1 RESEARCH STATUS ║" + echo "╚══════════════════════════════════════════╝" + echo + + printf " Total documents : %s\n" "$total" + printf " Completed : %s\n" "$completed" + printf " Remaining SOON : %s\n" "$soon" + + if (( total > 0 )); then + awk \ + -v done="$completed" \ + -v total="$total" \ + 'BEGIN { + printf " Progress : %.1f%%\n", + (done / total) * 100 + }' + fi + + echo + echo "Next topic:" + + find_next_task || echo " None" + + echo +} + +# ============================================================ +# Reset failure +# ============================================================ + +reset_failure() { + + local file="${1:-}" + + [[ -n "$file" ]] || + die "Usage: $0 reset-failure " + + [[ -f "$file" ]] || + die "File does not exist: $file" + + if [[ -n "$RUN_ID" && + -f "$STATE/runs/$RUN_ID/failed" ]]; then + + sed -i "\|^${file}$|d" \ + "$STATE/runs/$RUN_ID/failed" + fi + + log "Failure reset: $file" +} + +# ============================================================ +# Manual review +# ============================================================ + +review_command() { + + local file="${1:-}" + + [[ -n "$file" ]] || + die "Usage: $0 review " + + [[ -f "$file" ]] || + die "File does not exist: $file" + + local output="$STATE/manual-review-$(timestamp).md" + + api_call \ + "$REVIEW_PROVIDER" \ + "$REVIEW_MODEL" \ + "$(review_system_prompt)" \ + "$(review_user_prompt "$file")" \ + "$output" + + cat "$output" +} + +# ============================================================ +# Provider status +# ============================================================ + +providers() { + + echo + echo "XH-1 PROVIDERS" + echo "==============" + echo + + local provider + + for provider in $PROVIDERS; do + + local url + url="$(provider_url "$provider")" + + printf "%-15s %s" "$provider" "$url" + + if provider_check "$provider" >/dev/null 2>&1; then + echo " [OK]" + else + echo " [OFFLINE/ERROR]" + fi + + done + + echo +} + +# ============================================================ +# Help +# ============================================================ + +help() { + +cat <<'HELP' + +XH-1 RESEARCH HARNESS + +MULTI-PROVIDER AUTONOMOUS RESEARCH SYSTEM + + +COMMANDS +-------- + + init + + Create a configuration template. + + status + + Show research progress. + + providers + + Test configured API providers. + + next + + Show next SOON document. + + run + + Research the next topic. + + run + + Research a specific topic. + + run-all + + Process all SOON documents. + + review + + Manually review a document. + + reset-failure + + Retry a failed topic. + + help + + Show this help. + + +PIPELINE +-------- + + RESEARCH + | + v + REVIEW + | + +--+--+ + | | + PASS FAIL + | | + v v + WRITE REVISE + | + v + REVIEW + + +PROVIDERS +--------- + +Each agent can use a different provider. + +Example: + + Researcher -> OpenRouter + Reviewer -> Ollama + Revision -> OpenRouter + + +SUPPORTED PROVIDER STYLE +------------------------ + +Any OpenAI-compatible API. + +Examples: + + OpenRouter + Ollama + LM Studio + vLLM + LocalAI + OpenAI + Other compatible APIs + + +CONFIGURATION +------------- + +PROVIDERS + + Space-separated provider names. + + Example: + + PROVIDERS="openrouter ollama" + + +Provider URL: + + PROVIDER__URL + + +Provider API key environment variable: + + PROVIDER__KEY_ENV + + +AGENTS + + RESEARCH_PROVIDER + RESEARCH_MODEL + + REVIEW_PROVIDER + REVIEW_MODEL + + REVISION_PROVIDER + REVISION_MODEL + + +RESEARCH SETTINGS + + MAX_RESEARCH_ROUNDS + MAX_API_RETRIES + DELAY_SECONDS + MAX_ITERATIONS + REVIEW_ENABLED + AUTO_COMMIT + + +EXAMPLE +------- + + export OPENROUTER_API_KEY="..." + + ./xh1-research providers + + ./xh1-research status + + ./xh1-research run + + ./xh1-research run-all + +HELP +} + +# ============================================================ +# Main +# ============================================================ + +load_config +require_dependencies + +case "${1:-status}" in + + init) + echo "Edit $CONFIG to configure providers." + ;; + + status) + status + ;; + + providers) + providers + ;; + + next) + find_next_task || true + ;; + + run) + run_single "${2:-}" + ;; + + run-all|loop|resume) + run_all + ;; + + review) + review_command "${2:-}" + ;; + + reset-failure) + reset_failure "${2:-}" + ;; + + help|-h|--help) + help + ;; + + *) + die "Unknown command: $1" + ;; + +esac \ No newline at end of file