Hermes
659ad6f950
Use MM registration/packet-service events as the recovery trigger instead of public egress probes, so re-register heals with zero cellular traffic. Wait for wwan IPv4 after settle, coalesce bursts, and keep ip-monitor + poll as fallbacks.
440 lines
14 KiB
Bash
440 lines
14 KiB
Bash
#!/usr/bin/env bash
|
||
# 监控数据网卡 + ModemManager 重拨/重注册事件，自动等待附着并 regenerate。
|
||
# 策略：
|
||
# 1) 默认只 bind_interface（不写死源 IP）—— 多数 IP 漂移无需 pin
|
||
# 2) iface 丢失→重探；可选 BIND_SOURCE_IP 时同步源 IP
|
||
# 3) **优先以 ModemManager 日志为重拨标准**（零公网流量）：
|
||
# home→searching / attached→detached / bearer 断连 等 → 等 IPv4 回来 → generate/restart
|
||
# 4) 不默认做公网出口探针（省流量）；仅本地 iface/MM 状态
|
||
set -euo pipefail
|
||
|
||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
# shellcheck source=lib.sh
|
||
source "$ROOT_DIR/scripts/lib.sh"
|
||
|
||
CONFIG_FILE="${CONFIG_FILE:-$ROOT_DIR/etc/settings.conf}"
|
||
if [[ ! -f "$CONFIG_FILE" && -f /opt/cellular-proxy/etc/settings.conf ]]; then
|
||
CONFIG_FILE=/opt/cellular-proxy/etc/settings.conf
|
||
ROOT_DIR=/opt/cellular-proxy
|
||
fi
|
||
export CONFIG_FILE
|
||
|
||
load_config
|
||
|
||
INTERVAL="${WATCH_INTERVAL_SEC:-60}"
|
||
DEBOUNCE_SEC="${WATCH_DEBOUNCE_SEC:-3}"
|
||
# ModemManager 事件驱动（默认开；只读本机 journal，不耗蜂窝公网流量）
|
||
WATCH_MM_EVENTS="${WATCH_MM_EVENTS:-true}"
|
||
WATCH_MM_UNIT="${WATCH_MM_UNIT:-ModemManager}"
|
||
WATCH_MM_SETTLE_SEC="${WATCH_MM_SETTLE_SEC:-5}"
|
||
WATCH_MM_READY_TIMEOUT_SEC="${WATCH_MM_READY_TIMEOUT_SEC:-90}"
|
||
WATCH_MM_READY_POLL_SEC="${WATCH_MM_READY_POLL_SEC:-2}"
|
||
# 同一轮 MM 抖动合并窗口（秒）：连续 searching/detached 只恢复一次
|
||
WATCH_MM_COALESCE_SEC="${WATCH_MM_COALESCE_SEC:-20}"
|
||
|
||
LOCK_DIR="${LOG_DIR:-/var/log/cellular-proxy}"
|
||
mkdir -p "$LOCK_DIR" 2>/dev/null || true
|
||
STATE_FILE="${LOCK_DIR}/watch.state"
|
||
MM_CURSOR_FILE="${LOCK_DIR}/mm.cursor"
|
||
MM_LAST_EVENT_FILE="${LOCK_DIR}/mm.last_event"
|
||
LOG_TAG="cellular-proxy-watch"
|
||
|
||
logw() { printf '[%s] %s %s\n' "$(date '+%F %T')" "$LOG_TAG" "$*" >&2; }
|
||
|
||
truthy() {
|
||
case "${1,,}" in
|
||
1|true|yes|on) return 0 ;;
|
||
*) return 1 ;;
|
||
esac
|
||
}
|
||
|
||
current_cfg_iface() {
|
||
python3 - <<'PY' 2>/dev/null || true
|
||
import json
|
||
from pathlib import Path
|
||
for p in ("/opt/cellular-proxy/etc/config.json",):
|
||
try:
|
||
c=json.loads(Path(p).read_text())
|
||
for o in c.get("outbounds") or []:
|
||
if o.get("tag")=="cellular":
|
||
print(o.get("bind_interface") or "")
|
||
raise SystemExit
|
||
except Exception:
|
||
pass
|
||
print("")
|
||
PY
|
||
}
|
||
|
||
current_cfg_src() {
|
||
python3 - <<'PY' 2>/dev/null || true
|
||
import json
|
||
from pathlib import Path
|
||
try:
|
||
c=json.loads(Path("/opt/cellular-proxy/etc/config.json").read_text())
|
||
for o in c.get("outbounds") or []:
|
||
if o.get("tag")=="cellular":
|
||
print(o.get("inet4_bind_address") or "")
|
||
raise SystemExit
|
||
except Exception:
|
||
pass
|
||
print("")
|
||
PY
|
||
}
|
||
|
||
# 将 ModemManager 日志行判定为「重拨 / 脱网 / 重注册」类事件
|
||
is_mm_redial_line() {
|
||
local line="${1,,}"
|
||
# 忽略纯噪声
|
||
[[ -z "$line" ]] && return 1
|
||
[[ "$line" == *"-- no entries --"* ]] && return 1
|
||
|
||
# 核心：注册/分组附着状态机（本次 16:48 日志即此类）
|
||
if [[ "$line" == *"registration state changed"* ]]; then
|
||
return 0
|
||
fi
|
||
if [[ "$line" == *"packet service state changed"* ]]; then
|
||
return 0
|
||
fi
|
||
if [[ "$line" == *"consolidated registration state"* && ( "$line" == *"searching"* || "$line" == *"denied"* || "$line" == *"unknown"* ) ]]; then
|
||
return 0
|
||
fi
|
||
# bearer / 连接态
|
||
if [[ "$line" == *"bearer"* && ( "$line" == *"disconnect"* || "$line" == *"connect"* || "$line" == *"failed"* ) ]]; then
|
||
return 0
|
||
fi
|
||
if [[ "$line" == *"state changed"* && ( "$line" == *"connected"* || "$line" == *"disconnected"* || "$line" == *"searching"* || "$line" == *"registered"* ) ]]; then
|
||
# 避免把完全无关的 service 状态刷进来：需 modem 语境
|
||
if [[ "$line" == *"[modem"* || "$line" == *"modem"* || "$line" == *"3gpp"* ]]; then
|
||
return 0
|
||
fi
|
||
fi
|
||
if [[ "$line" == *"access technology changed"* ]]; then
|
||
return 0
|
||
fi
|
||
if [[ "$line" == *"unexpected lte system info"* ]]; then
|
||
return 0
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
init_mm_cursor_now() {
|
||
# 升级/首次启动：锚到「现在」，不回放历史，避免一启动就误 rebind
|
||
[[ -f "$MM_CURSOR_FILE" ]] && return 0
|
||
if ! command -v journalctl >/dev/null 2>&1; then
|
||
return 1
|
||
fi
|
||
local cur
|
||
cur="$(journalctl -u "$WATCH_MM_UNIT" -n 0 --show-cursor --no-pager 2>/dev/null | sed -n 's/^-- cursor: //p' | tail -1 || true)"
|
||
if [[ -n "$cur" ]]; then
|
||
printf '%s\n' "$cur" >"$MM_CURSOR_FILE"
|
||
logw "mm cursor initialized (no backlog replay)"
|
||
return 0
|
||
fi
|
||
# 无 journal 时写空标记
|
||
: >"$MM_CURSOR_FILE"
|
||
return 0
|
||
}
|
||
|
||
# 扫描 cursor 之后的 MM 日志；命中则打印摘要并 return 0
|
||
scan_mm_events_since_cursor() {
|
||
truthy "$WATCH_MM_EVENTS" || return 1
|
||
command -v journalctl >/dev/null 2>&1 || return 1
|
||
init_mm_cursor_now || true
|
||
|
||
local out hits=0 sample=""
|
||
# --after-cursor 需要已有 cursor；首次空文件则用 -n 0 建锚后返回无事件
|
||
if [[ ! -s "$MM_CURSOR_FILE" ]]; then
|
||
init_mm_cursor_now || true
|
||
return 1
|
||
fi
|
||
|
||
# 用临时 cursor 复制，避免 journalctl 在无新条目时搞乱；成功读后再写回
|
||
local cur
|
||
cur="$(tr -d '\r\n' <"$MM_CURSOR_FILE" 2>/dev/null || true)"
|
||
if [[ -z "$cur" ]]; then
|
||
init_mm_cursor_now || true
|
||
return 1
|
||
fi
|
||
|
||
out="$(journalctl -u "$WATCH_MM_UNIT" --after-cursor="$cur" --no-pager -o short-iso 2>/dev/null || true)"
|
||
# 推进 cursor 到最新（即使本轮无匹配事件也推进，避免重复扫）
|
||
local newcur
|
||
newcur="$(journalctl -u "$WATCH_MM_UNIT" -n 0 --show-cursor --no-pager 2>/dev/null | sed -n 's/^-- cursor: //p' | tail -1 || true)"
|
||
if [[ -n "$newcur" ]]; then
|
||
printf '%s\n' "$newcur" >"$MM_CURSOR_FILE"
|
||
fi
|
||
|
||
[[ -z "$out" ]] && return 1
|
||
while IFS= read -r line; do
|
||
[[ -z "$line" || "$line" == --* ]] && continue
|
||
if is_mm_redial_line "$line"; then
|
||
hits=$((hits + 1))
|
||
sample="$line"
|
||
fi
|
||
done <<<"$out"
|
||
|
||
if (( hits > 0 )); then
|
||
printf '%s\n' "$sample" >"$MM_LAST_EVENT_FILE" 2>/dev/null || true
|
||
logw "mm redial/re-register events=${hits} last=${sample}"
|
||
return 0
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
mm_line_looks_recovered() {
|
||
local line="${1,,}"
|
||
[[ "$line" == *"registration state changed"*"home"* ]] && return 0
|
||
[[ "$line" == *"packet service state changed"*"attached"* ]] && return 0
|
||
[[ "$line" == *"consolidated registration state"* && "$line" == *"'home'"* ]] && return 0
|
||
return 1
|
||
}
|
||
|
||
# 等待蜂窝数据面可用：iface 存在 + IPv4；可选 mmcli connected
|
||
wait_cellular_ready() {
|
||
local cell="${1:-}"
|
||
local deadline now src
|
||
deadline=$(( $(date +%s) + WATCH_MM_READY_TIMEOUT_SEC ))
|
||
logw "wait cellular ready iface=${cell:-?} timeout=${WATCH_MM_READY_TIMEOUT_SEC}s"
|
||
|
||
while true; do
|
||
now="$(date +%s)"
|
||
if (( now >= deadline )); then
|
||
logw "wait cellular ready: timeout"
|
||
return 1
|
||
fi
|
||
|
||
load_config
|
||
cell="${CELLULAR_IFACE:-$cell}"
|
||
if [[ -z "$cell" ]] || ! iface_exists "$cell"; then
|
||
cell="$(detect_cellular_iface "" || true)"
|
||
fi
|
||
if [[ -n "$cell" ]] && iface_exists "$cell"; then
|
||
src="$(detect_source_ip "$cell" || true)"
|
||
if [[ -n "$src" ]]; then
|
||
# 若 mmcli 可用，尽量确认 modem 已 connected/attached（失败不阻断，有 IPv4 即可）
|
||
if command -v mmcli >/dev/null 2>&1; then
|
||
if mmcli -L 2>/dev/null | grep -qi modem; then
|
||
if mmcli -m any 2>/dev/null | grep -qiE 'state:[[:space:]]*connected|packet service state:[[:space:]]*attached|registration:[[:space:]]*home'; then
|
||
logw "cellular ready: iface=$cell src=$src (mmcli ok)"
|
||
printf '%s\n' "$cell"
|
||
return 0
|
||
fi
|
||
# 有 IPv4 但 mmcli 尚未 home：再等一小会儿
|
||
sleep "$WATCH_MM_READY_POLL_SEC"
|
||
src2="$(detect_source_ip "$cell" || true)"
|
||
if [[ -n "$src2" ]]; then
|
||
logw "cellular ready: iface=$cell src=$src2 (ipv4 present; mmcli not yet home)"
|
||
printf '%s\n' "$cell"
|
||
return 0
|
||
fi
|
||
else
|
||
logw "cellular ready: iface=$cell src=$src"
|
||
printf '%s\n' "$cell"
|
||
return 0
|
||
fi
|
||
else
|
||
logw "cellular ready: iface=$cell src=$src"
|
||
printf '%s\n' "$cell"
|
||
return 0
|
||
fi
|
||
fi
|
||
fi
|
||
sleep "$WATCH_MM_READY_POLL_SEC"
|
||
done
|
||
}
|
||
|
||
need_refresh() {
|
||
load_config
|
||
local cell="${CELLULAR_IFACE:-}"
|
||
local live_src="" cfg_iface cfg_src
|
||
cfg_iface="$(current_cfg_iface)"
|
||
cfg_src="$(current_cfg_src)"
|
||
|
||
if [[ -z "$cell" ]] || ! iface_exists "$cell"; then
|
||
logw "iface missing or empty (CELLULAR_IFACE=${cell:-<空>}) → rebind"
|
||
return 0
|
||
fi
|
||
|
||
live_src="$(detect_source_ip "$cell" || true)"
|
||
if [[ -z "$live_src" ]]; then
|
||
# 网卡在但无 IPv4：可能刚重拨；不立刻 die，等下一轮或 MM 恢复路径
|
||
logw "wwan up but no IPv4 yet on $cell"
|
||
if [[ -n "$cfg_src" ]]; then
|
||
return 0
|
||
fi
|
||
return 1
|
||
fi
|
||
|
||
if [[ -n "$cfg_iface" && "$cfg_iface" != "$cell" ]]; then
|
||
logw "cfg iface $cfg_iface != settings $cell"
|
||
return 0
|
||
fi
|
||
|
||
# 仅当配置写了 inet4_bind_address 且与 live 不一致时才需要
|
||
if [[ -n "$cfg_src" && "$cfg_src" != "$live_src" ]]; then
|
||
logw "stale source IP cfg=$cfg_src live=$live_src"
|
||
return 0
|
||
fi
|
||
|
||
# settings 里的 SOURCE_IP 过期也同步（即使 config 未 pin）
|
||
if [[ -n "${CELLULAR_SOURCE_IP:-}" && "${CELLULAR_SOURCE_IP}" != "$live_src" ]]; then
|
||
persist_cellular_to_settings "$CONFIG_FILE" "$cell" "$live_src" || true
|
||
fi
|
||
|
||
return 1
|
||
}
|
||
|
||
do_refresh() {
|
||
load_config
|
||
local cell="${CELLULAR_IFACE:-}"
|
||
local src=""
|
||
local reason="${1:-manual}"
|
||
|
||
if [[ -z "$cell" ]] || ! iface_exists "$cell"; then
|
||
cell="$(detect_cellular_iface "" || true)"
|
||
if [[ -z "$cell" ]]; then
|
||
logw "detect failed; skip reason=$reason"
|
||
return 1
|
||
fi
|
||
fi
|
||
src="$(detect_source_ip "$cell" || true)"
|
||
if [[ -z "$src" ]]; then
|
||
logw "no IPv4 on $cell yet; skip generate reason=$reason"
|
||
return 1
|
||
fi
|
||
persist_cellular_to_settings "$CONFIG_FILE" "$cell" "$src" || true
|
||
logw "refresh: reason=$reason iface=$cell src=$src"
|
||
# 走 cpxy generate（含 check + restart）；失败则直接 scripts
|
||
if command -v cpxy >/dev/null 2>&1; then
|
||
cpxy generate >/dev/null 2>&1 || {
|
||
# shellcheck source=/dev/null
|
||
source "$ROOT_DIR/scripts/lib.sh"
|
||
load_config
|
||
"$ROOT_DIR/scripts/generate.sh" "$ROOT_DIR/generated"
|
||
install -m 0644 "$ROOT_DIR/generated/config.json" "$ROOT_DIR/etc/config.json"
|
||
install -m 0644 "$ROOT_DIR/generated/runtime.env" "$ROOT_DIR/etc/runtime.env" 2>/dev/null || true
|
||
systemctl restart cellular-proxy 2>/dev/null || true
|
||
}
|
||
else
|
||
"$ROOT_DIR/scripts/generate.sh" "$ROOT_DIR/generated"
|
||
install -m 0644 "$ROOT_DIR/generated/config.json" "$ROOT_DIR/etc/config.json"
|
||
systemctl restart cellular-proxy 2>/dev/null || true
|
||
fi
|
||
printf '%s reason=%s iface=%s src=%s\n' "$(date -Iseconds)" "$reason" "$cell" "$src" >"$STATE_FILE" 2>/dev/null || true
|
||
logw "refresh done reason=$reason"
|
||
}
|
||
|
||
# MM 事件后：settle → 等数据面 → refresh（即使未 pin 源 IP，也重启清半开连接）
|
||
handle_mm_recover() {
|
||
local now last=0
|
||
now="$(date +%s)"
|
||
if [[ -f "$STATE_FILE" ]]; then
|
||
# 合并窗口：避免 searching→registering→home 连打三次
|
||
last="$(stat -c %Y "$STATE_FILE" 2>/dev/null || echo 0)"
|
||
if (( now - last < WATCH_MM_COALESCE_SEC )); then
|
||
# 若 state 很新且是 mm 恢复，跳过
|
||
if grep -q 'reason=mm' "$STATE_FILE" 2>/dev/null; then
|
||
logw "mm recover coalesced (<${WATCH_MM_COALESCE_SEC}s)"
|
||
return 0
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
sleep "${WATCH_MM_SETTLE_SEC}"
|
||
local cell=""
|
||
cell="$(wait_cellular_ready "${CELLULAR_IFACE:-}" || true)"
|
||
if [[ -z "$cell" ]]; then
|
||
# 超时仍尝试一轮 need_refresh / refresh（可能仅有短暂 IPv4）
|
||
logw "mm recover: ready wait failed; try best-effort refresh"
|
||
fi
|
||
do_refresh "mm" || true
|
||
}
|
||
|
||
last_run=0
|
||
trigger() {
|
||
local now reason="${1:-poll}"
|
||
now="$(date +%s)"
|
||
if (( now - last_run < DEBOUNCE_SEC )); then
|
||
return 0
|
||
fi
|
||
last_run=$now
|
||
|
||
# 1) ModemManager 重拨标准（零公网流量）
|
||
if scan_mm_events_since_cursor; then
|
||
handle_mm_recover
|
||
return 0
|
||
fi
|
||
|
||
# 2) 原有 iface / stale pin 逻辑
|
||
if need_refresh; then
|
||
do_refresh "$reason" || true
|
||
fi
|
||
}
|
||
|
||
# oneshot mode (systemd timer)
|
||
if [[ "${1:-}" == "--once" ]]; then
|
||
trigger "once"
|
||
exit 0
|
||
fi
|
||
|
||
# long-running: periodic + ip monitor + MM journal follow
|
||
logw "start interval=${INTERVAL}s debounce=${DEBOUNCE_SEC}s mm_events=${WATCH_MM_EVENTS} mm_unit=${WATCH_MM_UNIT}"
|
||
init_mm_cursor_now || true
|
||
|
||
# initial
|
||
trigger "start"
|
||
|
||
child_pids=()
|
||
cleanup() {
|
||
local p
|
||
for p in "${child_pids[@]:-}"; do
|
||
kill "$p" 2>/dev/null || true
|
||
done
|
||
exit 0
|
||
}
|
||
trap cleanup TERM INT
|
||
|
||
# Prefer event-driven if available; always keep poll as safety net
|
||
if command -v ip >/dev/null 2>&1; then
|
||
(
|
||
ip -o monitor address link 2>/dev/null | while read -r line; do
|
||
cell="${CELLULAR_IFACE:-}"
|
||
if [[ -f "$CONFIG_FILE" ]]; then
|
||
cell="$(awk -F= '/^CELLULAR_IFACE=/{print $2; exit}' "$CONFIG_FILE" 2>/dev/null || true)"
|
||
fi
|
||
if [[ -n "$cell" ]] && [[ "$line" == *"$cell"* || "$line" == *Deleted* || "$line" == *wwan* || "$line" == *usb* || "$line" == *enx* ]]; then
|
||
sleep "$DEBOUNCE_SEC"
|
||
trigger "ip-monitor"
|
||
fi
|
||
done
|
||
) &
|
||
child_pids+=($!)
|
||
fi
|
||
|
||
# ModemManager 实时日志（标准：重拨/重注册）
|
||
if truthy "$WATCH_MM_EVENTS" && command -v journalctl >/dev/null 2>&1; then
|
||
(
|
||
# -n 0：不回放历史；-f：跟随
|
||
journalctl -u "$WATCH_MM_UNIT" -f -n 0 --no-pager 2>/dev/null | while IFS= read -r line; do
|
||
if is_mm_redial_line "$line"; then
|
||
logw "mm live: $line"
|
||
printf '%s\n' "$line" >"$MM_LAST_EVENT_FILE" 2>/dev/null || true
|
||
# 推进 cursor，避免 poll 再扫到同一批
|
||
newcur="$(journalctl -u "$WATCH_MM_UNIT" -n 0 --show-cursor --no-pager 2>/dev/null | sed -n 's/^-- cursor: //p' | tail -1 || true)"
|
||
[[ -n "$newcur" ]] && printf '%s\n' "$newcur" >"$MM_CURSOR_FILE"
|
||
sleep "$DEBOUNCE_SEC"
|
||
handle_mm_recover
|
||
fi
|
||
done
|
||
) &
|
||
child_pids+=($!)
|
||
logw "mm journal follow enabled unit=$WATCH_MM_UNIT"
|
||
else
|
||
logw "mm journal follow disabled or journalctl missing"
|
||
fi
|
||
|
||
while true; do
|
||
sleep "$INTERVAL"
|
||
trigger "poll"
|
||
done
|