From 3de2736882b4207a98f2cef2a21e0f53d5a96753 Mon Sep 17 00:00:00 2001 From: chick Date: Wed, 16 Sep 2026 18:57:06 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=A7=84=E8=8C=83=E5=8C=96=20JSON=20?= =?UTF-8?q?=E4=B8=8E=E4=B8=89=E8=AF=AD=E8=A8=80=E4=BA=92=E9=AA=8C=E5=8A=A0?= =?UTF-8?q?=E5=9B=BA=EF=BC=9Bapp-routes=20=E7=94=9F=E6=88=90=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - wire.CanonicalJSON 自己写字符串转义：Go 编码器即使在关闭 HTML 转义时仍会转义 U+2028/U+2029，而 Node 与 Foundation 输出字面量；两处拼写解码结果相同， 因此签名不同却看不出原因。孤立代理项改为显式拒绝（Go 会替换成 U+FFFD、 Node 保留转义文本、Foundation 直接解析失败，三种行为只能取交集为空） - 对象键排序按 UTF-8 字节序（Go 的 sort.Strings 语义）；Node 参考实现改用 Buffer.compare，因为 Array.sort 的 UTF-16 码元序在星形平面边界上与之相反 - test-vectors.json 新增转义、键序与必须拒绝的输入；Go/Node 双向断言 - 新增 scripts/export-app-routes.swift 生成 contracts/app-routes.json， 补上此前 vendored 路由表靠手抄的一环 - README 记录 iOS 侧向量校验为何是'互相验证'而非'逐字节相同'（CryptoKit 的 Ed25519 签名随机化） --- cmd/msa-vectors/main.go | 43 ++++++++++- internal/wire/json.go | 151 +++++++++++++++++++++++++++++++++---- internal/wire/wire_test.go | 35 +++++++++ 3 files changed, 214 insertions(+), 15 deletions(-) diff --git a/cmd/msa-vectors/main.go b/cmd/msa-vectors/main.go index ad13556..d2e2c6d 100644 --- a/cmd/msa-vectors/main.go +++ b/cmd/msa-vectors/main.go @@ -136,7 +136,8 @@ func main() { Body: `{"phone_number":"+8613800000000","content":"hello"}`, Digest: hexStr(intent), }, - CanonicalJSON: jsonPairs(), + CanonicalJSON: jsonPairs(), + CanonicalRejected: rejectedPairs(), } encoded, err := json.MarshalIndent(document, "", " ") @@ -167,6 +168,8 @@ type vectors struct { AEAD aeadVector `json:"aead"` Intent intentVector `json:"intent"` CanonicalJSON []jsonPairVector `json:"canonicalJson"` + // CanonicalRejected are inputs every implementation must refuse; see rejectedPairs. + CanonicalRejected []string `json:"canonicalJsonRejected"` } type canonicalVector struct { @@ -229,6 +232,31 @@ func jsonPairs() []jsonPairVector { `{"x":100}`, `{"a":{"c":[1,{"b":2,"a":1}],"b":0},"A":9}`, `{"phone_number":"+8613800000000","content":"hello"}`, + // Escaping edge cases. Each of these has been observed to differ between Go, Node and + // Foundation, so they are pinned rather than assumed: + // + // - U+2028/U+2029: Go's encoder escapes them even with HTML escaping off; Node's + // JSON.stringify and Foundation emit them literally. Both spellings decode to the same + // string, so a signature over the escape differs while the payload looks identical. + // - A paired surrogate: the three languages must agree on the astral character, which + // means agreeing that it is written as literal UTF-8 and not as two \u escapes. + // - Control characters below 0x20 with and without short forms. + // - A literal backslash-u sequence, which is text and must not be read as an escape. + `{"content":"a` + "\u2028" + `b"}`, + `{"content":"a` + "\u2029" + `b"}`, + `{"content":"` + "\U0001F600" + `"}`, + `{"content":"\b\f\u0001"}`, + `{"content":"\\u2028"}`, + // Key ordering. The rule is UTF-8 byte order, which is what Go's sort.Strings does, and it + // is not what either shortcut produces: Node's default sort compares UTF-16 code units + // (so an astral key starting with a D800 surrogate sorts below U+FFFD, while its UTF-8 + // form starting with F0 sorts above), and Swift's default String comparison uses Unicode + // collation (so "é" sorts next to "e" rather than after every ASCII byte). + // + // {"z","é"}: UTF-8 has z(0x7A) before é(0xC3A9); collation reverses that. + // {"\uFFFD","\U00010000"}: UTF-16 puts the surrogate first; UTF-8 puts F0 first. + "{\"\u00e9\":1,\"z\":2}", + "{\"\U00010000\":1,\"\uFFFD\":2}", } pairs := make([]jsonPairVector, 0, len(inputs)) for _, input := range inputs { @@ -239,6 +267,19 @@ func jsonPairs() []jsonPairVector { return pairs } +// rejectedPairs are inputs every implementation must refuse. They are in the vectors so the Swift +// and Node sides have the same list to check against rather than each inventing its own tolerance: +// a lone surrogate decodes to U+FFFD in Go, stays as escape text in Node, and fails to parse in +// Foundation, so the only consistent rule is to accept none of them. +func rejectedPairs() []string { + return []string{ + `{"content":"\ud800"}`, + `{"content":"\udc00"}`, + `{"content":"\ud800\ud800"}`, + `{"content":"\ud800x"}`, + } +} + func mustHex(value string) []byte { raw, err := hex.DecodeString(value) check(err) diff --git a/internal/wire/json.go b/internal/wire/json.go index 3864148..a084a65 100644 --- a/internal/wire/json.go +++ b/internal/wire/json.go @@ -10,8 +10,13 @@ import ( "sort" ) -// marshal is encoding/json with HTML escaping off. The default escapes <, > and &, which would make -// a body containing those characters hash differently here than it does in Node or Swift. +// marshal is encoding/json with HTML escaping off. +// +// It is used for the envelope's own fields (ids, base64), never for payload strings: Go's encoder +// escapes U+2028 and U+2029 even with HTML escaping disabled, while Node's JSON.stringify and +// Swift's JSONSerialization both emit them literally. That difference is invisible in a hex dump of +// ASCII payloads and shows up as an intent mismatch the first time an SMS body contains a line +// separator, so payload strings go through escapeCanonicalString instead. func marshal(value any) ([]byte, error) { buf := bytes.NewBuffer(nil) encoder := json.NewEncoder(buf) @@ -22,6 +27,48 @@ func marshal(value any) ([]byte, error) { return bytes.TrimRight(buf.Bytes(), "\n"), nil } +// escapeCanonicalString writes a JSON string literal following the documented canonical rules. +// +// It exists because Go's encoder is not a reference implementation for this protocol: it escapes +// U+2028/U+2029 (legal, unnecessary, and not what the other two languages do) and it silently +// rewrites lone surrogates to U+FFFD (which turns "reject this input" into "hash something else"). +// Both are the kind of difference that produces a signature failure with no useful cause. +func escapeCanonicalString(buf *bytes.Buffer, value string) { + const hexDigits = "0123456789abcdef" + buf.WriteByte('"') + for _, r := range value { + switch r { + case '"': + buf.WriteString(`\"`) + case '\\': + buf.WriteString(`\\`) + case '\n': + buf.WriteString(`\n`) + case '\r': + buf.WriteString(`\r`) + case '\t': + buf.WriteString(`\t`) + case '\b': + buf.WriteString(`\b`) + case '\f': + buf.WriteString(`\f`) + default: + switch { + case r < 0x20: + // The short forms above cover the ones JSON has names for; everything else below + // 0x20 is \u00XX, which is what both other languages emit. + buf.WriteString(`\u00`) + buf.WriteByte(hexDigits[(r>>4)&0xF]) + buf.WriteByte(hexDigits[r&0xF]) + default: + // Literal UTF-8, including DEL, U+FFFD and U+2028/U+2029. + buf.WriteRune(r) + } + } + } + buf.WriteByte('"') +} + func unmarshal(raw []byte, value any) error { decoder := json.NewDecoder(bytes.NewReader(raw)) decoder.UseNumber() @@ -44,14 +91,20 @@ func unmarshal(raw []byte, value any) error { // // 1. Object keys are sorted by UTF-8 byte order and emitted without whitespace. // 2. Arrays keep their order. -// 3. Strings use the same escapes as Go's encoder with HTML escaping disabled: \" \\ \n \r \t, -// \u00XX for other control characters, and literal UTF-8 for everything else. +// 3. Strings use the same escapes as Go's encoder with HTML escaping disabled: \" \\ \n \r \t +// \b \f, \u00XX for other control characters, and literal UTF-8 for everything else — +// including U+2028 and U+2029, which Go's encoder escapes and the other two implementations +// do not. Lone surrogates are rejected outright (rule 6). // 4. Numbers must be integers in canonical form: an optional minus, no leading zero, no fraction, // no exponent, no plus. This is stricter than JSON allows and on purpose. Any wider grammar has // values that are the same number but different bytes (1 and 1.0, 1e2 and 100), and "which // spelling did the phone hash" is not a question this protocol is willing to leave open. The // APIs behind it take integers, strings, booleans, arrays and objects, so nothing is lost. // 5. null, true and false are literal. +// 6. A lone surrogate escape (\uD800 without its pair, or an unpaired low surrogate) is an error. +// Go's decoder rewrites it to U+FFFD, Node keeps the escape text, and Foundation refuses to +// parse it at all: three behaviours, so the only way the three implementations can agree is to +// accept none of them. // // An empty or absent body canonicalises to the empty byte string, not to the four characters // "null", so "no body" and "empty body" cannot be confused. @@ -60,6 +113,9 @@ func CanonicalJSON(raw []byte) ([]byte, error) { if len(trimmed) == 0 { return nil, nil } + if err := rejectLoneSurrogates(trimmed); err != nil { + return nil, err + } decoder := json.NewDecoder(bytes.NewReader(trimmed)) decoder.UseNumber() var value any @@ -77,6 +133,81 @@ func CanonicalJSON(raw []byte) ([]byte, error) { return buf.Bytes(), nil } +// rejectLoneSurrogates scans the raw text for a \u escape in the surrogate range that is not +// correctly paired. +// +// It works on the encoded bytes rather than on the decoded string because decoding is exactly what +// destroys the evidence: Go replaces the lone surrogate with U+FFFD, after which "the caller sent +// an invalid escape" and "the caller sent U+FFFD" are the same bytes and only one of them is a +// legitimate thing to hash. +// +// The scan is escape-aware rather than a byte search: in the JSON text `"\\ud800"` the backslash is +// escaped, so the four characters after it are literal text and not a surrogate escape. Walking +// escapes one at a time is what distinguishes the two. +func rejectLoneSurrogates(raw []byte) error { + inString := false + for i := 0; i < len(raw); i++ { + switch raw[i] { + case '"': + inString = !inString + case '\\': + if !inString { + continue + } + if i+1 >= len(raw) { + return errors.New("wire: JSON string ends with a dangling backslash") + } + if raw[i+1] != 'u' { + i++ + continue + } + code, ok := hex4(raw, i+2) + if !ok { + return errors.New("wire: malformed \\u escape in JSON string") + } + switch { + case code >= 0xDC00 && code <= 0xDFFF: + return fmt.Errorf("wire: unpaired low surrogate U+%04X in JSON string", code) + case code >= 0xD800 && code <= 0xDBFF: + // A high surrogate must be followed immediately by its low half. + if i+11 >= len(raw) || raw[i+6] != '\\' || raw[i+7] != 'u' { + return fmt.Errorf("wire: unpaired high surrogate U+%04X in JSON string", code) + } + low, ok := hex4(raw, i+8) + if !ok || low < 0xDC00 || low > 0xDFFF { + return fmt.Errorf("wire: high surrogate U+%04X is not followed by a low surrogate", code) + } + i += 11 + continue + } + i += 5 + } + } + return nil +} + +// hex4 reads the four hex digits starting at offset, reporting whether all four were hex. +func hex4(raw []byte, offset int) (uint32, bool) { + if offset+4 > len(raw) { + return 0, false + } + var value uint32 + for _, c := range raw[offset : offset+4] { + value <<= 4 + switch { + case c >= '0' && c <= '9': + value |= uint32(c - '0') + case c >= 'a' && c <= 'f': + value |= uint32(c-'a') + 10 + case c >= 'A' && c <= 'F': + value |= uint32(c-'A') + 10 + default: + return 0, false + } + } + return value, true +} + // canonicalNumber is the only number spelling this protocol will hash. See rule 4 above. // // Negative zero is excluded even though it is an integer: no implementation can be relied on to @@ -100,11 +231,7 @@ func writeCanonical(buf *bytes.Buffer, value any) error { } buf.WriteString(literal) case string: - encoded, err := marshal(typed) - if err != nil { - return err - } - buf.Write(encoded) + escapeCanonicalString(buf, typed) case []any: buf.WriteByte('[') for index, item := range typed { @@ -127,11 +254,7 @@ func writeCanonical(buf *bytes.Buffer, value any) error { if index > 0 { buf.WriteByte(',') } - encodedKey, err := marshal(key) - if err != nil { - return err - } - buf.Write(encodedKey) + escapeCanonicalString(buf, key) buf.WriteByte(':') if err := writeCanonical(buf, typed[key]); err != nil { return err diff --git a/internal/wire/wire_test.go b/internal/wire/wire_test.go index 446965a..184c095 100644 --- a/internal/wire/wire_test.go +++ b/internal/wire/wire_test.go @@ -433,6 +433,17 @@ func TestCanonicalJSONRules(t *testing.T) { {``, ``}, {` `, ``}, {`{"a":{"c":[1,{"b":2,"a":1}],"b":0},"A":9}`, `{"A":9,"a":{"b":0,"c":[1,{"a":1,"b":2}]}}`}, + // Go's encoder escapes U+2028/U+2029 even with HTML escaping off; Node and Foundation do + // not. Pinned here because the two spellings decode identically, so nothing else in the + // suite would notice a regression until an SMS body contained a line separator. + {"{\"c\":\"a\u2028b\"}", "{\"c\":\"a\u2028b\"}"}, + {"{\"c\":\"a\u2029b\"}", "{\"c\":\"a\u2029b\"}"}, + // A literal backslash-u sequence is text, not an escape, and must survive unchanged. + {`{"c":"\\u2028"}`, `{"c":"\\u2028"}`}, + // Astral characters are literal UTF-8, not two \u escapes. + {"{\"c\":\"\U0001F600\"}", "{\"c\":\"\U0001F600\"}"}, + // Short escapes for the control characters JSON names, \u00XX for the rest. + {"{\"c\":\"\\b\\f\\u0001\"}", "{\"c\":\"\\b\\f\\u0001\"}"}, } for _, c := range cases { got, err := CanonicalJSON([]byte(c.in)) @@ -456,6 +467,30 @@ func TestCanonicalJSONRules(t *testing.T) { } } +// A lone surrogate is refused rather than silently rewritten. Go's decoder turns it into U+FFFD, +// Node keeps the escape text, and Foundation refuses to parse it: three behaviours, so accepting +// none of them is the only rule all three implementations can share. +func TestCanonicalJSONRejectsLoneSurrogates(t *testing.T) { + for _, raw := range []string{ + `{"c":"\ud800"}`, + `{"c":"\udc00"}`, + `{"c":"\ud800\ud800"}`, + `{"c":"\ud800x"}`, + } { + if got, err := CanonicalJSON([]byte(raw)); err == nil { + t.Errorf("CanonicalJSON(%q) accepted it as %q", raw, got) + } + } + // A correctly paired surrogate is a normal astral character and must still be accepted. + got, err := CanonicalJSON([]byte(`{"c":"\ud83d\ude00"}`)) + if err != nil { + t.Fatalf("a paired surrogate was refused: %v", err) + } + if string(got) != "{\"c\":\"\U0001F600\"}" { + t.Fatalf("paired surrogate canonicalised to %q", got) + } +} + func TestCanonicalJSONRejectsGarbage(t *testing.T) { for _, raw := range []string{`{`, `{"a":1} trailing`, `{"a":+1}`, `{"a":.5}`, `{"a":NaN}`, `{"a":1.0}`, `{"a":1e3}`, `{"a":01}`, `{"a":-01}`, `{"a":-0}`} { if _, err := CanonicalJSON([]byte(raw)); err == nil {