diff --git a/.github/workflows/middlewares.yml b/.github/workflows/middlewares.yml index edec427..f15a85f 100644 --- a/.github/workflows/middlewares.yml +++ b/.github/workflows/middlewares.yml @@ -14,18 +14,18 @@ jobs: sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v7 with: - go-version: 1.19 + go-version: "1.27" - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Install xcaddy run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest - name: Build go-esi as caddy module - run: cd middleware/caddy && xcaddy build --with github.com/darkweak/go-esi/middleware/caddy=./ --with github.com/darkweak/go-esi@latest=../.. + run: cd middleware/caddy && xcaddy build --with github.com/darkweak/go-esi=../.. --with github.com/darkweak/go-esi/middleware/caddy=./ - name: Run Caddy tests run: cd middleware/caddy && go test -v ./... @@ -39,12 +39,12 @@ jobs: steps: - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v7 with: - go-version: 1.19 + go-version: "1.27" - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: Run Roadrunner tests run: cd middleware/roadrunner && go test -v ./... diff --git a/.github/workflows/non-regression.yml b/.github/workflows/non-regression.yml index 276a958..6deca48 100644 --- a/.github/workflows/non-regression.yml +++ b/.github/workflows/non-regression.yml @@ -15,15 +15,17 @@ jobs: sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts - name: checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v7 - name: install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v7 with: - go-version: 1.19 + go-version: "1.27" - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v9 + with: + version: v2.15 - name: install xcaddy run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5928221..0c352e7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,20 +11,20 @@ jobs: steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v7 with: - go-version: 1.19 + go-version: "1.27" - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v7 with: fetch-depth: 0 - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v3 + uses: goreleaser/goreleaser-action@v7 with: - version: latest - args: release --rm-dist + version: "~> v2" + args: release --clean workdir: ./middleware/server env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.golangci.yml b/.golangci.yml index 88ad0e9..17620d7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,25 +1,53 @@ +version: "2" run: - timeout: 30s issues-exit-code: 1 - linters: - enable-all: true + default: all disable: - bodyclose - cyclop - - exhaustivestruct + - depguard + - embeddedstructfieldcheck - exhaustruct + - exhaustruct_v5 - forbidigo + - funcorder - gochecknoglobals - - gci - - golint - - gomnd - - ifshort + - inamedparam - ireturn + - mnd - nestif + - noinlineerr - nonamedreturns - - nosnakecase - revive - testpackage - - stylecheck - varnamelen + - wrapcheck + - wsl + exclusions: + generated: lax + rules: + - path: _test\.go + linters: + - gosec + - noctx + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - gofumpt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/README.md b/README.md index 9ab2c50..dabefa7 100644 --- a/README.md +++ b/README.md @@ -26,16 +26,17 @@ go get -u github.com/darkweak/go-esi ``` ## Usage +### Parse a whole document ```go import ( // ... - github.com/darkweak/go-esi/esi + "github.com/darkweak/go-esi/esi" ) //... func functionToParseESITags(b []byte, r *http.Request) []byte { - // Returns the parsed response. + // Returns the parsed response. The include tags are fetched in parallel. res := esi.Parse(b, r) //... @@ -43,6 +44,37 @@ func functionToParseESITags(b []byte, r *http.Request) []byte { } ``` +### Stream a document +`esi.Stream` processes the ESI tags while the document is written to it. The static content is forwarded as soon as it is +read, the include tags are fetched concurrently, and their content is written in the document order. The destination is +flushed (if it implements `http.Flusher`) whenever the output has to wait for an include or for more input. +```go +func streamESI(dst io.Writer, src io.Reader, r *http.Request) error { + s := esi.NewStream(dst, r) + if _, err := io.Copy(s, src); err != nil { + _ = s.Close() + + return err + } + + // Close processes the remaining content and waits for the pending includes. + return s.Close() +} +``` + +### Wrap an `http.ResponseWriter` +```go +import "github.com/darkweak/go-esi/writer" + +func esiMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + cw := writer.NewWriter(rw, r) + next.ServeHTTP(cw, r) + _ = cw.Close() + }) +} +``` + ## Available as middleware - [x] Caddy - [x] Træfik @@ -99,7 +131,7 @@ experimental: plugins: souin: moduleName: github.com/darkweak/go-esi - version: v0.0.6 + version: v0.0.7 ``` ```yaml # anywhere/dynamic-configuration diff --git a/esi/choose.go b/esi/choose.go index 5c4030d..ae2cddb 100644 --- a/esi/choose.go +++ b/esi/choose.go @@ -1,6 +1,7 @@ package esi import ( + "bytes" "net/http" "regexp" ) @@ -8,7 +9,7 @@ import ( const choose = "choose" var ( - closeChoose = regexp.MustCompile("") + closeChoose = []byte("") whenRg = regexp.MustCompile(`(?s)(.+?)`) otherwiseRg = regexp.MustCompile(`(?s)(.+?)`) ) @@ -33,12 +34,12 @@ type chooseTag struct { // // ). func (c *chooseTag) Process(b []byte, req *http.Request) ([]byte, int) { - found := closeChoose.FindIndex(b) - if found == nil { + end := closeEnd(b, closeChoose) + if end == 0 { return nil, len(b) } - c.length = found[1] + c.length = end tagIdxs := whenRg.FindAllSubmatch(b, -1) var res []byte @@ -60,13 +61,9 @@ func (c *chooseTag) Process(b []byte, req *http.Request) ([]byte, int) { } func (*chooseTag) HasClose(b []byte) bool { - return closeChoose.FindIndex(b) != nil + return bytes.Contains(b, closeChoose) } func (*chooseTag) GetClosePosition(b []byte) int { - if idx := closeChoose.FindIndex(b); idx != nil { - return idx[1] - } - - return 0 + return closeEnd(b, closeChoose) } diff --git a/esi/comment.go b/esi/comment.go index 0aef0e6..5056aa6 100644 --- a/esi/comment.go +++ b/esi/comment.go @@ -1,36 +1,47 @@ package esi import ( + "bytes" "net/http" - "regexp" ) const comment = "comment" -var closeComment = regexp.MustCompile("/>((\n| +)+)?") +var closeComment = []byte("/>") type commentTag struct { *baseTag } +// commentEnd returns the end of the closing "/>" including the spaces and +// newlines following it, or 0. +func commentEnd(b []byte) int { + end := closeEnd(b, closeComment) + if end == 0 { + return 0 + } + + for end < len(b) && isSpace(b[end]) { + end++ + } + + return end +} + // Input (e.g. comment text="This is a comment." />). -func (c *commentTag) Process(b []byte, req *http.Request) ([]byte, int) { - found := closeComment.FindIndex(b) - if found == nil { +func (c *commentTag) Process(b []byte, _ *http.Request) ([]byte, int) { + end := commentEnd(b) + if end == 0 { return nil, len(b) } - return []byte{}, found[1] + return []byte{}, end } func (*commentTag) HasClose(b []byte) bool { - return closeComment.FindIndex(b) != nil + return bytes.Contains(b, closeComment) } func (*commentTag) GetClosePosition(b []byte) int { - if idx := closeComment.FindIndex(b); idx != nil { - return idx[1] - } - - return 0 + return commentEnd(b) } diff --git a/esi/escape.go b/esi/escape.go index 9bf7564..1a39047 100644 --- a/esi/escape.go +++ b/esi/escape.go @@ -1,48 +1,38 @@ package esi import ( + "bytes" "net/http" - "regexp" ) const escape = "") - startEscape = regexp.MustCompile("((\n| +)+)?") -) +var closeEscape = []byte("-->") type escapeTag struct { *baseTag } func (e *escapeTag) Process(b []byte, req *http.Request) ([]byte, int) { - closeIdx := closeEscape.FindIndex(b) - - if closeIdx == nil { + closeStart, end := findClose(b, closeEscape) + if closeStart == -1 { return nil, len(b) } startPosition := 0 - if startIdx := startEscape.FindIndex(b); startIdx != nil { - startPosition = startIdx[1] + for startPosition < closeStart && isSpace(b[startPosition]) { + startPosition++ } - e.length = closeIdx[1] - b = b[startPosition:closeIdx[0]] + e.length = end - return b, e.length + return b[startPosition:closeStart], e.length } func (*escapeTag) HasClose(b []byte) bool { - return closeEscape.FindIndex(b) != nil + return bytes.Contains(b, closeEscape) } func (*escapeTag) GetClosePosition(b []byte) int { - if idx := closeEscape.FindIndex(b); idx != nil { - return idx[1] - } - - return 0 + return closeEnd(b, closeEscape) } diff --git a/esi/esi.go b/esi/esi.go index c6ce30e..6e539f6 100644 --- a/esi/esi.go +++ b/esi/esi.go @@ -1,50 +1,26 @@ package esi import ( + "bytes" "net/http" + "sync" ) func findTagName(b []byte) Tag { - name := tagname.FindSubmatch(b) - if name == nil { - return nil + if bytes.HasPrefix(b, escapePrefix) { + return &escapeTag{baseTag: newBaseTag()} } - switch string(name[1]) { - case comment: - return &commentTag{ - baseTag: newBaseTag(), - } - case choose: - return &chooseTag{ - baseTag: newBaseTag(), - } - case escape: - return &escapeTag{ - baseTag: newBaseTag(), - } - case include: - return &includeTag{ - baseTag: newBaseTag(), - } - case remove: - return &removeTag{ - baseTag: newBaseTag(), - } - case try: - case vars: - return &varsTag{ - baseTag: newBaseTag(), - } - default: - return nil - } + name, _ := readTagName(b) - return nil + return newTag(name) } +// HasOpenedTags reports whether b contains an ESI opening sequence. func HasOpenedTags(b []byte) bool { - return esi.FindIndex(b) != nil || escapeRg.FindIndex(b) != nil + _, kind := nextTagStart(b, true) + + return kind != kindNone } func CanProcess(b []byte) bool { @@ -55,64 +31,107 @@ func CanProcess(b []byte) bool { return false } -func ReadToTag(next []byte, pointer int) (startTagPosition, esiPointer int, t Tag) { - var isEscapeTag bool +// ReadToTag returns the position of the next ESI tag in next, the position +// right after its opening sequence and the matching Tag. +// +// Deprecated: use Parse or NewStream which handle the whole tag lifecycle. +func ReadToTag(next []byte, _ int) (startTagPosition, esiPointer int, t Tag) { + i, kind := nextTagStart(next, true) + if kind == kindNone { + return len(next), 0, nil + } + + if kind == kindEscape { + return i, i + len(escapePrefix), &escapeTag{baseTag: newBaseTag()} + } + + esiPointer = i + len(esiPrefix) + + return i, esiPointer, findTagName(next[esiPointer:]) +} + +// asyncPart is a tag processed concurrently whose output fills a slot of the +// final document. +type asyncPart struct { + t Tag + body []byte + out []byte +} + +type part struct { + static []byte + async *asyncPart +} - tagIdx := esi.FindIndex(next) +// collector is the sink used by Parse. It runs I/O-bound tags concurrently, +// except the last one which is processed on the calling goroutine since it +// would wait anyway. +type collector struct { + req *http.Request + parts []part + pending *asyncPart + wg sync.WaitGroup +} - if escIdx := escapeRg.FindIndex(next); escIdx != nil && (tagIdx == nil || escIdx[0] < tagIdx[0]) { - tagIdx = escIdx - tagIdx[1] = escIdx[0] - isEscapeTag = true +func (c *collector) static(b []byte) { + if len(b) > 0 { + c.parts = append(c.parts, part{static: b}) } +} - if tagIdx == nil { - return len(next), 0, nil +func (c *collector) tag(t Tag, body []byte) { + if !isAsync(t, body) { + c.static(process(t, body, c.req)) + + return } - esiPointer = tagIdx[1] - startTagPosition = tagIdx[0] - t = findTagName(next[esiPointer:]) + a := &asyncPart{t: t, body: body} + c.parts = append(c.parts, part{async: a}) - if isEscapeTag { - esiPointer += 7 + if p := c.pending; p != nil { + c.wg.Go(func() { p.out = process(p.t, p.body, c.req) }) } - return + c.pending = a } +// Parse processes the ESI tags of b and returns the resulting document. +// Independent include tags are fetched in parallel. b is not modified but the +// result may alias it. func Parse(b []byte, req *http.Request) []byte { - pointer := 0 + if !HasOpenedTags(b) { + return b + } - for pointer < len(b) { - var escapeTag bool + c := collector{req: req} + scan(b, true, &c) - next := b[pointer:] - tagIdx := esi.FindIndex(next) + if p := c.pending; p != nil { + p.out = process(p.t, p.body, req) + } - if escIdx := escapeRg.FindIndex(next); escIdx != nil && (tagIdx == nil || escIdx[0] < tagIdx[0]) { - tagIdx = escIdx - tagIdx[1] = escIdx[0] - escapeTag = true - } + c.wg.Wait() - if tagIdx == nil { - break - } + size := 0 - esiPointer := tagIdx[1] - t := findTagName(next[esiPointer:]) - - if escapeTag { - esiPointer += 7 + for _, p := range c.parts { + if p.async != nil { + size += len(p.async.out) + } else { + size += len(p.static) } + } - res, p := t.Process(next[esiPointer:], req) - esiPointer += p + out := make([]byte, 0, size) - b = append(b[:pointer], append(next[:tagIdx[0]], append(res, next[esiPointer:]...)...)...) - pointer += len(res) + tagIdx[0] + for _, p := range c.parts { + if p.async != nil { + out = append(out, p.async.out...) + } else { + out = append(out, p.static...) + } } - return b + return out } diff --git a/esi/esi_test.go b/esi/esi_test.go index 24f3b47..3771b55 100644 --- a/esi/esi_test.go +++ b/esi/esi_test.go @@ -1,14 +1,55 @@ package esi_test import ( + "errors" + "net" "net/http" "net/http/httptest" "os" "testing" + "time" "github.com/darkweak/go-esi/esi" ) +// fixtureRoutes mirrors the routes of middleware/caddy/Caddyfile. +func fixtureRoutes() *http.ServeMux { + mux := http.NewServeMux() + respond := func(path, body string) { + mux.HandleFunc(path, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(body)) + }) + } + + respond("/chained-esi-include-1", ``) + respond("/chained-esi-include-2", "

CHAINED 2

") + respond("/esi-include", "

ESI INCLUDE

") + respond("/alt-esi-include", "

ALTERNATE ESI INCLUDE

") + + return mux +} + +// TestMain serves the fixture routes on :9080 (domain.com must resolve to +// the loopback) unless another server, e.g. the detached caddy of the CI, +// already listens there. +func TestMain(m *testing.M) { + srv := &http.Server{Handler: fixtureRoutes(), ReadHeaderTimeout: time.Second} + + if ln, err := net.Listen("tcp", "127.0.0.1:9080"); err == nil { + go func() { + if e := srv.Serve(ln); e != nil && !errors.Is(e, http.ErrServerClosed) { + panic(e) + } + }() + } + + code := m.Run() + _ = srv.Close() + + os.Exit(code) +} + func loadFromFixtures(name string) []byte { b, e := os.ReadFile("../fixtures/" + name) if e != nil { @@ -114,7 +155,7 @@ func Test_Parse_fullMock(t *testing.T) { // Benchmarks. func BenchmarkInclude(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { esi.Parse( []byte( ``, @@ -132,7 +173,7 @@ var remove = ` ` func BenchmarkRemove(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { esi.Parse( []byte(remove), httptest.NewRequest(http.MethodGet, "http://domain.com:9080", nil), @@ -171,7 +212,7 @@ const full = ` ` func BenchmarkFull(b *testing.B) { - for i := 0; i < b.N; i++ { + for b.Loop() { esi.Parse( []byte(full), httptest.NewRequest(http.MethodGet, "http://domain.com:9080", nil), diff --git a/esi/include.go b/esi/include.go index 09a34bd..5147774 100644 --- a/esi/include.go +++ b/esi/include.go @@ -2,17 +2,19 @@ package esi import ( "bytes" - "context" "io" "net/http" "net/url" "regexp" ) -const include = "include" +const ( + include = "include" + maxIdleConnsPerHost = 64 +) var ( - closeInclude = regexp.MustCompile("/>") + closeInclude = []byte("/>") srcAttribute = regexp.MustCompile(`src="?(.+?)"?( |/>)`) altAttribute = regexp.MustCompile(`alt="?(.+?)"?( |/>)`) onErrorAttribute = regexp.MustCompile(`onerror="?(.+?)"?( |/>)`) @@ -73,45 +75,84 @@ func addHeaders(headers []string, req *http.Request, rq *http.Request) { } } +// client is shared by every include so connections are pooled and reused. +// Includes run in parallel, so keep more idle connections per host than the +// default (2) to avoid reconnecting on every page. +var client = &http.Client{Transport: newTransport()} + +func newTransport() http.RoundTripper { + t, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return http.DefaultTransport + } + + t = t.Clone() + t.MaxIdleConnsPerHost = maxIdleConnsPerHost + + return t +} + +func fetch(src string, req *http.Request) (*http.Request, *http.Response, error) { + //nolint:gosec // G704: fetching the include source is the purpose of the tag. + rq, err := http.NewRequestWithContext(req.Context(), http.MethodGet, sanitizeURL(src, req.URL), nil) + if err != nil { + return nil, nil, err + } + + addHeaders(headersSafe, req, rq) + + if rq.URL.Scheme == req.URL.Scheme && rq.URL.Host == req.URL.Host { + addHeaders(headersUnsafe, req, rq) + } + + response, err := client.Do(rq) //nolint:gosec // G704: same as above. + + return rq, response, err +} + +func failed(response *http.Response, err error) bool { + return err != nil || response.StatusCode >= http.StatusBadRequest +} + // Input (e.g. include src="https://domain.com/esi-include" alt="https://domain.com/alt-esi-include" />) // With or without the alt // With or without a space separator before the closing // With or without the quotes around the src/alt value. func (i *includeTag) Process(b []byte, req *http.Request) ([]byte, int) { - closeIdx := closeInclude.FindIndex(b) - - if closeIdx == nil { + closeIdx := bytes.Index(b, closeInclude) + if closeIdx == -1 { return nil, len(b) } - i.length = closeIdx[1] + i.length = closeIdx + len(closeInclude) if e := i.loadAttributes(b[8:i.length]); e != nil { return nil, len(b) } - rq, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, sanitizeURL(i.src, req.URL), nil) - addHeaders(headersSafe, req, rq) - - if rq.URL.Scheme == req.URL.Scheme && rq.URL.Host == req.URL.Host { - addHeaders(headersUnsafe, req, rq) + rq, response, err := fetch(i.src, req) + if rq == nil { + return nil, i.length } - client := &http.Client{} - response, err := client.Do(rq) - req = rq + if failed(response, err) && i.alt != "" { + if response != nil { + _ = response.Body.Close() + } - if (err != nil || response.StatusCode >= 400) && i.alt != "" { - rq, _ = http.NewRequestWithContext(context.Background(), http.MethodGet, sanitizeURL(i.alt, req.URL), nil) - addHeaders(headersSafe, req, rq) + var altRq *http.Request - if rq.URL.Scheme == req.URL.Scheme && rq.URL.Host == req.URL.Host { - addHeaders(headersUnsafe, req, rq) + altRq, response, err = fetch(i.alt, rq) + if altRq == nil { + return nil, i.length } - response, err = client.Do(rq) - req = rq + rq = altRq + + if !i.silent && failed(response, err) { + if response != nil { + _ = response.Body.Close() + } - if !i.silent && (err != nil || response.StatusCode >= 400) { return nil, len(b) } } @@ -120,23 +161,23 @@ func (i *includeTag) Process(b []byte, req *http.Request) ([]byte, int) { return nil, i.length } - var buf bytes.Buffer + body, err := io.ReadAll(response.Body) + _ = response.Body.Close() - defer response.Body.Close() - _, _ = io.Copy(&buf, response.Body) - - b = Parse(buf.Bytes(), req) + if err != nil { + return nil, i.length + } - return b, i.length + return Parse(body, rq), i.length } func (*includeTag) HasClose(b []byte) bool { - return closeInclude.FindIndex(b) != nil + return bytes.Contains(b, closeInclude) } func (*includeTag) GetClosePosition(b []byte) int { - if idx := closeInclude.FindIndex(b); idx != nil { - return idx[1] + if idx := bytes.Index(b, closeInclude); idx != -1 { + return idx + len(closeInclude) } return 0 diff --git a/esi/remove.go b/esi/remove.go index 1c9df9a..d839860 100644 --- a/esi/remove.go +++ b/esi/remove.go @@ -1,37 +1,33 @@ package esi import ( + "bytes" "net/http" - "regexp" ) const remove = "remove" -var closeRemove = regexp.MustCompile("") +var closeRemove = []byte("") type removeTag struct { *baseTag } func (r *removeTag) Process(b []byte, req *http.Request) ([]byte, int) { - closeIdx := closeRemove.FindIndex(b) - if closeIdx == nil { + end := closeEnd(b, closeRemove) + if end == 0 { return []byte{}, len(b) } - r.length = closeIdx[1] + r.length = end return []byte{}, r.length } func (*removeTag) HasClose(b []byte) bool { - return closeRemove.FindIndex(b) != nil + return bytes.Contains(b, closeRemove) } func (*removeTag) GetClosePosition(b []byte) int { - if idx := closeRemove.FindIndex(b); idx != nil { - return idx[1] - } - - return 0 + return closeEnd(b, closeRemove) } diff --git a/esi/scanner.go b/esi/scanner.go new file mode 100644 index 0000000..1ab8019 --- /dev/null +++ b/esi/scanner.go @@ -0,0 +1,225 @@ +package esi + +import ( + "bytes" + "net/http" +) + +var ( + esiPrefix = []byte("= 'a' && b[n] <= 'z' { + n++ + } + + return b[:n], n < len(b) +} + +func newTag(name []byte) Tag { + switch string(name) { + case comment: + return &commentTag{baseTag: newBaseTag()} + case choose: + return &chooseTag{baseTag: newBaseTag()} + case escape: + return &escapeTag{baseTag: newBaseTag()} + case include: + return &includeTag{baseTag: newBaseTag()} + case remove: + return &removeTag{baseTag: newBaseTag()} + case vars: + return &varsTag{baseTag: newBaseTag()} + default: + return nil + } +} + +// needsLookahead reports whether the closing sequence of t may extend beyond +// its first match (e.g. trailing whitespace), in which case a match ending at +// the very end of a non-final buffer must wait for more data. +func needsLookahead(t Tag) bool { + _, ok := t.(*commentTag) + + return ok +} + +// isAsync reports whether processing t may block on I/O and is therefore +// worth running concurrently. +func isAsync(t Tag, body []byte) bool { + switch t.(type) { + case *includeTag: + return true + case *chooseTag: + return bytes.Contains(body, includePrefix) + default: + return false + } +} + +// scan walks b, sending static content and complete tags to s in document +// order, and returns the number of bytes consumed. When final is false it +// stops before the first construct that may still be incomplete so the caller +// can retry once more data is available. +func scan(b []byte, final bool, s sink) int { + pos := 0 + + for pos < len(b) { + rest := b[pos:] + + i, kind := nextTagStart(rest, final) + if kind == kindNone { + s.static(rest) + + return len(b) + } + + if i > 0 { + s.static(rest[:i]) + } + + pos += i + if kind == kindPartial { + return pos + } + + rest = rest[i:] + + var ( + t Tag + prefix int + ) + + if kind == kindEscape { + t, prefix = &escapeTag{baseTag: newBaseTag()}, len(escapePrefix) + } else { + name, complete := readTagName(rest[len(esiPrefix):]) + if !complete && !final { + return pos + } + + if t = newTag(name); t == nil { + // Unsupported tag: emit the opening sequence verbatim. + s.static(rest[:len(esiPrefix)]) + pos += len(esiPrefix) + + continue + } + + prefix = len(esiPrefix) + } + + end := t.GetClosePosition(rest) + if end == 0 || (!final && end == len(rest) && needsLookahead(t)) { + if !final { + return pos + } + + // Unclosed tag at the end of the document: drop the remainder. + return len(b) + } + + s.tag(t, rest[prefix:end]) + pos += end + } + + return pos +} + +// process runs t on body. A panic in a tag implementation is contained so it +// cannot take down the whole process when running in its own goroutine. +func process(t Tag, body []byte, req *http.Request) (out []byte) { + defer func() { + if recover() != nil { + out = nil + } + }() + + out, _ = t.Process(body, req) + + return out +} + +func isSpace(c byte) bool { + return c == ' ' || c == '\n' +} + +// findClose locates the first occurrence of lit in b, extended backwards over +// the spaces and newlines preceding it. It is the byte-level equivalent of the +// leftmost match of `((\n| +)+)?lit`, without the regexp engine cost. +func findClose(b, lit []byte) (start, end int) { + i := bytes.Index(b, lit) + if i == -1 { + return -1, -1 + } + + start = i + for start > 0 && isSpace(b[start-1]) { + start-- + } + + return start, i + len(lit) +} + +// closeEnd returns the end of the first occurrence of lit in b, or 0. +func closeEnd(b, lit []byte) int { + if i := bytes.Index(b, lit); i != -1 { + return i + len(lit) + } + + return 0 +} diff --git a/esi/stream.go b/esi/stream.go new file mode 100644 index 0000000..074ad51 --- /dev/null +++ b/esi/stream.go @@ -0,0 +1,331 @@ +package esi + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "sync" +) + +// ErrClosed is returned when writing to a closed Stream. +var ErrClosed = errors.New("esi: write to closed stream") + +// result is the output of a tag processed concurrently. +type result struct { + done chan struct{} + data []byte +} + +// segment is a piece of output: either ready bytes or a pending result. +type segment struct { + data []byte + async *result +} + +// queue is an unbounded FIFO between the producer (Write) and the output +// goroutine. Being unbounded, the producer never blocks so every include is +// started as soon as it is parsed. Batches are swapped between two slices so +// the steady state does not allocate. +type queue struct { + mu sync.Mutex + items []segment + closed bool + notify chan struct{} +} + +func (q *queue) push(s segment) { + q.mu.Lock() + q.items = append(q.items, s) + q.mu.Unlock() + q.wake() +} + +func (q *queue) close() { + q.mu.Lock() + q.closed = true + q.mu.Unlock() + q.wake() +} + +func (q *queue) wake() { + select { + case q.notify <- struct{}{}: + default: + } +} + +// take returns every queued segment, reusing buf as the next backing array of +// the queue. It calls idle before blocking and returns false once the queue +// is closed and drained. +func (q *queue) take(buf []segment, idle func()) ([]segment, bool) { + for { + q.mu.Lock() + if len(q.items) > 0 { + batch := q.items + q.items = buf[:0] + q.mu.Unlock() + + return batch, true + } + + closed := q.closed + q.mu.Unlock() + + if closed { + return buf, false + } + + idle() + <-q.notify + } +} + +// Stream processes ESI tags on the fly. Data written to it is scanned +// incrementally: static content is forwarded as soon as possible, include +// tags are fetched concurrently and their output is written in document +// order. Close must be called once the input is complete; it waits for the +// pending tags and returns the first error encountered while writing. +// +// Write, Flush and Close must not be called concurrently. +type Stream struct { + dst io.Writer + req *http.Request + cancel context.CancelFunc + + // buf holds the unprocessed tail of the input (an incomplete tag). + buf []byte + // pending accumulates static output not yet handed to the queue. + pending []byte + + // q is nil until the first concurrent tag. Until then output is written + // synchronously by the caller, without copying. + q *queue + done chan struct{} + + dirty bool // written since the last flush; owned by the writing goroutine. + closed bool + + // err is the first write error. It is guarded by a mutex rather than an + // atomic.Pointer because Traefik's Yaegi interpreter cannot handle generic + // standard library types. + errMu sync.Mutex + err error +} + +// NewStream returns a Stream writing the processed document to dst. The +// context of req bounds the lifetime of the include requests. If dst +// implements http.Flusher (or Flush() error), it is flushed whenever the +// output has to wait for a tag or for more input. +func NewStream(dst io.Writer, req *http.Request) *Stream { + ctx, cancel := context.WithCancel(req.Context()) + + return &Stream{ + dst: dst, + req: req.WithContext(ctx), + cancel: cancel, + } +} + +// Write processes p. It never blocks on include requests. +func (s *Stream) Write(p []byte) (int, error) { + if s.closed { + return 0, ErrClosed + } + + if err := s.loadErr(); err != nil { + return 0, err + } + + data := p + if len(s.buf) > 0 { + s.buf = append(s.buf, p...) + data = s.buf + } + + n := scan(data, false, s) + // Keep the unprocessed tail. copy handles the overlap when data is s.buf. + s.buf = append(s.buf[:0], data[n:]...) + s.flushPending() + + return len(p), s.loadErr() +} + +// Flush forwards the processed output to the destination as soon as possible. +func (s *Stream) Flush() { + if s.closed { + return + } + + if s.q == nil { + s.flush() + + return + } + + // The output goroutine flushes when it becomes idle. + s.flushPending() +} + +// Close processes the remaining input, waits for the pending tags and +// releases the resources. It is safe to call it more than once. +func (s *Stream) Close() error { + if s.closed { + return s.loadErr() + } + + s.closed = true + + if len(s.buf) > 0 { + scan(s.buf, true, s) + s.buf = nil + } + + if s.q != nil { + s.flushPending() + s.q.close() + <-s.done + } + + s.cancel() + + return s.loadErr() +} + +func (s *Stream) static(b []byte) { + if len(b) == 0 { + return + } + + if s.q == nil { + s.write(b) + + return + } + + s.pending = append(s.pending, b...) +} + +func (s *Stream) tag(t Tag, body []byte) { + if !isAsync(t, body) { + s.static(process(t, body, s.req)) + + return + } + + if s.q == nil { + s.q = &queue{notify: make(chan struct{}, 1)} + s.done = make(chan struct{}) + + go s.run() + } + + s.flushPending() + + r := &result{done: make(chan struct{})} + body = bytes.Clone(body) + + go func() { + defer close(r.done) + + r.data = process(t, body, s.req) + }() + + s.q.push(segment{async: r}) +} + +func (s *Stream) flushPending() { + if s.q != nil && len(s.pending) > 0 { + s.q.push(segment{data: s.pending}) + s.pending = nil + } +} + +// run writes the queued segments in order until the queue is closed. +func (s *Stream) run() { + defer close(s.done) + + var batch []segment + + for { + var ok bool + if batch, ok = s.q.take(batch, s.flush); !ok { + return + } + + for i := range batch { + seg := &batch[i] + if r := seg.async; r != nil { + select { + case <-r.done: + default: + s.flush() + <-r.done + } + + seg.data = r.data + } + + s.write(seg.data) + } + + // Release the references (the clear builtin is unknown to Yaegi). + for i := range batch { + batch[i] = segment{} + } + } +} + +func (s *Stream) write(b []byte) { + if len(b) == 0 || s.loadErr() != nil { + return + } + + if _, err := s.dst.Write(b); err != nil { + s.setErr(err) + // Nobody will read the output anymore: abort the pending includes. + s.cancel() + + return + } + + s.dirty = true +} + +func (s *Stream) flush() { + if !s.dirty || s.loadErr() != nil { + return + } + + s.dirty = false + + switch f := s.dst.(type) { + case http.Flusher: + f.Flush() + case interface{ Flush() error }: + if err := f.Flush(); err != nil { + s.setErr(err) + s.cancel() + } + } +} + +func (s *Stream) loadErr() error { + s.errMu.Lock() + defer s.errMu.Unlock() + + return s.err +} + +// setErr records err unless an error was already recorded. +func (s *Stream) setErr(err error) { + s.errMu.Lock() + defer s.errMu.Unlock() + + if s.err == nil { + s.err = err + } +} + +var _ io.WriteCloser = (*Stream)(nil) diff --git a/esi/stream_test.go b/esi/stream_test.go new file mode 100644 index 0000000..bf28cad --- /dev/null +++ b/esi/stream_test.go @@ -0,0 +1,294 @@ +package esi_test + +import ( + "bytes" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/darkweak/go-esi/esi" +) + +// offline exercises every tag that does not need the network, including +// unsupported tags and constructs straddling chunk boundaries. +const offline = `<esi:vars>Hello from $(HTTP_HOST)</esi:vars> + + + +

before

www.example.com

after

+unsupported +$(HTTP_COOKIE{missing}|'fallback') $(QUERY_STRING{q}) +< esi not a tag ` + +func streamInChunks(t *testing.T, input []byte, req *http.Request, chunks ...int) string { + t.Helper() + + var out bytes.Buffer + + s := esi.NewStream(&out, req) + + for _, n := range chunks { + n = min(n, len(input)) + if _, err := s.Write(input[:n]); err != nil { + t.Fatal(err) + } + + input = input[n:] + } + + if _, err := s.Write(input); err != nil { + t.Fatal(err) + } + + if err := s.Close(); err != nil { + t.Fatal(err) + } + + return out.String() +} + +func Test_Stream_offline_everySplit(t *testing.T) { + t.Parallel() + + input := []byte(offline) + req := getRequest() + want := string(esi.Parse(bytes.Clone(input), req)) + + for i := range len(input) + 1 { + if got := streamInChunks(t, input, req, i); got != want { + t.Fatalf("split at %d:\nwant:\n%s\ngot:\n%s", i, want, got) + } + } + + ones := make([]int, len(input)) + for i := range ones { + ones[i] = 1 + } + + if got := streamInChunks(t, input, req, ones...); got != want { + t.Fatalf("byte by byte:\nwant:\n%s\ngot:\n%s", want, got) + } +} + +func Test_Stream_fixtures(t *testing.T) { + t.Parallel() + + for fixture, want := range expected { + if fixture == "vars" { + continue // needs a dedicated request, covered by the offline test. + } + + input := loadFromFixtures(fixture) + + ones := make([]int, len(input)) + for i := range ones { + ones[i] = 1 + } + + for name, chunks := range map[string][]int{"whole": nil, "half": {len(input) / 2}, "byte by byte": ones} { + if got := streamInChunks(t, input, getRequest(), chunks...); got != want { + t.Errorf("%s (%s):\nwant:\n%s\ngot:\n%s", fixture, name, want, got) + } + } + } +} + +// slowServer answers /?d= with after the delay. +func slowServer(t *testing.T) *httptest.Server { + t.Helper() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + d, _ := time.ParseDuration(r.URL.Query().Get("d")) + time.Sleep(d) + + _, _ = w.Write([]byte(strings.TrimPrefix(r.URL.Path, "/"))) + })) + t.Cleanup(srv.Close) + + return srv +} + +func includes(srv *httptest.Server, n int, delay time.Duration) (doc, want string) { + var d, w strings.Builder + for i := range n { + fmt.Fprintf(&d, `[%d]`, i, srv.URL, i, delay) + fmt.Fprintf(&w, `[%d]inc%d`, i, i) + } + + return d.String(), w.String() +} + +func Test_Parse_parallel(t *testing.T) { + t.Parallel() + + const ( + n = 20 + delay = 200 * time.Millisecond + ) + + srv := slowServer(t) + doc, want := includes(srv, n, delay) + + start := time.Now() + got := string(esi.Parse([]byte(doc), httptest.NewRequest(http.MethodGet, srv.URL, nil))) + elapsed := time.Since(start) + + if got != want { + t.Fatalf("want %q, got %q", want, got) + } + + if elapsed > 3*delay { + t.Fatalf("%d includes of %s took %s, they are not processed in parallel", n, delay, elapsed) + } +} + +func Test_Stream_parallel(t *testing.T) { + t.Parallel() + + const ( + n = 20 + delay = 200 * time.Millisecond + ) + + srv := slowServer(t) + doc, want := includes(srv, n, delay) + + start := time.Now() + got := streamInChunks(t, []byte(doc), httptest.NewRequest(http.MethodGet, srv.URL, nil), 7, 13, 50) + elapsed := time.Since(start) + + if got != want { + t.Fatalf("want %q, got %q", want, got) + } + + if elapsed > 3*delay { + t.Fatalf("%d includes of %s took %s, they are not processed in parallel", n, delay, elapsed) + } +} + +// recorder records the writes and flushes with their timestamp. +type recorder struct { + mu sync.Mutex + buf bytes.Buffer + flushed []string + first time.Time +} + +func (r *recorder) Write(b []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if r.first.IsZero() { + r.first = time.Now() + } + + return r.buf.Write(b) +} + +func (r *recorder) Flush() { + r.mu.Lock() + defer r.mu.Unlock() + + r.flushed = append(r.flushed, r.buf.String()) +} + +func Test_Stream_flushesBeforeSlowInclude(t *testing.T) { + t.Parallel() + + const delay = 300 * time.Millisecond + + srv := slowServer(t) + rec := &recorder{} + s := esi.NewStream(rec, httptest.NewRequest(http.MethodGet, srv.URL, nil)) + start := time.Now() + + _, _ = s.Write([]byte(`
`)) + _, _ = s.Write([]byte(`