Skip to content

Commit 3ef708c

Browse files
committed
feat: make both feed versions race together (#5287)
1 parent 07d486e commit 3ef708c

10 files changed

Lines changed: 195 additions & 81 deletions

File tree

.github/workflows/beekeeper.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,9 @@ jobs:
158158
- name: Test manifest
159159
id: manifest
160160
run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-manifest
161+
- name: Test manifest v1
162+
id: manifest-v1
163+
run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-manifest-v1
161164
- name: Test postage stamps
162165
id: postage-stamps
163166
run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks ci-postage
@@ -173,6 +176,9 @@ jobs:
173176
- name: Test act
174177
id: act
175178
run: timeout ${TIMEOUT} bash -c 'until beekeeper check --cluster-name local-dns --checks ci-act; do echo "waiting for act..."; sleep .3; done'
179+
- name: Test feeds v1
180+
id: feeds-v1
181+
run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-feed-v1
176182
- name: Test feeds
177183
id: feeds
178184
run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-feed
@@ -190,6 +196,9 @@ jobs:
190196
if ${{ steps.pushsync-chunks-2.outcome=='failure' }}; then FAILED=pushsync-chunks-2; fi
191197
if ${{ steps.retrieval.outcome=='failure' }}; then FAILED=retrieval; fi
192198
if ${{ steps.manifest.outcome=='failure' }}; then FAILED=manifest; fi
199+
if ${{ steps.manifest-v1.outcome=='failure' }}; then FAILED=manifest-v1; fi
200+
if ${{ steps.feeds.outcome=='failure' }}; then FAILED=feeds; fi
201+
if ${{ steps.feeds-v1.outcome=='failure' }}; then FAILED=feeds-v1; fi
193202
if ${{ steps.act.outcome=='failure' }}; then FAILED=act; fi
194203
curl -sSf -X POST -H "Content-Type: application/json" -d "{\"text\": \"**${RUN_TYPE}** Beekeeper Error\nBranch: \`${{ github.head_ref }}\`\nUser: @${{ github.event.pull_request.user.login }}\nDebugging artifacts: [click](https://$BUCKET_NAME.$AWS_ENDPOINT/artifacts_$VERTAG.tar.gz)\nStep failed: \`${FAILED}\`\"}" https://beehive.ethswarm.org/hooks/${{ secrets.TUNSHELL_KEY }}
195204
echo "Failed test: ${FAILED}"

openapi/Swarm.yaml

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -435,7 +435,6 @@ paths:
435435
$ref: "SwarmCommon.yaml#/components/schemas/SwarmReference"
436436
required: true
437437
description: Swarm address of content
438-
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmFeedLegacyResolve"
439438
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache"
440439
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter"
441440
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyFallbackModeParameter"
@@ -454,6 +453,11 @@ paths:
454453
schema:
455454
type: string
456455
format: binary
456+
headers:
457+
"swarm-feed-resolved-version":
458+
$ref: "SwarmCommon.yaml#/components/headers/SwarmFeedResolvedVersion"
459+
460+
457461
"400":
458462
$ref: "SwarmCommon.yaml#/components/responses/400"
459463
"404":
@@ -504,7 +508,6 @@ paths:
504508
type: string
505509
required: true
506510
description: Path to the file in the collection.
507-
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmFeedLegacyResolve"
508511
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter"
509512
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyFallbackModeParameter"
510513
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmChunkRetrievalTimeoutParameter"
@@ -516,6 +519,9 @@ paths:
516519
schema:
517520
type: string
518521
format: binary
522+
headers:
523+
"swarm-feed-resolved-version":
524+
$ref: "SwarmCommon.yaml#/components/headers/SwarmFeedResolvedVersion"
519525

520526
"400":
521527
$ref: "SwarmCommon.yaml#/components/responses/400"
@@ -1032,7 +1038,6 @@ paths:
10321038
$ref: "SwarmCommon.yaml#/components/schemas/FeedType"
10331039
required: false
10341040
description: "Feed indexing scheme (default: sequence)"
1035-
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmFeedLegacyResolve"
10361041
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmOnlyRootChunkParameter"
10371042
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache"
10381043
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter"
@@ -1048,6 +1053,9 @@ paths:
10481053
$ref: "SwarmCommon.yaml#/components/headers/SwarmFeedIndex"
10491054
"swarm-feed-index-next":
10501055
$ref: "SwarmCommon.yaml#/components/headers/SwarmFeedIndexNext"
1056+
"swarm-feed-resolved-version":
1057+
$ref: "SwarmCommon.yaml#/components/headers/SwarmFeedResolvedVersion"
1058+
10511059
content:
10521060
application/octet-stream:
10531061
schema:

openapi/SwarmCommon.yaml

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1071,6 +1071,13 @@ components:
10711071
schema:
10721072
type: string
10731073

1074+
SwarmFeedResolvedVersion:
1075+
schema:
1076+
type: string
1077+
required: false
1078+
description: "Indicates which feed version was resolved (v1 or v2)"
1079+
1080+
10741081
parameters:
10751082
GasPriceParameter:
10761083
in: header
@@ -1276,14 +1283,6 @@ components:
12761283
required: false
12771284
description: "ACT history Unix timestamp"
12781285

1279-
SwarmFeedLegacyResolve:
1280-
in: query
1281-
name: swarm-feed-legacy-resolve
1282-
schema:
1283-
type: boolean
1284-
required: false
1285-
description: "Resolves feed payloads in legacy structure (timestamp, content address)."
1286-
12871286
responses:
12881287
"200":
12891288
description: OK.

pkg/api/api.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ const (
7979
SwarmSocSignatureHeader = "Swarm-Soc-Signature"
8080
SwarmFeedIndexHeader = "Swarm-Feed-Index"
8181
SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next"
82+
SwarmFeedResolvedVersionHeader = "Swarm-Feed-Resolved-Version"
8283
SwarmOnlyRootChunk = "Swarm-Only-Root-Chunk"
8384
SwarmCollectionHeader = "Swarm-Collection"
8485
SwarmPostageBatchIdHeader = "Swarm-Postage-Batch-Id"

pkg/api/bzz.go

Lines changed: 117 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -337,15 +337,7 @@ func (s *Service) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) {
337337
paths.Path = strings.TrimRight(paths.Path, "/") + "/" // NOTE: leave one slash if there was some.
338338
}
339339

340-
queries := struct {
341-
FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"`
342-
}{}
343-
if response := s.mapStructure(r.URL.Query(), &queries); response != nil {
344-
response("invalid query params", logger, w)
345-
return
346-
}
347-
348-
s.serveReference(logger, address, paths.Path, w, r, false, queries.FeedLegacyResolve)
340+
s.serveReference(logger, address, paths.Path, w, r, false)
349341
}
350342

351343
func (s *Service) bzzHeadHandler(w http.ResponseWriter, r *http.Request) {
@@ -360,14 +352,6 @@ func (s *Service) bzzHeadHandler(w http.ResponseWriter, r *http.Request) {
360352
return
361353
}
362354

363-
queries := struct {
364-
FeedLegacyResolve bool `map:"swarm-feed-legacy-resolve"`
365-
}{}
366-
if response := s.mapStructure(r.URL.Query(), &queries); response != nil {
367-
response("invalid query params", logger, w)
368-
return
369-
}
370-
371355
address := paths.Address
372356
if v := getAddressFromContext(r.Context()); !v.IsZero() {
373357
address = v
@@ -377,10 +361,119 @@ func (s *Service) bzzHeadHandler(w http.ResponseWriter, r *http.Request) {
377361
paths.Path = strings.TrimRight(paths.Path, "/") + "/" // NOTE: leave one slash if there was some.
378362
}
379363

380-
s.serveReference(logger, address, paths.Path, w, r, true, queries.FeedLegacyResolve)
364+
s.serveReference(logger, address, paths.Path, w, r, true)
381365
}
382366

383-
func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathVar string, w http.ResponseWriter, r *http.Request, headerOnly bool, feedLegacyResolve bool) {
367+
type getWrappedResult struct {
368+
ch swarm.Chunk
369+
v1 bool // indicates whether the feed that was resolved is v1. false if v2
370+
err error
371+
}
372+
373+
// resolveFeed races the resolution of both types of feeds.
374+
// figure out if its a v1 or v2 chunk.
375+
// it returns the first correct feed found, the type found ("v1" or "v2") or an error.
376+
func (s *Service) resolveFeed(ctx context.Context, getter storage.Getter, ch swarm.Chunk) (swarm.Chunk, string, error) {
377+
innerCtx, cancel := context.WithCancel(ctx)
378+
defer cancel()
379+
getWrapped := func(v1 bool) chan getWrappedResult {
380+
ret := make(chan getWrappedResult)
381+
go func() {
382+
wc, err := feeds.GetWrappedChunk(innerCtx, getter, ch, v1)
383+
if err != nil {
384+
select {
385+
case ret <- getWrappedResult{nil, v1, err}:
386+
return
387+
case <-innerCtx.Done():
388+
return
389+
}
390+
}
391+
392+
// here we just check whether the address is retrievable.
393+
// if it returns an error we send that over the channel, otherwise
394+
// we send the wc chunk back to the caller so that the feed can be
395+
// dereferenced.
396+
_, err = getter.Get(innerCtx, wc.Address())
397+
if err != nil {
398+
select {
399+
case ret <- getWrappedResult{wc, v1, err}:
400+
return
401+
case <-innerCtx.Done():
402+
return
403+
}
404+
}
405+
select {
406+
case ret <- getWrappedResult{wc, v1, nil}:
407+
return
408+
case <-innerCtx.Done():
409+
return
410+
}
411+
}()
412+
return ret
413+
}
414+
isV1, err := feeds.IsV1Payload(ch)
415+
if err != nil {
416+
return nil, "", err
417+
}
418+
// if we have v1 length, it means there's ambiguity so we
419+
// should fetch both feed versions. if the length isn't v1
420+
// then we should only try to fetch v2.
421+
var (
422+
v1C, v2C chan getWrappedResult
423+
both = false
424+
)
425+
if isV1 {
426+
both = true
427+
v1C = getWrapped(true)
428+
v2C = getWrapped(false)
429+
} else {
430+
v2C = getWrapped(false)
431+
}
432+
433+
// closure to handle processing one channel then the other.
434+
// the "resolving" parameter is meant to tell the closure which feed type is in the result struct
435+
// which in turns allows it to return which feed type was resolved.
436+
processChanOutput := func(resolving string, result getWrappedResult, other chan getWrappedResult) (swarm.Chunk, string, error) {
437+
defer cancel()
438+
if !both {
439+
if resolving == "v2" {
440+
return result.ch, resolving, nil
441+
}
442+
return result.ch, resolving, result.err
443+
}
444+
// both are being checked. if there's no err return the chunk
445+
// otherwise wait for the other channel
446+
if result.err == nil {
447+
return result.ch, resolving, nil
448+
}
449+
if resolving == "v1" {
450+
resolving = "v2"
451+
} else {
452+
resolving = "v1"
453+
}
454+
// wait for the other one
455+
select {
456+
case result := <-other:
457+
if !result.v1 {
458+
// resolving v2
459+
return result.ch, resolving, nil
460+
}
461+
return result.ch, resolving, result.err
462+
case <-innerCtx.Done():
463+
return nil, "", ctx.Err()
464+
}
465+
}
466+
select {
467+
case v1r := <-v1C:
468+
return processChanOutput("v1", v1r, v2C)
469+
case v2r := <-v2C:
470+
return processChanOutput("v2", v2r, v1C)
471+
case <-innerCtx.Done():
472+
return nil, "", ctx.Err()
473+
}
474+
}
475+
476+
func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathVar string, w http.ResponseWriter, r *http.Request, headerOnly bool) {
384477
loggerV1 := logger.V(1).Build()
385478

386479
headers := struct {
@@ -415,7 +508,6 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV
415508
jsonhttp.BadRequest(w, "could not parse headers")
416509
return
417510
}
418-
419511
FETCH:
420512
// read manifest entry
421513
m, err := manifest.NewDefaultManifestReference(
@@ -449,7 +541,8 @@ FETCH:
449541
jsonhttp.NotFound(w, "no update found")
450542
return
451543
}
452-
wc, err := feeds.GetWrappedChunk(ctx, s.storer.Download(cache), ch, feedLegacyResolve)
544+
545+
wc, feedVer, err := s.resolveFeed(ctx, s.storer.Download(cache), ch)
453546
if err != nil {
454547
if errors.Is(err, feeds.ErrNotLegacyPayload) {
455548
logger.Debug("bzz: download: feed is not a legacy payload")
@@ -468,10 +561,10 @@ FETCH:
468561
jsonhttp.InternalServerError(w, "mapStructure feed update")
469562
return
470563
}
564+
471565
address = wc.Address()
472566
// modify ls and init with non-existing wrapped chunk
473567
ls = loadsave.NewReadonlyWithRootCh(s.storer.Download(cache), s.storer.Cache(), wc, rLevel)
474-
475568
feedDereferenced = true
476569
curBytes, err := cur.MarshalBinary()
477570
if err != nil {
@@ -482,6 +575,7 @@ FETCH:
482575
}
483576

484577
w.Header().Set(SwarmFeedIndexHeader, hex.EncodeToString(curBytes))
578+
w.Header().Set(SwarmFeedResolvedVersionHeader, feedVer)
485579
// this header might be overriding others. handle with care. in the future
486580
// we should implement an append functionality for this specific header,
487581
// since different parts of handlers might be overriding others' values
@@ -490,7 +584,6 @@ FETCH:
490584
goto FETCH
491585
}
492586
}
493-
494587
if pathVar == "" {
495588
loggerV1.Debug("bzz download: handle empty path", "address", address)
496589

@@ -505,6 +598,7 @@ FETCH:
505598
return
506599
}
507600
}
601+
508602
logger.Debug("bzz download: address not found or incorrect", "address", address, "path", pathVar)
509603
logger.Error(nil, "address not found or incorrect")
510604
jsonhttp.NotFound(w, "address not found or incorrect")

pkg/api/bzz_test.go

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -764,12 +764,8 @@ func TestFeedIndirection(t *testing.T) {
764764
Logger: logger,
765765
Post: mockpost.New(mockpost.WithAcceptAll()),
766766
})
767-
bzzDownloadResource = func(addr, path string, legacyFeed bool) string {
767+
bzzDownloadResource = func(addr, path string) string {
768768
values := url.Values{}
769-
if legacyFeed {
770-
values.Set("swarm-feed-legacy-resolve", strconv.FormatBool(legacyFeed))
771-
}
772-
773769
baseURL := "/bzz/" + addr + "/" + path
774770
if len(values) > 0 {
775771
return baseURL + "?" + values.Encode()
@@ -835,7 +831,6 @@ func TestFeedIndirection(t *testing.T) {
835831

836832
t.Run("legacy feed", func(t *testing.T) {
837833
feedUpdate := toChunk(t, 121212, resp.Reference.Bytes())
838-
839834
var (
840835
look = newMockLookup(-1, 0, feedUpdate, nil, &id{}, nil)
841836
factory = newMockFactory(look)
@@ -846,16 +841,15 @@ func TestFeedIndirection(t *testing.T) {
846841
Feeds: factory,
847842
})
848843

849-
jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", true), http.StatusOK,
844+
jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), ""), http.StatusOK,
850845
jsonhttptest.WithExpectedResponse(updateData),
851846
jsonhttptest.WithExpectedContentLength(len(updateData)),
852847
jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader),
853848
jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader),
854849
jsonhttptest.WithExpectedResponseHeader(api.ContentDispositionHeader, `inline; filename="index.html"`),
855850
jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "text/html; charset=utf-8"),
851+
jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedResolvedVersionHeader, "v1"),
856852
)
857-
858-
jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", false), http.StatusNotFound)
859853
})
860854

861855
t.Run("wrapped feed", func(t *testing.T) {
@@ -876,16 +870,15 @@ func TestFeedIndirection(t *testing.T) {
876870
Feeds: factory,
877871
})
878872

879-
jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", false), http.StatusOK,
873+
jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), ""), http.StatusOK,
880874
jsonhttptest.WithExpectedResponse(updateData),
881875
jsonhttptest.WithExpectedContentLength(len(updateData)),
882876
jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader),
883877
jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.ContentDispositionHeader),
884878
jsonhttptest.WithExpectedResponseHeader(api.ContentDispositionHeader, `inline; filename="index.html"`),
885879
jsonhttptest.WithExpectedResponseHeader(api.ContentTypeHeader, "text/html; charset=utf-8"),
880+
jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedResolvedVersionHeader, "v2"),
886881
)
887-
888-
jsonhttptest.Request(t, client, http.MethodGet, bzzDownloadResource(manifRef.String(), "", true), http.StatusBadRequest)
889882
})
890883
}
891884

0 commit comments

Comments
 (0)