Skip to content

Commit 1a7afbc

Browse files
committed
feat(search): index and expose the motion photo facet
1 parent 541a2b7 commit 1a7afbc

12 files changed

Lines changed: 443 additions & 116 deletions

File tree

protogen/gen/opencloud/messages/search/v0/search.pb.go

Lines changed: 189 additions & 86 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

protogen/gen/opencloud/messages/search/v0/search.pb.web.go

Lines changed: 36 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

protogen/gen/opencloud/services/search/v0/search.swagger.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,9 @@
288288
"items": {
289289
"type": "string"
290290
}
291+
},
292+
"motionPhoto": {
293+
"$ref": "#/definitions/v0MotionPhoto"
291294
}
292295
}
293296
},
@@ -352,6 +355,23 @@
352355
}
353356
}
354357
},
358+
"v0MotionPhoto": {
359+
"type": "object",
360+
"properties": {
361+
"version": {
362+
"type": "integer",
363+
"format": "int32"
364+
},
365+
"presentationTimestampUs": {
366+
"type": "string",
367+
"format": "int64"
368+
},
369+
"videoSize": {
370+
"type": "string",
371+
"format": "int64"
372+
}
373+
}
374+
},
355375
"v0Photo": {
356376
"type": "object",
357377
"properties": {

protogen/proto/opencloud/messages/search/v0/search.proto

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@ message Photo {
5959
optional google.protobuf.Timestamp takenDateTime = 9;
6060
}
6161

62+
message MotionPhoto {
63+
optional int32 version = 1;
64+
optional int64 presentationTimestampUs = 2;
65+
optional int64 videoSize = 3;
66+
}
67+
6268
message Entity {
6369
Reference ref = 1;
6470
ResourceID id = 2;
@@ -80,6 +86,7 @@ message Entity {
8086
Image image = 18;
8187
Photo photo = 19;
8288
repeated string favorites = 20;
89+
MotionPhoto motionPhoto = 21;
8390
}
8491

8592
message Match {

services/graph/pkg/service/v0/driveitems.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,7 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
456456
driveItem.Image = metadataToFacet[libregraph.Image](metadata, "image")
457457
driveItem.Location = metadataToFacet[libregraph.GeoCoordinates](metadata, "location")
458458
driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo")
459+
driveItem.LibreGraphMotionPhoto = metadataToFacet[libregraph.MotionPhoto](metadata, "motionPhoto")
459460
}
460461

461462
return driveItem, nil

services/search/pkg/bleve/backend.go

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -127,20 +127,21 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
127127
ResourceId: resourceIDtoSearchID(rootID),
128128
Path: getFieldValue[string](hit.Fields, "Path"),
129129
},
130-
Id: resourceIDtoSearchID(rID),
131-
Name: getFieldValue[string](hit.Fields, "Name"),
132-
ParentId: resourceIDtoSearchID(pID),
133-
Size: uint64(getFieldValue[float64](hit.Fields, "Size")),
134-
Type: uint64(getFieldValue[float64](hit.Fields, "Type")),
135-
MimeType: getFieldValue[string](hit.Fields, "MimeType"),
136-
Deleted: getFieldValue[bool](hit.Fields, "Deleted"),
137-
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
138-
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
139-
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
140-
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
141-
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
142-
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
143-
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
130+
Id: resourceIDtoSearchID(rID),
131+
Name: getFieldValue[string](hit.Fields, "Name"),
132+
ParentId: resourceIDtoSearchID(pID),
133+
Size: uint64(getFieldValue[float64](hit.Fields, "Size")),
134+
Type: uint64(getFieldValue[float64](hit.Fields, "Type")),
135+
MimeType: getFieldValue[string](hit.Fields, "MimeType"),
136+
Deleted: getFieldValue[bool](hit.Fields, "Deleted"),
137+
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
138+
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
139+
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
140+
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
141+
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
142+
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
143+
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
144+
MotionPhoto: hitToFacet[searchMessage.MotionPhoto](hit.Fields, "motionPhoto"),
144145
},
145146
}
146147

services/search/pkg/content/content.go

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,19 @@ func init() {
1515
// Document wraps all resource meta fields,
1616
// it is used as a content extraction result.
1717
type Document struct {
18-
Title string `json:"Title"`
19-
Name string `json:"Name"`
20-
Content string `json:"Content"`
21-
Size uint64 `json:"Size"`
22-
Mtime *time.Time `json:"Mtime"`
23-
MimeType string `json:"MimeType"`
24-
Tags []string `json:"Tags"`
25-
Favorites []string `json:"Favorites"`
26-
Audio *libregraph.Audio `json:"audio,omitempty"`
27-
Image *libregraph.Image `json:"image,omitempty"`
28-
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
29-
Photo *libregraph.Photo `json:"photo,omitempty"`
18+
Title string `json:"Title"`
19+
Name string `json:"Name"`
20+
Content string `json:"Content"`
21+
Size uint64 `json:"Size"`
22+
Mtime *time.Time `json:"Mtime"`
23+
MimeType string `json:"MimeType"`
24+
Tags []string `json:"Tags"`
25+
Favorites []string `json:"Favorites"`
26+
Audio *libregraph.Audio `json:"audio,omitempty"`
27+
Image *libregraph.Image `json:"image,omitempty"`
28+
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
29+
Photo *libregraph.Photo `json:"photo,omitempty"`
30+
MotionPhoto *libregraph.MotionPhoto `json:"motionPhoto,omitempty"`
3031
}
3132

3233
func CleanString(content, langCode string) string {

services/search/pkg/content/tika.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,13 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
9191
doc.Image = t.getImage(meta)
9292
doc.Photo = t.getPhoto(meta)
9393
doc.Audio = t.getAudio(meta)
94+
doc.MotionPhoto = t.getMotionPhoto(meta)
95+
}
96+
97+
// verify against the file itself: a shared motion photo can keep the XMP but
98+
// lose the appended video, which would leave an unplayable facet.
99+
if doc.MotionPhoto != nil && !t.motionPhotoHasVideo(ctx, ri, doc.MotionPhoto.GetVideoSize()) {
100+
doc.MotionPhoto = nil
94101
}
95102

96103
if langCode, _ := t.tika.LanguageString(ctx, doc.Content); langCode != "" && t.CleanStopWords {
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
package content
2+
3+
import (
4+
"context"
5+
"io"
6+
"strconv"
7+
"strings"
8+
9+
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
10+
libregraph "github.com/opencloud-eu/libre-graph-api-go"
11+
)
12+
13+
// motionPhotoVideoSignatureLen is the number of trailing bytes we read to confirm
14+
// an actual video is present. Enough to cover the ISO base media (MP4) box size
15+
// and the "ftyp" box type at bytes [4:8].
16+
const motionPhotoVideoSignatureLen = 12
17+
18+
// getMotionPhoto reads Google Motion Photo XMP, which Tika exposes under the
19+
// canonical Camera/Container prefixes. It covers both the current MotionPhoto
20+
// scheme and the legacy MicroVideo scheme. videoSize (the embedded video's byte
21+
// length, needed to range-fetch it) is required, so the facet is dropped without it.
22+
func (t Tika) getMotionPhoto(meta map[string][]string) *libregraph.MotionPhoto {
23+
var motionPhoto *libregraph.MotionPhoto
24+
initMotionPhoto := func() {
25+
if motionPhoto == nil {
26+
motionPhoto = libregraph.NewMotionPhoto()
27+
}
28+
}
29+
30+
if v, err := getFirstValue(meta, "Camera:MotionPhotoVersion", "Camera:MicroVideoVersion"); err == nil {
31+
if i, err := strconv.ParseInt(v, 0, 32); err == nil {
32+
initMotionPhoto()
33+
motionPhoto.SetVersion(int32(i))
34+
}
35+
}
36+
37+
if v, err := getFirstValue(meta, "Camera:MotionPhotoPresentationTimestampUs", "Camera:MicroVideoPresentationTimestampUs"); err == nil {
38+
if i, err := strconv.ParseInt(v, 0, 64); err == nil {
39+
initMotionPhoto()
40+
motionPhoto.SetPresentationTimestampUs(i)
41+
}
42+
}
43+
44+
if size, ok := motionPhotoVideoSize(meta); ok {
45+
initMotionPhoto()
46+
motionPhoto.SetVideoSize(size)
47+
}
48+
49+
if motionPhoto == nil || !motionPhoto.HasVideoSize() {
50+
return nil
51+
}
52+
return motionPhoto
53+
}
54+
55+
// motionPhotoVideoSize returns the embedded video's byte length. Legacy files
56+
// carry it as the MicroVideo offset (bytes from EOF to the video start, which
57+
// equals its length); current files expose it as the length of the Container
58+
// item whose semantic is "MotionPhoto".
59+
func motionPhotoVideoSize(meta map[string][]string) (int64, bool) {
60+
if v, err := getFirstValue(meta, "Camera:MicroVideoOffset"); err == nil {
61+
if i, err := strconv.ParseInt(v, 0, 64); err == nil {
62+
return i, true
63+
}
64+
}
65+
for k, vals := range meta {
66+
if !strings.HasSuffix(k, "/Item:Semantic") || len(vals) == 0 || vals[0] != "MotionPhoto" {
67+
continue
68+
}
69+
if v, err := getFirstValue(meta, strings.TrimSuffix(k, "/Item:Semantic")+"/Item:Length"); err == nil {
70+
if i, err := strconv.ParseInt(v, 0, 64); err == nil {
71+
return i, true
72+
}
73+
}
74+
}
75+
return 0, false
76+
}
77+
78+
// looksLikeMP4 reports whether buf begins with an ISO base media (MP4/QuickTime)
79+
// "ftyp" box, which Google Motion Photo and legacy MicroVideo clips start with.
80+
func looksLikeMP4(buf []byte) bool {
81+
return len(buf) >= 8 && string(buf[4:8]) == "ftyp"
82+
}
83+
84+
// motionPhotoHasVideo confirms that the file actually contains the embedded video
85+
// the XMP advertises. A photos.google.com share strips the appended video but
86+
// keeps the XMP, which would otherwise make us expose an unplayable facet. The
87+
// video is appended at the end, so it starts at fileSize-videoSize; we read a few
88+
// bytes there and require an MP4 signature.
89+
func (t Tika) motionPhotoHasVideo(ctx context.Context, ri *provider.ResourceInfo, videoSize int64) bool {
90+
size := int64(ri.GetSize())
91+
if videoSize <= 0 || videoSize >= size {
92+
return false
93+
}
94+
95+
rc, err := t.RetrieveRange(ctx, ri.GetId(), size-videoSize, motionPhotoVideoSignatureLen)
96+
if err != nil {
97+
t.logger.Debug().Err(err).Interface("ResourceID", ri.GetId()).Msg("could not read motion photo video header, dropping facet")
98+
return false
99+
}
100+
defer rc.Close()
101+
102+
buf := make([]byte, motionPhotoVideoSignatureLen)
103+
n, _ := io.ReadFull(rc, buf)
104+
return looksLikeMP4(buf[:n])
105+
}
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package content
2+
3+
import (
4+
. "github.com/onsi/ginkgo/v2"
5+
. "github.com/onsi/gomega"
6+
libregraph "github.com/opencloud-eu/libre-graph-api-go"
7+
)
8+
9+
var _ = Describe("getMotionPhoto", func() {
10+
It("maps the current MotionPhoto XMP scheme (container item length)", func() {
11+
mp := Tika{}.getMotionPhoto(map[string][]string{
12+
"Camera:MotionPhotoVersion": {"1"},
13+
"Camera:MotionPhotoPresentationTimestampUs": {"1500000"},
14+
"Container:Directory/Item[2]/Item:Semantic": {"MotionPhoto"},
15+
"Container:Directory/Item[2]/Item:Length": {"1048576"},
16+
})
17+
Expect(mp).ToNot(BeNil())
18+
Expect(mp.Version).To(Equal(libregraph.PtrInt32(1)))
19+
Expect(mp.PresentationTimestampUs).To(Equal(libregraph.PtrInt64(1500000)))
20+
Expect(mp.VideoSize).To(Equal(libregraph.PtrInt64(1048576)))
21+
})
22+
23+
It("maps the legacy MicroVideo XMP scheme (offset is the length)", func() {
24+
mp := Tika{}.getMotionPhoto(map[string][]string{
25+
"Camera:MicroVideoVersion": {"1"},
26+
"Camera:MicroVideoPresentationTimestampUs": {"1500000"},
27+
"Camera:MicroVideoOffset": {"2097152"},
28+
})
29+
Expect(mp).ToNot(BeNil())
30+
Expect(mp.Version).To(Equal(libregraph.PtrInt32(1)))
31+
Expect(mp.PresentationTimestampUs).To(Equal(libregraph.PtrInt64(1500000)))
32+
Expect(mp.VideoSize).To(Equal(libregraph.PtrInt64(2097152)))
33+
})
34+
35+
It("drops the facet without a video size", func() {
36+
Expect(Tika{}.getMotionPhoto(map[string][]string{
37+
"Camera:MotionPhotoVersion": {"1"},
38+
})).To(BeNil())
39+
})
40+
41+
It("returns nil when no motion photo metadata is present", func() {
42+
Expect(Tika{}.getMotionPhoto(map[string][]string{})).To(BeNil())
43+
})
44+
})

0 commit comments

Comments
 (0)