Skip to content

Commit e71f348

Browse files
committed
feat(search): index and expose the motion photo facet
1 parent c141b4a commit e71f348

15 files changed

Lines changed: 663 additions & 116 deletions

File tree

protogen/gen/opencloud/messages/search/v0/search.pb.go

Lines changed: 189 additions & 86 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

protogen/gen/opencloud/messages/search/v0/search.pb.web.go

Lines changed: 36 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

protogen/gen/opencloud/services/search/v0/search.swagger.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,9 @@
298298
"items": {
299299
"type": "string"
300300
}
301+
},
302+
"motionPhoto": {
303+
"$ref": "#/definitions/v0MotionPhoto"
301304
}
302305
}
303306
},
@@ -390,6 +393,23 @@
390393
}
391394
}
392395
},
396+
"v0MotionPhoto": {
397+
"type": "object",
398+
"properties": {
399+
"version": {
400+
"type": "integer",
401+
"format": "int32"
402+
},
403+
"presentationTimestampUs": {
404+
"type": "string",
405+
"format": "int64"
406+
},
407+
"videoSize": {
408+
"type": "string",
409+
"format": "int64"
410+
}
411+
}
412+
},
393413
"v0Photo": {
394414
"type": "object",
395415
"properties": {

protogen/proto/opencloud/messages/search/v0/search.proto

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@ message Photo {
5959
optional google.protobuf.Timestamp takenDateTime = 9;
6060
}
6161

62+
message MotionPhoto {
63+
optional int32 version = 1;
64+
optional int64 presentationTimestampUs = 2;
65+
optional int64 videoSize = 3;
66+
}
67+
6268
message Entity {
6369
Reference ref = 1;
6470
ResourceID id = 2;
@@ -80,6 +86,7 @@ message Entity {
8086
Image image = 18;
8187
Photo photo = 19;
8288
repeated string favorites = 20;
89+
MotionPhoto motionPhoto = 21;
8390
}
8491

8592
message Match {

services/graph/pkg/service/v0/driveitems.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,7 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
456456
driveItem.Image = metadataToFacet[libregraph.Image](metadata, "image")
457457
driveItem.Location = metadataToFacet[libregraph.GeoCoordinates](metadata, "location")
458458
driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo")
459+
driveItem.LibreGraphMotionPhoto = metadataToFacet[libregraph.MotionPhoto](metadata, "motionPhoto")
459460
}
460461

461462
return driveItem, nil

services/search/pkg/bleve/backend.go

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -125,20 +125,21 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
125125
ResourceId: resourceIDtoSearchID(rootID),
126126
Path: getFieldValue[string](hit.Fields, "Path"),
127127
},
128-
Id: resourceIDtoSearchID(rID),
129-
Name: getFieldValue[string](hit.Fields, "Name"),
130-
ParentId: resourceIDtoSearchID(pID),
131-
Size: uint64(getFieldValue[float64](hit.Fields, "Size")),
132-
Type: uint64(getFieldValue[float64](hit.Fields, "Type")),
133-
MimeType: getFieldValue[string](hit.Fields, "MimeType"),
134-
Deleted: getFieldValue[bool](hit.Fields, "Deleted"),
135-
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
136-
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
137-
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
138-
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
139-
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
140-
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
141-
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
128+
Id: resourceIDtoSearchID(rID),
129+
Name: getFieldValue[string](hit.Fields, "Name"),
130+
ParentId: resourceIDtoSearchID(pID),
131+
Size: uint64(getFieldValue[float64](hit.Fields, "Size")),
132+
Type: uint64(getFieldValue[float64](hit.Fields, "Type")),
133+
MimeType: getFieldValue[string](hit.Fields, "MimeType"),
134+
Deleted: getFieldValue[bool](hit.Fields, "Deleted"),
135+
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
136+
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
137+
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
138+
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
139+
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
140+
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
141+
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
142+
MotionPhoto: hitToFacet[searchMessage.MotionPhoto](hit.Fields, "motionPhoto"),
142143
},
143144
}
144145

services/search/pkg/bleve/testdata/mapping.golden.json

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,51 @@
760760
}
761761
]
762762
},
763+
"motionPhoto": {
764+
"enabled": true,
765+
"dynamic": true,
766+
"properties": {
767+
"presentationTimestampUs": {
768+
"enabled": true,
769+
"dynamic": true,
770+
"fields": [
771+
{
772+
"type": "number",
773+
"store": true,
774+
"index": true,
775+
"include_in_all": true,
776+
"docvalues": true
777+
}
778+
]
779+
},
780+
"version": {
781+
"enabled": true,
782+
"dynamic": true,
783+
"fields": [
784+
{
785+
"type": "number",
786+
"store": true,
787+
"index": true,
788+
"include_in_all": true,
789+
"docvalues": true
790+
}
791+
]
792+
},
793+
"videoSize": {
794+
"enabled": true,
795+
"dynamic": true,
796+
"fields": [
797+
{
798+
"type": "number",
799+
"store": true,
800+
"index": true,
801+
"include_in_all": true,
802+
"docvalues": true
803+
}
804+
]
805+
}
806+
}
807+
},
763808
"photo": {
764809
"enabled": true,
765810
"dynamic": true,

services/search/pkg/content/content.go

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,19 @@ func init() {
1515
// Document wraps all resource meta fields,
1616
// it is used as a content extraction result.
1717
type Document struct {
18-
Title string `json:"Title"`
19-
Name string `json:"Name"`
20-
Content string `json:"Content"`
21-
Size uint64 `json:"Size"`
22-
Mtime *time.Time `json:"Mtime,omitempty"`
23-
MimeType string `json:"MimeType"`
24-
Tags []string `json:"Tags"`
25-
Favorites []string `json:"Favorites"`
26-
Audio *libregraph.Audio `json:"audio,omitempty"`
27-
Image *libregraph.Image `json:"image,omitempty"`
28-
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
29-
Photo *libregraph.Photo `json:"photo,omitempty"`
18+
Title string `json:"Title"`
19+
Name string `json:"Name"`
20+
Content string `json:"Content"`
21+
Size uint64 `json:"Size"`
22+
Mtime *time.Time `json:"Mtime,omitempty"`
23+
MimeType string `json:"MimeType"`
24+
Tags []string `json:"Tags"`
25+
Favorites []string `json:"Favorites"`
26+
Audio *libregraph.Audio `json:"audio,omitempty"`
27+
Image *libregraph.Image `json:"image,omitempty"`
28+
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
29+
Photo *libregraph.Photo `json:"photo,omitempty"`
30+
MotionPhoto *libregraph.MotionPhoto `json:"motionPhoto,omitempty"`
3031
}
3132

3233
func CleanString(content, langCode string) string {

services/search/pkg/content/tika.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,15 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
108108
if v := t.getAudio(meta); v != nil {
109109
doc.Audio = v
110110
}
111+
if v := t.getMotionPhoto(meta); v != nil {
112+
doc.MotionPhoto = v
113+
}
114+
}
115+
116+
// verify against the file itself: a shared motion photo can keep the XMP but
117+
// lose the appended video, which would leave an unplayable facet.
118+
if doc.MotionPhoto != nil && !t.motionPhotoHasVideo(ctx, ri, doc.MotionPhoto.GetVideoSize()) {
119+
doc.MotionPhoto = nil
111120
}
112121

113122
if langCode, _ := t.tika.LanguageString(ctx, doc.Content); langCode != "" && t.CleanStopWords {
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
package content
2+
3+
import (
4+
"context"
5+
"io"
6+
"sort"
7+
"strconv"
8+
"strings"
9+
10+
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
11+
libregraph "github.com/opencloud-eu/libre-graph-api-go"
12+
)
13+
14+
// motionPhotoVideoSignatureLen is the number of trailing bytes we read to confirm
15+
// an actual video is present. Enough to cover the ISO base media (MP4) box size
16+
// and the "ftyp" box type at bytes [4:8].
17+
const motionPhotoVideoSignatureLen = 12
18+
19+
// getMotionPhoto reads Google Motion Photo XMP, which Tika exposes under the
20+
// canonical Camera/Container prefixes. It covers both the current MotionPhoto
21+
// scheme and the legacy MicroVideo scheme. videoSize (the embedded video's byte
22+
// length, needed to range-fetch it) is required, so the facet is dropped without it.
23+
func (t Tika) getMotionPhoto(meta map[string][]string) *libregraph.MotionPhoto {
24+
// per the spec only a MotionPhoto/MicroVideo marker of 1 means motion
25+
// photo, every other value is "treat as a still image". An absent marker
26+
// is tolerated on purpose, the byte-level video check below decides.
27+
if v, err := getFirstValue(meta, "Camera:MotionPhoto", "Camera:MicroVideo"); err == nil && v != "1" {
28+
return nil
29+
}
30+
31+
var motionPhoto *libregraph.MotionPhoto
32+
initMotionPhoto := func() {
33+
if motionPhoto == nil {
34+
motionPhoto = libregraph.NewMotionPhoto()
35+
}
36+
}
37+
38+
if v, err := getFirstValue(meta, "Camera:MotionPhotoVersion", "Camera:MicroVideoVersion"); err == nil {
39+
if i, err := strconv.ParseInt(v, 10, 32); err == nil {
40+
initMotionPhoto()
41+
motionPhoto.SetVersion(int32(i))
42+
}
43+
}
44+
45+
if v, err := getFirstValue(meta, "Camera:MotionPhotoPresentationTimestampUs", "Camera:MicroVideoPresentationTimestampUs"); err == nil {
46+
if i, err := strconv.ParseInt(v, 10, 64); err == nil {
47+
initMotionPhoto()
48+
motionPhoto.SetPresentationTimestampUs(i)
49+
}
50+
}
51+
52+
if size, ok := motionPhotoVideoSize(meta); ok {
53+
initMotionPhoto()
54+
motionPhoto.SetVideoSize(size)
55+
}
56+
57+
if motionPhoto == nil || !motionPhoto.HasVideoSize() {
58+
return nil
59+
}
60+
return motionPhoto
61+
}
62+
63+
// motionPhotoVideoSize returns the embedded video's byte length: the length of
64+
// the Container item whose semantic is "MotionPhoto", or for legacy files the
65+
// MicroVideo offset (bytes from EOF to the video start, which equals its
66+
// length). The current scheme wins when both are present, like everywhere else.
67+
func motionPhotoVideoSize(meta map[string][]string) (int64, bool) {
68+
keys := make([]string, 0, len(meta))
69+
for k := range meta {
70+
keys = append(keys, k)
71+
}
72+
// map order is random, the first matching container item must be stable
73+
sort.Strings(keys)
74+
for _, k := range keys {
75+
if vals := meta[k]; !strings.HasSuffix(k, "/Item:Semantic") || len(vals) == 0 || vals[0] != "MotionPhoto" {
76+
continue
77+
}
78+
if v, err := getFirstValue(meta, strings.TrimSuffix(k, "/Item:Semantic")+"/Item:Length"); err == nil {
79+
if i, err := strconv.ParseInt(v, 10, 64); err == nil {
80+
return i, true
81+
}
82+
}
83+
}
84+
if v, err := getFirstValue(meta, "Camera:MicroVideoOffset"); err == nil {
85+
if i, err := strconv.ParseInt(v, 10, 64); err == nil {
86+
return i, true
87+
}
88+
}
89+
return 0, false
90+
}
91+
92+
// looksLikeMP4 reports whether buf begins with an ISO base media (MP4/QuickTime)
93+
// "ftyp" box, which Google Motion Photo and legacy MicroVideo clips start with.
94+
func looksLikeMP4(buf []byte) bool {
95+
return len(buf) >= 8 && string(buf[4:8]) == "ftyp"
96+
}
97+
98+
// motionPhotoHasVideo confirms that the file actually contains the embedded video
99+
// the XMP advertises. A photos.google.com share strips the appended video but
100+
// keeps the XMP, which would otherwise make us expose an unplayable facet. The
101+
// video is appended at the end, so it starts at fileSize-videoSize; we read a few
102+
// bytes there and require an MP4 signature.
103+
func (t Tika) motionPhotoHasVideo(ctx context.Context, ri *provider.ResourceInfo, videoSize int64) bool {
104+
size := int64(ri.GetSize())
105+
if videoSize <= 0 || videoSize >= size {
106+
return false
107+
}
108+
109+
rc, err := t.RetrieveRange(ctx, ri.GetId(), size-videoSize, motionPhotoVideoSignatureLen)
110+
if err != nil {
111+
t.logger.Debug().Err(err).Interface("ResourceID", ri.GetId()).Msg("could not read motion photo video header, dropping facet")
112+
return false
113+
}
114+
defer rc.Close()
115+
116+
buf := make([]byte, motionPhotoVideoSignatureLen)
117+
n, _ := io.ReadFull(rc, buf)
118+
return looksLikeMP4(buf[:n])
119+
}

0 commit comments

Comments
 (0)