Skip to content

Commit c243d5e

Browse files
author
Tarik Eshaq
committed
[DO NOT MERGE] prototype to use a rust experiments api and expose it through glean
1 parent 200f9d3 commit c243d5e

File tree

12 files changed

+661
-12
lines changed

12 files changed

+661
-12
lines changed

.buildconfig.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
libraryVersion: 31.4.0
1+
libraryVersion: 31.4.0-TESTING28
22
groupId: org.mozilla.telemetry
33
projects:
44
glean:

Cargo.lock

+195-5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ buildscript {
6666
// Docs generation
6767
classpath "org.jetbrains.dokka:dokka-android-gradle-plugin:$versions.dokka"
6868

69+
classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.12'
6970
// NOTE: Do not place your application dependencies here; they belong
7071
// in the individual module build.gradle files
7172
}

glean-core/android/build.gradle

+32
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ apply plugin: 'kotlin-android'
1717
apply plugin: 'kotlin-android-extensions'
1818
apply plugin: 'org.jetbrains.dokka-android'
1919
apply plugin: 'jacoco'
20+
apply plugin: 'com.google.protobuf'
2021

2122
/*
2223
* This defines the location of the JSON schema used to validate the pings
@@ -65,6 +66,13 @@ android {
6566
}
6667

6768
sourceSets {
69+
// We define where the gradle tasks can find
70+
// the .proto file used in the generation
71+
main {
72+
proto {
73+
srcDir '../src'
74+
}
75+
}
6876
main.jniLibs.srcDirs += "$buildDir/nativeLibs/android"
6977
test.resources.srcDirs += "$buildDir/rustJniLibs/desktop"
7078
test.resources.srcDirs += "$buildDir/nativeLibs/desktop"
@@ -172,13 +180,37 @@ configurations {
172180
jnaForTest
173181
}
174182

183+
// We generate a bunch of gradle tasks that will help us
184+
// generate the kotlin files associated with the .proto file
185+
protobuf {
186+
protoc {
187+
artifact = 'com.google.protobuf:protoc:3.11.4'
188+
}
189+
plugins {
190+
javalite {
191+
artifact = 'com.google.protobuf:protoc-gen-javalite:3.0.0'
192+
}
193+
}
194+
generateProtoTasks {
195+
all().each { task ->
196+
task.builtins {
197+
java {
198+
option "lite"
199+
}
200+
}
201+
}
202+
}
203+
}
204+
175205
dependencies {
206+
api "org.mozilla.components:concept-fetch:$versions.android_components"
176207
jnaForTest "net.java.dev.jna:jna:$versions.jna@jar"
177208
implementation "net.java.dev.jna:jna:$versions.jna@aar"
178209

179210
// Note: the following version must be kept in sync
180211
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$versions.kotlin"
181212
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$versions.coroutines"
213+
implementation 'com.google.protobuf:protobuf-javalite:3.11.4'
182214

183215
implementation "androidx.annotation:annotation:$versions.androidx_annotation"
184216
implementation "androidx.lifecycle:lifecycle-extensions:$versions.androidx_lifecycle_extensions"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package mozilla.telemetry.glean
2+
3+
import android.content.Context
4+
import android.os.Build
5+
import com.sun.jna.Pointer
6+
import mozilla.telemetry.glean.rust.LibGleanFFI
7+
import mozilla.telemetry.glean.rust.RustError
8+
import java.util.*
9+
import java.util.concurrent.atomic.AtomicLong
10+
import com.google.protobuf.CodedOutputStream
11+
import com.google.protobuf.MessageLite
12+
import com.sun.jna.Native
13+
import java.nio.ByteBuffer
14+
import java.nio.ByteOrder
15+
16+
// A LOT OF THIS IS COPIED FOR THE SAKE OF THE PROTOTYPE, NOT COMPLETE
17+
fun <T : MessageLite> T.toNioDirectBuffer(): Pair<ByteBuffer, Int> {
18+
val len = this.serializedSize
19+
val nioBuf = ByteBuffer.allocateDirect(len)
20+
nioBuf.order(ByteOrder.nativeOrder())
21+
val output = CodedOutputStream.newInstance(nioBuf)
22+
this.writeTo(output)
23+
output.checkNoSpaceLeft()
24+
return Pair(first = nioBuf, second = len)
25+
}
26+
27+
open class ExperimentsInternalAPI internal constructor () {
28+
private var raw: AtomicLong = AtomicLong(0)
29+
30+
fun initialize(
31+
applicationContext: Context,
32+
dbPath: String
33+
) {
34+
val appCtx = MsgTypes.AppContext.newBuilder()
35+
.setAppId(applicationContext.packageName)
36+
.setAppVersion(applicationContext.packageManager.getPackageInfo(applicationContext.packageName, 0).versionName)
37+
.setDeviceManufacturer(Build.MANUFACTURER)
38+
.setLocaleCountry(
39+
try {
40+
Locale.getDefault().isO3Country
41+
} catch (e: MissingResourceException) {
42+
Locale.getDefault().country
43+
}
44+
)
45+
.setLocaleLanguage(
46+
try {
47+
Locale.getDefault().isO3Language
48+
} catch (e: MissingResourceException) {
49+
Locale.getDefault().language
50+
}
51+
)
52+
.setDeviceModel(Build.DEVICE)
53+
.build()
54+
val (nioBuf, len) = appCtx.toNioDirectBuffer()
55+
raw.set( rustCall { error ->
56+
val ptr = Native.getDirectBufferPointer(nioBuf)
57+
LibGleanFFI.INSTANCE.experiments_new(ptr, len, dbPath, error)
58+
})
59+
}
60+
61+
fun getBranch(experimentName: String): String {
62+
var ptr = rustCall { error ->
63+
LibGleanFFI.INSTANCE.experiments_get_branch(raw.get(), experimentName, error)
64+
}
65+
return ptr.getAndConsumeRustString()
66+
}
67+
68+
/**
69+
* Helper to read a null terminated String out of the Pointer and free it.
70+
*
71+
* Important: Do not use this pointer after this! For anything!
72+
*/
73+
internal fun Pointer.getAndConsumeRustString(): String {
74+
return this.getRustString()
75+
// PLEASE INSERT A FREE HERE!!!!!!!
76+
}
77+
78+
/**
79+
* Helper to read a null terminated string out of the pointer.
80+
*
81+
* Important: doesn't free the pointer, use [getAndConsumeRustString] for that!
82+
*/
83+
internal fun Pointer.getRustString(): String {
84+
return this.getString(0, "utf8")
85+
}
86+
87+
// In practice we usually need to be synchronized to call this safely, so it doesn't
88+
// synchronize itself
89+
private inline fun <U> nullableRustCall(callback: (RustError.ByReference) -> U?): U? {
90+
val e = RustError.ByReference()
91+
try {
92+
val ret = callback(e)
93+
if (e.isFailure()) {
94+
// We ignore it for now, although we shouldn't just cuz protoype
95+
//throw e.intoException()
96+
}
97+
return ret
98+
} finally {
99+
// This only matters if `callback` throws (or does a non-local return, which
100+
// we currently don't do)
101+
e.ensureConsumed()
102+
}
103+
}
104+
105+
private inline fun <U> rustCall(callback: (RustError.ByReference) -> U?): U {
106+
return nullableRustCall(callback)!!
107+
}
108+
}
109+
110+
/**
111+
* The main experiments object
112+
* ```
113+
*/
114+
object Experiments : ExperimentsInternalAPI()
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
package mozilla.telemetry.glean
2+
3+
/* This Source Code Form is subject to the terms of the Mozilla Public
4+
* License, v. 2.0. If a copy of the MPL was not distributed with this
5+
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6+
7+
// THIS IS A CLONE OF THE OTHER HTTPCONFIG USED FOR APPLICATION SERVICES
8+
// SEEMED CONVIENIENT FOR THE SAKE OF THE PROTOTYPE, PREFERRABLY
9+
// WE CAN USE THE SAME HTTPCONFIG, BUT I COULDN'T WRAP MY HEAD ON A QUICK
10+
// WAY TO DO THAT FOR A PROTOTYPE
11+
12+
import com.google.protobuf.ByteString
13+
import mozilla.components.concept.fetch.Client
14+
import mozilla.components.concept.fetch.MutableHeaders
15+
import mozilla.components.concept.fetch.Request
16+
import mozilla.telemetry.glean.rust.LibGleanFFI
17+
import mozilla.telemetry.glean.rust.RawFetchCallback
18+
import mozilla.telemetry.glean.rust.RustBuffer
19+
import java.util.concurrent.TimeUnit
20+
import java.util.concurrent.locks.ReentrantReadWriteLock
21+
import kotlin.concurrent.read
22+
import kotlin.concurrent.write
23+
24+
/**
25+
* Singleton allowing management of the HTTP backend
26+
* used by Rust components.
27+
*/
28+
object RustHttpConfig {
29+
// Protects imp/client
30+
private var lock = ReentrantReadWriteLock()
31+
@Volatile
32+
private var client: Lazy<Client>? = null
33+
// Important note to future maintainers: if you mess around with
34+
// this code, you have to make sure `imp` can't get GCed. Extremely
35+
// bad things will happen if it does!
36+
@Volatile
37+
private var imp: CallbackImpl? = null
38+
39+
/**
40+
* Set the HTTP client to be used by all Rust code.
41+
* the `Lazy`'s value is not read until the first request is made.
42+
*/
43+
@Synchronized
44+
fun setClient(c: Lazy<Client>) {
45+
lock.write {
46+
client = c
47+
if (imp == null) {
48+
imp = CallbackImpl()
49+
LibGleanFFI.INSTANCE.viaduct_initialize(imp!!)
50+
}
51+
}
52+
}
53+
54+
internal fun convertRequest(request: MsgTypes.Request): Request {
55+
val headers = MutableHeaders()
56+
for (h in request.headersMap) {
57+
headers.append(h.key, h.value)
58+
}
59+
return Request(
60+
url = request.url,
61+
method = convertMethod(request.method),
62+
headers = headers,
63+
connectTimeout = Pair(request.connectTimeoutSecs.toLong(), TimeUnit.SECONDS),
64+
readTimeout = Pair(request.readTimeoutSecs.toLong(), TimeUnit.SECONDS),
65+
body = if (request.hasBody()) {
66+
Request.Body(request.body.newInput())
67+
} else {
68+
null
69+
},
70+
redirect = if (request.followRedirects) {
71+
Request.Redirect.FOLLOW
72+
} else {
73+
Request.Redirect.MANUAL
74+
},
75+
cookiePolicy = Request.CookiePolicy.OMIT,
76+
useCaches = request.useCaches
77+
)
78+
}
79+
80+
@Suppress("TooGenericExceptionCaught", "ReturnCount")
81+
internal fun doFetch(b: RustBuffer.ByValue): RustBuffer.ByValue {
82+
lock.read {
83+
try {
84+
val request = MsgTypes.Request.parseFrom(b.asCodedInputStream())
85+
val rb = try {
86+
// Note: `client!!` is fine here, since if client is null,
87+
// we wouldn't have yet initialized
88+
val resp = client!!.value.fetch(convertRequest(request))
89+
val rb = MsgTypes.Response.newBuilder()
90+
.setUrl(resp.url)
91+
.setStatus(resp.status)
92+
.setBody(resp.body.useStream {
93+
ByteString.readFrom(it)
94+
})
95+
96+
for (h in resp.headers) {
97+
rb.putHeaders(h.name, h.value)
98+
}
99+
rb
100+
} catch (e: Throwable) {
101+
MsgTypes.Response.newBuilder().setExceptionMessage("fetch error: ${e.message ?: e.javaClass.canonicalName}")
102+
}
103+
val built = rb.build()
104+
val needed = built.serializedSize
105+
val outputBuf = LibGleanFFI.INSTANCE.viaduct_alloc_bytebuffer(needed)
106+
try {
107+
// This is only null if we passed a negative number or something to
108+
// viaduct_alloc_bytebuffer.
109+
val stream = outputBuf.asCodedOutputStream()!!
110+
built.writeTo(stream)
111+
return outputBuf
112+
} catch (e: Throwable) {
113+
// Note: we want to clean this up only if we are not returning it to rust.
114+
LibGleanFFI.INSTANCE.viaduct_destroy_bytebuffer(outputBuf)
115+
LibGleanFFI.INSTANCE.viaduct_log_error("Failed to write buffer: ${e.message}")
116+
throw e
117+
}
118+
} finally {
119+
LibGleanFFI.INSTANCE.viaduct_destroy_bytebuffer(b)
120+
}
121+
}
122+
}
123+
}
124+
125+
internal fun convertMethod(m: MsgTypes.Request.Method): Request.Method {
126+
return when (m) {
127+
MsgTypes.Request.Method.GET -> Request.Method.GET
128+
MsgTypes.Request.Method.POST -> Request.Method.POST
129+
MsgTypes.Request.Method.HEAD -> Request.Method.HEAD
130+
MsgTypes.Request.Method.OPTIONS -> Request.Method.OPTIONS
131+
MsgTypes.Request.Method.DELETE -> Request.Method.DELETE
132+
MsgTypes.Request.Method.PUT -> Request.Method.PUT
133+
MsgTypes.Request.Method.TRACE -> Request.Method.TRACE
134+
MsgTypes.Request.Method.CONNECT -> Request.Method.CONNECT
135+
}
136+
}
137+
138+
internal class CallbackImpl : RawFetchCallback {
139+
@Suppress("TooGenericExceptionCaught")
140+
override fun invoke(b: RustBuffer.ByValue): RustBuffer.ByValue {
141+
try {
142+
return RustHttpConfig.doFetch(b)
143+
} catch (e: Throwable) {
144+
LibGleanFFI.INSTANCE.viaduct_log_error("doFetch failed: ${e.message}")
145+
// This is our last resort. It's bad news should we fail to
146+
// return something from this function.
147+
return RustBuffer.ByValue()
148+
}
149+
}
150+
}

glean-core/android/src/main/java/mozilla/telemetry/glean/rust/LibGleanFFI.kt

+18-4
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,7 @@
66

77
package mozilla.telemetry.glean.rust
88

9-
import com.sun.jna.Library
10-
import com.sun.jna.Native
11-
import com.sun.jna.Pointer
12-
import com.sun.jna.StringArray
9+
import com.sun.jna.*
1310
import java.lang.reflect.Proxy
1411
import mozilla.telemetry.glean.config.FfiConfiguration
1512
import mozilla.telemetry.glean.net.FfiPingUploadTask
@@ -567,4 +564,21 @@ internal interface LibGleanFFI : Library {
567564
// Misc
568565

569566
fun glean_str_free(ptr: Pointer)
567+
568+
// DO NOT MERGE ME PLEASE. THIS WAS ONLY ADDED FOR PROTOTYPING PURPOSES!!!
569+
fun experiments_new(appContext: Pointer, appContextLen: Int, dbPath: String, error: RustError.ByReference): Long
570+
571+
fun experiments_get_branch(handle: Long, branchName: String, error: RustError.ByReference): Pointer?
572+
573+
fun viaduct_destroy_bytebuffer(b: RustBuffer.ByValue)
574+
// Returns null buffer to indicate failure
575+
fun viaduct_alloc_bytebuffer(sz: Int): RustBuffer.ByValue
576+
// Returns 0 to indicate redundant init.
577+
fun viaduct_initialize(cb: RawFetchCallback): Byte
578+
579+
fun viaduct_log_error(s: String)
580+
}
581+
582+
internal interface RawFetchCallback: Callback {
583+
fun invoke(b: RustBuffer.ByValue): RustBuffer.ByValue
570584
}

0 commit comments

Comments
 (0)