Skip to content

Commit 2d092ae

Browse files
Kanamarlapudipraveen-kanamarlapudi
authored andcommitted
Updated code as per the comments
1 parent e94f507 commit 2d092ae

File tree

6 files changed

+15
-10
lines changed

6 files changed

+15
-10
lines changed

conf/livy.conf.template

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
# livy.impersonation.enabled = true
3333

3434
# Logs size livy can cache for each session/batch. 0 means don't cache the logs.
35-
# livy.spark.logs.size = 200
35+
# livy.cache-log.size = 200
3636

3737
# Comma-separated list of Livy RSC jars. By default Livy will upload jars from its installation
3838
# directory every time a session is started. By caching these files in HDFS, for example, startup

server/src/main/scala/com/cloudera/livy/LivyConf.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ object LivyConf {
116116
val RECOVERY_STATE_STORE_URL = Entry("livy.server.recovery.state-store.url", "")
117117

118118
// Livy will cache the max no of logs specified. 0 means don't cache the logs.
119-
val SPARK_LOGS_SIZE = Entry("livy.spark.logs.size", 200)
119+
val SPARK_LOGS_SIZE = Entry("livy.cache-log.size", 200)
120120

121121
// If Livy can't find the yarn app within this time, consider it lost.
122122
val YARN_APP_LOOKUP_TIMEOUT = Entry("livy.server.yarn.app-lookup-timeout", "60s")

server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,9 +380,9 @@ class InteractiveSession(
380380
heartbeat()
381381

382382
private val app = mockApp.orElse {
383-
val driverProcess = client.flatMap { c => Option(c.getDriverProcess) }
383+
val driverProcess = client.flatMap { c => Option(c.getDriverProcess) }
384384
.map(new LineBufferedProcess(_, livyConf.getInt(LivyConf.SPARK_LOGS_SIZE)))
385-
Option(SparkApp.create(appTag, appId, driverProcess, livyConf, Some(this)))
385+
driverProcess.map { _ => SparkApp.create(appTag, appId, driverProcess, livyConf, Some(this)) }
386386
}
387387

388388
if (client.isEmpty) {

server/src/main/scala/com/cloudera/livy/utils/LineBufferedStream.scala

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,11 @@ class LineBufferedStream(inputStream: InputStream, logSize: Int) extends Logging
3838
private val thread = new Thread {
3939
override def run() = {
4040
val lines = Source.fromInputStream(inputStream).getLines()
41-
for (line <- lines) {
41+
for (line <- lines) {
42+
info(s"stdout: $line")
4243
_lock.lock()
4344
try {
44-
info(s"stdout: $line")
45-
if(logSize > 0) _lines add line
45+
_lines.add(line)
4646
_condition.signalAll()
4747
} finally {
4848
_lock.unlock()
@@ -61,7 +61,12 @@ class LineBufferedStream(inputStream: InputStream, logSize: Int) extends Logging
6161
thread.setDaemon(true)
6262
thread.start()
6363

64-
def lines: IndexedSeq[String] = IndexedSeq.empty[String] ++ _lines.toArray(Array.empty[String])
64+
def lines: IndexedSeq[String] = {
65+
_lock.lock()
66+
val lines = IndexedSeq.empty[String] ++ _lines.toArray(Array.empty[String])
67+
_lock.unlock()
68+
lines
69+
}
6570

6671
def iterator: Iterator[String] = {
6772
new LinesIterator

server/src/main/scala/com/cloudera/livy/utils/SparkApp.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,7 @@ object SparkApp {
9292
if (livyConf.isRunningOnYarn()) {
9393
new SparkYarnApp(uniqueAppTag, appId, process, listener, livyConf)
9494
} else {
95-
// process is None in recovery mode
96-
// require(process.isDefined, "process must not be None when Livy master is not YARN.")
95+
require(process.isDefined, "process must not be None when Livy master is not YARN.")
9796
new SparkProcApp(process.get, listener)
9897
}
9998
}

server/src/main/scala/com/cloudera/livy/utils/SparkProcApp.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ class SparkProcApp (
4242

4343
override def log(): IndexedSeq[String] =
4444
("stdout: " +: process.inputLines) ++ ("\nstderr: " +: process.errorLines)
45+
4546
private def changeState(newState: SparkApp.State.Value) = {
4647
if (state != newState) {
4748
listener.foreach(_.stateChanged(state, newState))

0 commit comments

Comments
 (0)