FLUME-467: Merge cloudera/hbase with trunk cloudera/master

build.xml

@@ -104,7 +104,8 @@ to call at top-level: ant deploy-contrib compile-core-test
   <!-- for now ivy.lib and ivy.test.lib stay at legacy location -->
   <property name="ivy.lib" value="${lib.dir}"/>
   <property name="ivy.test.lib" value="${test.lib.dir}"/>
-  <property name="ivy.tools.lib" value="${build.dir}/libtools"/>
+  <property name="ivysettings.xml" value="${basedir}/ivysettings.xml"/>
+
 
   <!-- eclipse property set -->
   <property name="build.dir.eclipse" value=".eclipse"/>
@@ -131,6 +132,8 @@ to call at top-level: ant deploy-contrib compile-core-test
   <path id="libs.classpath">
     <pathelement location="${conf.dir}"/>
     <fileset dir="${lib.dir}">
+      <!-- hadoop webapp has to be first on the classpath -->
+      <include name="**/*hadoop-core*.jar" />
       <include name="**/*.jar" />
       <exclude name="**/excluded/" />
     </fileset>
@@ -218,21 +221,23 @@ to call at top-level: ant deploy-contrib compile-core-test
     <property name="ivy.initialized" value="true"/>
   </target>
 
-  <target name="ivy-init" depends="ivy-download,ivy-taskdef"/>
+  <target name="ivy-init" depends="ivy-download,ivy-taskdef">
+    <ivy:settings id="${ant.project.name}" file="${ivysettings.xml}"/>
+  </target>
 
   <target name="ivy-retrieve" depends="init,ivy-init">
-    <ivy:retrieve type="jar" conf="default"
+    <ivy:retrieve settingsRef="${ant.project.name}" type="jar" conf="default"
                   pattern="${ivy.lib}/[artifact]-[revision].[ext]"/>
   </target>
 
   <target name="ivy-retrieve-test" depends="init,ivy-init">
-    <ivy:retrieve type="jar" conf="test"
+    <ivy:retrieve settingsRef="${ant.project.name}" type="jar" conf="test"
                   pattern="${ivy.test.lib}/[artifact]-[revision].[ext]"/>
   </target>
 
   <target name="ivy-retrieve-findbugs" depends="init,ivy-init">
-    <ivy:retrieve type="jar" conf="findbugs"
-                  pattern="${ivy.tools.lib}/[artifact]-[revision].[ext]"/>
+    <ivy:retrieve settingsRef="${ant.project.name}" type="jar" conf="findbugs"
+                  pattern="${ivy.test.lib}/[artifact]-[revision].[ext]"/>
   </target>
 
   <target name="ivy-retrieve-pmd" depends="init,ivy-init">
@@ -241,13 +246,12 @@ to call at top-level: ant deploy-contrib compile-core-test
   </target>
 
   <target name="ivy-retrieve-coverage" depends="init,ivy-init">
-    <ivy:retrieve type="jar" conf="coverage"
-                  pattern="${ivy.tools.lib}/[artifact]-[revision].[ext]"/>
+    <ivy:retrieve settingsRef="${ant.project.name}" type="jar" conf="coverage"
+                  pattern="${ivy.test.lib}/[artifact]-[revision].[ext]"/>
   </target>
-
   <target name="ivy-retrieve-releaseaudit" depends="init,ivy-init">
-    <ivy:retrieve type="jar" conf="releaseaudit"
-                  pattern="${ivy.tools.lib}/[artifact]-[revision].[ext]"/>
+    <ivy:retrieve settingsRef="${ant.project.name}" type="jar" conf="releaseaudit"
+                  pattern="${ivy.test.lib}/[artifact]-[revision].[ext]"/>
     <ivy:cachepath pathid="releaseaudit-classpath" conf="releaseaudit"/>
   </target>
 

ivy.xml

@@ -41,8 +41,16 @@
     <dependency org="jline" name="jline" rev="0.9.94" transitive="false"/>
     -->
 
-    <dependency org="junit" name="junit" rev="4.8.1" conf="test->default"/> 
+    <dependency org="com.cloudera.hadoop" name="hadoop-core" rev="0.20.2-320" conf="default"/>
+
+    <dependency org="org.slf4j" name="slf4j-api" rev="1.5.11"/>
+    <dependency org="org.slf4j" name="slf4j-simple" rev="1.5.11"
+        conf="test->default"/>
+
+    <dependency org="junit" name="junit" rev="4.8.1" conf="test->default"/>
     <dependency org="org.mockito" name="mockito-all" rev="1.8.1" conf="test->default"/>
+    <dependency org="com.cloudera.hadoop" name="hadoop-test" rev="0.20.2-320" conf="test->default"/>
+
     <dependency org="com.google.code.findbugs" name="findbugs"
                 rev="1.3.9" conf="findbugs->default" />
     <dependency org="com.google.code.findbugs" name="findbugs-ant"

ivysettings.xml

@@ -0,0 +1,41 @@
+<ivysettings>
+
+ <!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+  <property name="repo.maven.org"
+    value="http://repo1.maven.org/maven2/" override="false"/>
+  <property name="repo.cloudera.com"
+    value="https://repository.cloudera.com/content/repositories/releases/" override="false"/>
+  <property name="maven2.pattern"
+    value="[organisation]/[module]/[revision]/[module]-[revision]"/>
+  <property name="maven2.pattern.ext" value="${maven2.pattern}.[ext]"/>
+  <include url="${ivy.default.conf.dir}/ivyconf-local.xml"/>
+  <settings defaultResolver="default"/>
+  <resolvers>
+    <ibiblio name="maven2" root="${repo.maven.org}"
+      pattern="${maven2.pattern.ext}" m2compatible="true"/>
+    <ibiblio name="cloudera-maven2" root="${repo.cloudera.com}"
+      pattern="${maven2.pattern.ext}" m2compatible="true"/>
+
+    <chain name="default" dual="true">
+      <resolver ref="maven2"/>
+      <resolver ref="cloudera-maven2"/>
+    </chain>
+
+  </resolvers>
+</ivysettings>

plugins/hbasesink/build.xml

@@ -0,0 +1,173 @@
+<?xml version="1.0"?>
+<!--
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<!--
+   Before you can run these subtargets directly, you need
+   to call at top-level: ant
+-->
+
+<project name="flume-hbase" default="jar">
+  <property name="javac.debug" value="on"/>
+  <property name="javac.optimize" value="on"/>
+  <property name="javac.deprecation" value="off"/>
+  <property name="javac.version" value="1.5"/>
+  <property name="javac.args" value=""/>
+  <property name="javac.args.warnings" value="-Xlint:unchecked"/>
+
+  <property name="flume.base" value="../.."/>
+  <property name="build.dir" value="build"/>
+  <property name="build.test" value="${build.dir}/test"/>
+  <property name="build.encoding" value="ISO-8859-1"/>
+
+  <property name="test.src.dir" value="src/javatest"/>
+  <property name="test.lib.dir" value="${flume.base}/libtest"/>
+  <property name="test.build.dir" value="${build.dir}/test"/>
+  <property name="test.generated.dir" value="${test.build.dir}/src"/>
+  <property name="test.build.data" value="${test.build.dir}/data"/>
+  <property name="test.log.dir" value="${test.build.dir}/logs"/>
+  <property name="test.build.classes" value="${test.build.dir}/classes"/>
+  <property name="test.include" value="Test*"/>
+  <property name="test.classpath.id" value="test.classpath"/>
+  <property name="test.output" value="yes"/>
+  <property name="test.timeout" value="900000"/>
+  <property name="test.junit.output.format" value="plain"/>
+
+  <path id="classpath">
+    <!-- in case we are running in dev env -->
+    <pathelement location="${flume.base}/build/classes"/>
+    <fileset dir="${flume.base}">
+      <include name="flume-*.jar" />
+    </fileset>
+    <fileset dir="${flume.base}/lib">
+      <include name="**/slf4j-*.jar" />
+      <include name="**/hadoop-*.jar" />
+      <include name="**/guava-*.jar" />
+    </fileset>
+    <fileset dir="${flume.base}/plugins/hbasesink/lib">
+      <include name="**/*.jar" />
+    </fileset>
+  </path>
+
+  <!-- the unit test classpath: uses test.src.dir for configuration -->
+  <path id="test.classpath">
+    <pathelement location="${test.build.classes}" />
+    <pathelement location="${test.src.dir}"/>
+    <pathelement location="${build.dir}"/>
+    <fileset dir="${test.lib.dir}">
+      <include name="**/*.jar" />
+      <exclude name="**/excluded/" />
+    </fileset>
+    <fileset dir="${flume.base}/lib">
+      <include name="**/*.jar" />
+      <exclude name="**/excluded/" />
+    </fileset>
+    <fileset dir="${flume.base}/plugins/hbasesink">
+      <include name="**/*.jar" />
+      <exclude name="**/excluded/" />
+    </fileset>
+    <fileset dir="${flume.base}/plugins/hbasesink/lib">
+      <include name="**/*.jar" />
+    </fileset>
+    <path refid="classpath"/>
+  </path>
+
+  <target name="jar">
+    <mkdir dir="${build.dir}"/>
+    <mkdir dir="${build.dir}/classes"/>
+
+    <javac srcdir="./src/java" destdir="${build.dir}/classes" debug="${javac.debug}">
+      <classpath refid="classpath"/>
+    </javac>
+
+    <jar jarfile="hbase_sink.jar" basedir="${build.dir}/classes"/>
+  </target>
+
+  <target name="clean">
+    <echo message="Cleaning generated files and stuff"/>
+    <delete dir="build" />
+    <delete file="hbase_sink.jar" />
+  </target>
+
+  <!-- ================================================================== -->
+  <!-- Compile test code                                                  -->
+  <!-- ================================================================== -->
+  <target name="compile-plugin-test" depends="jar">
+    <mkdir dir="${build.dir}/test"/>
+    <mkdir dir="${build.dir}/test/classes"/>
+
+    <javac
+     encoding="${build.encoding}"
+     srcdir="${test.src.dir}"
+     includes="**/*.java"
+     destdir="${test.build.classes}"
+     debug="${javac.debug}"
+     optimize="${javac.optimize}"
+     target="${javac.version}"
+     source="${javac.version}"
+     deprecation="${javac.deprecation}">
+      <compilerarg line="${javac.args}" />
+      <classpath refid="test.classpath"/>
+    </javac>
+ </target>
+
+
+  <!-- ================================================================== -->
+  <!-- Run unit tests                                                     -->
+  <!-- ================================================================== -->
+  <target name="test" depends="jar,compile-plugin-test" >
+    <echo message="Unit Testing of HBase Sink"/>
+    <junit
+      printsummary="yes" showoutput="${test.output}"
+      haltonfailure="no" fork="yes" maxmemory="1024m"
+      errorProperty="tests.failed" failureProperty="tests.failed"
+      timeout="${test.timeout}"
+      dir="${test.build.dir}">
+
+      <!-- uncomment this if you want to attach a debugger -->
+      <!--
+      <jvmarg line="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=2601" />
+      -->
+      <env key="FLUME_HOME" value="${basedir}" />
+      <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl" />
+      <sysproperty key="java.library.path" value="${lib.dir}" />
+      <sysproperty key="build.test" value="${build.test}"/>
+
+      <!-- we want more log4j output when running unit tests -->
+      <sysproperty key="hadoop.root.logger"
+        value="INFO,console" />
+
+      <!-- tools.jar from Sun JDK also required to invoke javac. -->
+      <classpath>
+        <path refid="test.classpath"/>
+        <pathelement path="${env.JAVA_HOME}/lib/tools.jar" />
+      </classpath>
+      <formatter type="${test.junit.output.format}" />
+      <batchtest todir="${build.test}" unless="testcase">
+        <fileset dir="${test.src.dir}"
+                 includes="**/Test*.java" excludes="**/${test.exclude}.java" />
+      </batchtest>
+      <batchtest todir="${build.test}" if="testcase">
+        <fileset dir="${test.src.dir}" includes="**/${testcase}.java"/>
+      </batchtest>
+    </junit>
+    <fail if="tests.failed">Tests failed!</fail>
+  </target>
+
+</project>

plugins/hbasesink/docs/README

@@ -0,0 +1,56 @@
+Running Flume on HBase
+----------------------
+
+1) Compile top-level flume
+   $ ant
+
+2) Copy your version of hbase-*.jar into the plugins/hbasesink/lib
+   I tested against hbase-0.89.20100621+17.jar
+
+3) Compile hbase-sink from this directory
+   ../plugins/hbasesink$ ant
+
+4) Successful compilation will produce "hbase_sink.jar" in ../plugins/hbasesink/
+
+5) Add the plugin classes in flume/conf/flume-site.xml
+<configuration>
+  <property>
+    <name>flume.plugin.classes</name>
+    <value>com.cloudera.flume.hbase.Attr2HBaseEventSink</value>
+    <description>Comma separated list of plugin classes</description>
+  </property>
+</configuration>
+
+
+6) Include the jars in the "FLUME_CLASSPATH"
+From the terminal where you would start the Flume nodes:
+  $ export FLUME_CLASSPATH=/your_path/flume/plugins/hbasesink/hbase_sink.jar:/your_path/hbase-*/conf/
+  Include all the jars that your plugin refers to
+
+Here's how I've run flume against hbase in dev mode:
+
+7) From your hbase-*, "bin/start-hbase.sh"
+
+Note that starting hbase will also start ZooKeeper
+
+8) using the hbase shell create a table for the flume sink to write events
+
+$ bin/hbase shell
+> create 't1', 'f1', 'f2'
+
+9) Undertand the attr2hbase sink usage and configure the source:
+   "usage: attr2hbase(\"table\" [,\"sysFamily\"[, \"writeBody\"[,\"attrPrefix\"[,\"writeBufferSize\"[,\"writeToWal\"]]]]])
+   Refer parameter_mapping.html in /docs for more details.
+
+10) start flume, I started a node with a rssatomSource and attr2hbase("t1","f1","f2:event","attr-prefix","10","false")'
+
+> scan 't1'
+(This is output of my rowkey and data)
+
+hbase(main):002:0> scan 't1'
+ROW                          COLUMN+CELL
+ \x00\x00\xF6_\x0Fk\xF4\x80  column=f2:event, timestamp=1274227444388, value=hello
+ \x00\x00\xF6_\x0Fk\xF4\x80  column=f1:host, timestamp=1274227444388, value=valhalla
+ \x00\x00\xF6_\x0Fk\xF4\x80  column=f1:timestamp, timestamp=1274227444388, value=\x00\x00\x01\x28\xAD\xDF\xCA\
+                             x7C
+1 row(s) in 0.0550 seconds