Merge develop into release

Conflicts:
	hoot-core-test/hoot-core-test.pro
	scripts/jenkins/ConfigureTestUbuntu1404VagrantEnvironment.sh
	scripts/jenkins/ConfigureUbuntu1404VagrantEnvironment.sh

Author: sisskind

.gitignore

@@ -12,9 +12,10 @@ plugins/config.js
 *.kate-swp
 aclocal.m4
 autom4te.cache
+BuildDate.txt
 bin/
 callgrind.*
-conf/DatabaseConfig.sh
+conf/DatabaseConfigLocal.sh
 conf/LocalHoot.json
 conf/*.rf
 conf/words*.sqlite
@@ -144,7 +145,6 @@ customscript/
 */buildInfo.json/*
 **/buildInfo.json/*.*
 scripts/MakeHootRelease.sh
-hoot-services/pom.xml
 hoot-services/src/main/webapp/WEB-INF/workspace/jdbc/WFS_Connection.xml
 BigReport.asciidoc
 BigReport.pdf
@@ -173,3 +173,4 @@ vagrant*-*
 MingsSuperAwesomePrompt.sh
 id_rsa.pub
 coverage/
+hoot-services/db.properties

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "hoot-ui"]
 	path = hoot-ui
 	url = https://github.com/ngageoint/hootenanny-ui.git
-	branch = release
+	branch = develop

Makefile.am

@@ -84,6 +84,9 @@ test-all:
 test-all-no-core:
 	scripts/HootEnv.sh $(MAKE) -f Makefile.old test-all-no-core
 
+test-all-core:
+	scripts/HootEnv.sh $(MAKE) -f Makefile.old test-all-core
+
 test-quick:
 	scripts/HootEnv.sh $(MAKE) -f Makefile.old test-quick
 
@@ -126,9 +129,6 @@ services-clean-coverage:
 services-coverage:
 	$(MAKE) -f Makefile.old services-coverage
 
-conf/DatabaseConfig.sh:
-	$(MAKE) -f Makefile.old conf/DatabaseConfig.sh
-
 # Use some tar fanciness to grab all the files in conf, plugins, translations and scripts
 install-other:
 	mkdir -p tmp

Makefile.old

@@ -12,7 +12,7 @@ SHELL=/bin/bash
 
 # If you're adding another schema file be sure and update the archive target
 # too.
-SCHEMA_FILES=plugins/tds40_schema.js plugins/tds61_schema.js plugins/mgcp_schema.js plugins/etds_rules.js plugins/etds61_rules.js plugins/etds61_osm_rules.js plugins/tds40_full_schema.js plugins/tds61_full_schema.js plugins/emgcp_rules.js plugins/emgcp_osm_rules.js
+SCHEMA_FILES=plugins/tds61_schema.js plugins/tds61_full_schema.js plugins/etds61_rules.js plugins/etds61_osm_rules.js plugins/tds40_schema.js plugins/tds40_full_schema.js plugins/etds40_rules.js plugins/etds40_osm_rules.js plugins/mgcp_schema.js plugins/emgcp_rules.js plugins/emgcp_osm_rules.js
 
 -include Makefile.inc
 
@@ -85,6 +85,11 @@ docs: always HOOT_VERSION_FILE services-docs
 test-quick: build pp-test
 	bin/HootTest $(HOOT_TEST_DIFF) --quick
 
+test-all-core: services-test-all pp-test plugins-test
+	# Don't run services tests at the same time as glacial -- that may stomp
+	# on each other's DB changes.
+	bin/HootTest $(HOOT_TEST_DIFF) --glacial
+
 test-all-no-core: services-test-all pp-test plugins-test
 	$(MAKE) ui-test
 
@@ -154,10 +159,6 @@ tgs/src/main/cpp/tgs/TgsConfig.h: config.h
 	/bin/sed -r -e 's/^(#[ \t]*define) /\1 TGS_/' config.h >> tgs/src/main/cpp/tgs/TgsConfig.h
 	echo "#endif" >> tgs/src/main/cpp/tgs/TgsConfig.h
 
-conf/DatabaseConfig.sh: conf/DatabaseConfig.sh.orig
-	# This will copy conf/DatabaseConfig.sh.orig to conf/DatabaseConfig.sh if necessary
-	if [ ! -f conf/DatabaseConfig.sh ]; then echo "Using default database configuration. You can safely customize: conf/DatabaseConfig.sh"; cp conf/DatabaseConfig.sh.orig conf/DatabaseConfig.sh; else echo; echo; echo "Check your conf/DatabaseConfig.sh to see if it needs updating with conf/DatabaseConfig.sh.orig, if not just 'touch conf/DatabaseConfig.sh'"; echo; echo; false; fi
-
 # We now have different versions of the NFDD Schema that are built from Excel files:
 # LTDSv40.csv.gz - Local TDS v4.0
 # SUTDSv40.csv.gz - Specialized Urban TDS v4.0
@@ -183,21 +184,26 @@ plugins/emgcp_osm_rules.js: scripts/ConvertMGCPSchema_XML.py conf/mgcp/MGCP_Feat
 	mkdir -p $(@D)
 	$< --fromenglish $(word 2,$^) $(word 3,$^) > $@ || (rm -f $@ ; exit -1)
 
+#### TDSv40
 plugins/tds40_schema.js: scripts/ConvertTDSv40Schema.py conf/nfdd/LTDSv40.csv.gz
 	mkdir -p $(@D)
 	$^ > $@ || (rm -f $@ ; exit -1)
 
-#### TDSv40
 plugins/tds40_full_schema.js: scripts/ConvertTDSv40Schema.py conf/nfdd/TDSv40.csv.gz
 	mkdir -p $(@D)
 	$< --fullschema $(word 2,$^) > $@ || (rm -f $@ ; exit -1)
 
 # Build the TDS40 "To English" rules
 # This need to be full TDS so we have access to the full range of attributes
-plugins/etds_rules.js: scripts/ConvertTDSv40Schema.py conf/nfdd/TDSv40.csv.gz
+plugins/etds40_rules.js: scripts/ConvertTDSv40Schema.py conf/nfdd/TDSv40.csv.gz
 	mkdir -p $(@D)
 	$< --toenglish $(word 2,$^) > $@ || (rm -f $@ ; exit -1)
 
+# Build the TDS40 "From English" rules
+plugins/etds40_osm_rules.js: scripts/ConvertTDSv40Schema.py conf/nfdd/TDSv40.csv.gz
+	mkdir -p $(@D)
+	$< --fromenglish $(word 2,$^) $(word 3,$^) > $@ || (rm -f $@ ; exit -1)
+
 #### TDSv61
 # TDSv61 is built from the TDSv60 spec with NGA's extensions
 plugins/tds61_schema.js: scripts/ConvertTDSv61Schema.py conf/nfdd/TDSv60.csv.gz conf/nfdd/TDS_NGAv01.csv.gz
@@ -221,7 +227,7 @@ plugins/etds61_osm_rules.js: scripts/ConvertTDSv61Schema.py conf/nfdd/TDSv60.csv
 	$< --fromenglish $(word 2,$^) $(word 3,$^) > $@ || (rm -f $@ ; exit -1)
 
 # Build osmapi db
-osmapidb: conf/DatabaseConfig.sh
+osmapidb:
 ifeq ($(BUILD_SERVICES),services)
 	scripts/SetupOsmApiDB.sh
 endif
@@ -247,7 +253,7 @@ qt-make: HOOT_VERSION_FILE Makefile.qmake hoot-core/src/main/cpp/hoot/core/HootC
 	  || true \
 	  ; [ "$${PIPESTATUS[0]}" == "0" ] && true || false
 
-services-build:	HOOT_VERSION_FILE conf/DatabaseConfig.sh
+services-build:	HOOT_VERSION_FILE
 ifeq ($(BUILD_SERVICES),services)
 	cd hoot-services && $(MAKE) -f Makefile build
 endif
@@ -352,7 +358,8 @@ archive: js-make services-build docs
 	# TDSv40
 	cp plugins/tds40_schema.js $(HOOT_TARNAME)/plugins/tds40_schema.js
 	cp plugins/tds40_full_schema.js $(HOOT_TARNAME)/plugins/tds61_full_schema.js
-	cp plugins/etds_rules.js $(HOOT_TARNAME)/plugins/etds_rules.js
+	cp plugins/etds40_rules.js $(HOOT_TARNAME)/plugins/etds40_osm_rules.js
+	cp plugins/etds40_osm_rules.js $(HOOT_TARNAME)/plugins/etds40_osm_rules.js
 	# TDSv61
 	cp plugins/tds61_schema.js $(HOOT_TARNAME)/plugins/tds61_schema.js
 	cp plugins/tds61_full_schema.js $(HOOT_TARNAME)/plugins/tds61_full_schema.js

README.md

@@ -15,8 +15,7 @@ and user-friendly conflation experience without the extra overhead of thick desk
 benefits which include its open license allowing users to easily customize and add functionality, a robust editing capability 
 originally targeted for interactive editing of OSM features, and a web service based architecture for integrating into SOA environments.  
 A REST API is in place to connect the web browser based User Interface (UI) with the core conflation algorithms and database. 
-The translation and conflation operations are also exposed through node.js service endpoints, Open Geospatial Consortium (OGC) 
-Web Processing Service (WPS) and the resulting vector data is accessible via a Web Feature Service (WFS) for additional open 
+The translation and conflation operations are also exposed through node.js service endpoints and the resulting vector data is accessible via an Open Geospatial Consortium (OGC) Web Feature Service (WFS) for additional open 
 interoperability. The application is primarily supported in either the Chrome or Chromium browser with limited supported in Firefox.  
 
 If you have any support questions please create an Issue in this repo or contact us via email: [email protected].
@@ -33,12 +32,25 @@ it is important to note that conflation occurs between similar feature types (e.
 polyline to Transportation polyline, etc.).
 
 ## Supported Data Formats
-_Import:_ Hootenanny can ingest shapefiles, openstreetmap (.osm), ESRI File Geodatabase (.gdb), and geonames.org (.geonames).  Additionally, 
-.zip files containing shapefiles and/or .gdb files can be uploaded.
-
-_Export:_ There are currently four options for exporting data from Hootenanny: File Geodatabase (FGDB), Shapefile, OSM and Web 
-Feature Service (WFS). Note that .gdb, Shapefile, and OSM formats are exported as a zip file containing all of the relevant 
-associated files, while WFS is exported as WFS Get Capabilities service URL that can be added into an OGC-enabled third party application
+_Import:_ Hootenanny can ingest from:
+* Shapefile (.shp)
+* OpenStreetMap (.osm)
+* ESRI File Geodatabase (.gdb)
+* .zip files containing shapefiles and/or .gdb files
+* geonames.org (.geonames)
+* OSM API database sources (MapEdit, etc.; experimental feature; see documentation for workflow)
+
+_Export:_ Hootenanny can export to: 
+* Shapefile (.shp)
+* OpenStreetMap (.osm)
+* ESRI File Geodatabase (.gdb)
+* Web Feature Service (WFS)
+* OSM API database (MapEdit, etc.; experimental feature; see documentation for workflow)
+
+Note that .gdb, Shapefile, and OSM formats are exported as a zip file containing all of the relevant 
+associated files, while WFS is exported as WFS Get Capabilities service URL that can be added into 
+an OGC-enabled third party application, and OSM API database export is done with an OSM changeset
+representing the before/after difference of conflated data.
 
 ## Tag Schemas
 Hootenanny leverages the OSM key value pair tag concept and PostgreSQL database structure to support translation between various 
@@ -53,6 +65,12 @@ or on the [releases](https://github.com/ngageoint/hootenanny/releases) under eac
 Instructions to launch a Hootenanny virtual machine using [Vagrant](https://www.vagrantup.com/) may
 be found [here](https://github.com/ngageoint/hootenanny/blob/master/VAGRANT.md).
 
+Optional file download:
+* https://s3.amazonaws.com/hoot-rpms/support-files/words1.sqlite.bz2
+
+If you're compiling manually on CentOS, you'll need:
+** https://github.com/ngageoint/hootenanny/releases/download/v0.2.16/geos-3.3.8-patched.tgz
+
 Additional instructions for setting up a Hootenanny development environment may be found in the Hootenanny
 Developer's Guide.
 
@@ -92,9 +110,11 @@ In alphabetical order:
 * Brian Hatchl ([email protected])
 * Matt Jackson ([email protected])
 * Ben Marchant ([email protected])
-* Terry Ott ([email protected])
+* Dmitriy Mylov ([email protected])
+* Seth Putnam ([email protected])
+* Micah Schicker ([email protected])
 * Greg Schmidt ([email protected])
 * Josh Sisskind ([email protected])
 * Ming Su ([email protected])
 * Jason R. Surratt ([email protected])
-* Brandon Witham ([email protected])
+* Brandon Witham ([email protected])

VAGRANT.md

@@ -50,15 +50,7 @@ vagrant plugin install vagrant-vmware-workstation
 
 # Vagrant Provisioning
 
-The initialization of the vagrant vm will take about an hour to download required software from the internet and set it up as a running system. Once it is complete, uncomment the `#, group: "tomcat6"` portion of the in Vagrantfile to allow the webapp to write to shared folders.
-
-    # argument is a set of non-required options.
-    config.vm.synced_folder ".", "/home/vagrant/hoot", group: "tomcat6", mount_options: ["dmode=775,fmode=775"]
-    # UNCOMMENT group after inital provisioning, then run vagrant reload
-
-Then issue the reload command for the changes to take effect on the vm:
-
-    vagrant reload
+The initialization of the vagrant vm will take about an hour to download required software from the internet and set it up as a running system. ~~Once it is complete, uncomment the `#, group: "tomcat6"` portion of the in Vagrantfile to allow the webapp to write to shared folders.~~  The previous workaround step is no longer needed as the provision script adds vagrant and tomcat6 users to each others group.
 
 You should be able to log into the running VM by typing:
 
@@ -89,5 +81,5 @@ If you've updated the code, you must connect to the vm via ssh to build and rede
     cd hoot
     source ./SetupEnv.sh
     make -sj$(nproc)
-    scripts/CopyWebAppsToTomcat.sh
+    sudo -u tomcat6 scripts/vagrantDeployTomcat.sh 
 

VagrantBuild.sh

@@ -9,9 +9,8 @@ source ./SetupEnv.sh
 
 echo "### Configuring Hoot..."
 echo HOOT_HOME: $HOOT_HOME
-cp conf/DatabaseConfig.sh.orig conf/DatabaseConfig.sh
 
-aclocal && autoconf && autoheader && automake && ./configure --quiet --with-rnd --with-services --with-uitests
+aclocal && autoconf && autoheader && automake --add-missing && ./configure --quiet --with-rnd --with-services --with-uitests
 
 if [ ! -f LocalConfig.pri ] && ! grep --quiet QMAKE_CXX LocalConfig.pri; then
     echo 'Customizing LocalConfig.pri...'

VagrantProvision.sh

@@ -29,7 +29,7 @@ if [ ! -f /etc/apt/sources.list.d/pgdg.list ]; then
 fi
 
 echo "### Installing dependencies from repos..."
-sudo apt-get -q -y install texinfo g++ libicu-dev libqt4-dev git-core libboost-dev libcppunit-dev libcv-dev libopencv-dev libgdal-dev liblog4cxx10-dev libnewmat10-dev libproj-dev python-dev libjson-spirit-dev automake1.11 protobuf-compiler libprotobuf-dev gdb libqt4-sql-psql libgeos++-dev swig lcov tomcat6 openjdk-7-jdk openjdk-7-dbg maven libstxxl-dev nodejs-dev nodejs-legacy doxygen xsltproc asciidoc pgadmin3 curl npm libxerces-c28 libglpk-dev libboost-all-dev source-highlight texlive-lang-arabic texlive-lang-hebrew texlive-lang-cyrillic graphviz w3m python-setuptools python python-pip git ccache libogdi3.2-dev gnuplot python-matplotlib libqt4-sql-sqlite ruby ruby-dev xvfb zlib1g-dev patch x11vnc openssh-server htop unzip postgresql-9.5  postgresql-client-9.5 postgresql-9.5-postgis-scripts postgresql-9.5-postgis-2.2 >> Ubuntu_upgrade.txt 2>&1
+sudo apt-get -q -y install texinfo g++ libicu-dev libqt4-dev git-core libboost-dev libcppunit-dev libcv-dev libopencv-dev libgdal-dev liblog4cxx10-dev libnewmat10-dev libproj-dev python-dev libjson-spirit-dev automake protobuf-compiler libprotobuf-dev gdb libqt4-sql-psql libgeos++-dev swig lcov tomcat6 openjdk-7-jdk openjdk-7-dbg maven libstxxl-dev nodejs-dev nodejs-legacy doxygen xsltproc asciidoc pgadmin3 curl npm libxerces-c28 libglpk-dev libboost-all-dev source-highlight texlive-lang-arabic texlive-lang-hebrew texlive-lang-cyrillic graphviz w3m python-setuptools python python-pip git ccache libogdi3.2-dev gnuplot python-matplotlib libqt4-sql-sqlite ruby ruby-dev xvfb zlib1g-dev patch x11vnc openssh-server htop unzip postgresql-9.5  postgresql-client-9.5 postgresql-9.5-postgis-scripts postgresql-9.5-postgis-2.2 >> Ubuntu_upgrade.txt 2>&1
 
 if ! dpkg -l | grep --quiet dictionaries-common; then
     # See /usr/share/doc/dictionaries-common/README.problems for details
@@ -71,43 +71,68 @@ fi
 
 if ! grep --quiet "PATH=" ~/.profile; then
     echo "Adding path vars to profile..."
-    echo "export PATH=\$PATH:\$HOME/.gem/ruby/1.9.1/bin:\$HOME/bin:$HOOT_HOME/bin" >> ~/.profile
+    #echo "export PATH=\$PATH:\$HOME/.gem/ruby/1.9.1/bin:\$HOME/bin:$HOOT_HOME/bin" >> ~/.profile
+    echo "export PATH=\$PATH:\$HOME/bin:$HOOT_HOME/bin" >> ~/.profile
     source ~/.profile
 fi
 
+# Ruby via rvm - from rvm.io
+gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+
+curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s stable
+
+source /home/vagrant/.rvm/scripts/rvm
+
+rvm install ruby-2.3
+rvm --default use 2.3
+
+# Don't install documentation for gems
+cat > ~/.gemrc <<EOT
+  install: --no-document
+  update: --no-document
+EOT
+
 # gem installs are *very* slow, hence all the checks in place here to facilitate debugging
 echo "### Installing cucumber gems..."
 gem list --local | grep -q mime-types
 if [ $? -eq 1 ]; then
-   sudo gem install mime-types -v 2.6.2
+   #sudo gem install mime-types -v 2.6.2
+   gem install mime-types 
 fi
 gem list --local | grep -q capybara
 if [ $? -eq 1 ]; then
-   sudo gem install capybara -v 2.5.0
+   #sudo gem install capybara -v 2.5.0
+   gem install capybara
 fi
 gem list --local | grep -q cucumber
 if [ $? -eq 1 ]; then
-   sudo gem install cucumber
+   #sudo gem install cucumber
+   gem install cucumber
 fi
 gem list --local | grep -q capybara-webkit
 if [ $? -eq 1 ]; then
-   sudo gem install capybara-webkit
+   #sudo gem install capybara-webkit
+   gem install capybara-webkit
 fi
 gem list --local | grep -q selenium-webdriver
 if [ $? -eq 1 ]; then
-   sudo gem install selenium-webdriver
+   #sudo gem install selenium-webdriver
+   gem install selenium-webdriver
 fi
 gem list --local | grep -q rspec
 if [ $? -eq 1 ]; then
-   sudo gem install rspec
+   #sudo gem install rspec
+   gem install rspec
 fi
 gem list --local | grep -q capybara-screenshot
 if [ $? -eq 1 ]; then
-   sudo gem install capybara-screenshot
+   #sudo gem install capybara-screenshot
+   gem install capybara-screenshot
 fi
 gem list --local | grep -q selenium-cucumber
 if [ $? -eq 1 ]; then
-   sudo gem install selenium-cucumber
+   #sudo gem install selenium-cucumber
+   gem install selenium-cucumber
 fi
 
 # Make sure that we are in ~ before trying to wget & install stuff

Vagrantfile

@@ -67,8 +67,7 @@ Vagrant.configure(2) do |config|
   # the path on the host to the actual folder. The second argument is
   # the path on the guest to mount the folder. And the optional third
   # argument is a set of non-required options.
-  config.vm.synced_folder ".", "/home/vagrant/hoot"#, group: "tomcat6"#, mount_options: ["dmode=775,fmode=775"]
-  # UNCOMMENT group after initial provisioning, then run vagrant reload
+  config.vm.synced_folder ".", "/home/vagrant/hoot"
 
   # Provider-specific configuration so you can fine-tune various
   # backing providers for Vagrant. These expose provider-specific options.