1212import java .sql .SQLException ;
1313import java .util .List ;
1414import java .util .Map ;
15+ import java .util .concurrent .locks .ReentrantLock ;
1516
1617/**
1718 * Nominatim update logic
2122
2223public class NominatimUpdater {
2324 private static final org .slf4j .Logger LOGGER = org .slf4j .LoggerFactory .getLogger (NominatimUpdater .class );
24- private final Integer minRank = 1 ;
25- private final Integer maxRank = 30 ;
26- private final JdbcTemplate template ;
25+
26+ private static final int CREATE = 1 ;
27+ private static final int UPDATE = 2 ;
28+ private static final int DELETE = 100 ;
29+
30+ private static final int MIN_RANK = 1 ;
31+ private static final int MAX_RANK = 30 ;
32+
33+ private final JdbcTemplate template ;
2734 private final NominatimConnector exporter ;
2835
2936 private Updater updater ;
3037
38+ /**
39+ * when updating lockout other threads
40+ */
41+ private ReentrantLock updateLock = new ReentrantLock ();
42+
3143 public void setUpdater (Updater updater ) {
3244 this .updater = updater ;
3345 }
3446
3547 public void update () {
36- for (Integer rank = this .minRank ; rank <= this .maxRank ; rank ++) {
37- LOGGER .info (String .format ("Starting rank %d" , rank ));
38- for (Map <String , Object > sector : getIndexSectors (rank ))
39- for (UpdateRow place : getIndexSectorPlaces (rank , (Integer ) sector .get ("geometry_sector" ))) {
40-
41- template .update ("update placex set indexed_status = 0 where place_id = ?" , place .getPlaceId ());
42- final PhotonDoc updatedDoc = exporter .getByPlaceId (place .getPlaceId ());
43-
44- switch (place .getIndexdStatus ()) {
45- case 1 :
46- if (updatedDoc .isUsefulForIndex ())
47- updater .create (updatedDoc );
48- break ;
49- case 2 :
50- if (!updatedDoc .isUsefulForIndex ())
51- updater .delete (place .getPlaceId ());
52-
53- updater .updateOrCreate (updatedDoc );
54- break ;
55- case 100 :
56- updater .delete (place .getPlaceId ());
57- break ;
58- default :
59- LOGGER .error (String .format ("Unknown index status %d" , place .getIndexdStatus ()));
60- break ;
48+ if (updateLock .tryLock ()) {
49+ try {
50+ int updatedPlaces = 0 ;
51+ int deletedPlaces = 0 ;
52+ for (int rank = MIN_RANK ; rank <= MAX_RANK ; rank ++) {
53+ LOGGER .info (String .format ("Starting rank %d" , rank ));
54+ for (Map <String , Object > sector : getIndexSectors (rank )) {
55+ for (UpdateRow place : getIndexSectorPlaces (rank , (Integer ) sector .get ("geometry_sector" ))) {
56+ long placeId = place .getPlaceId ();
57+ template .update ("update placex set indexed_status = 0 where place_id = ?;" , placeId );
58+
59+ Integer indexedStatus = place .getIndexdStatus ();
60+ if (indexedStatus == DELETE || (indexedStatus == UPDATE && rank == MAX_RANK )) {
61+ updater .delete (placeId );
62+ if (indexedStatus == DELETE ) {
63+ deletedPlaces ++;
64+ continue ;
65+ }
66+ indexedStatus = CREATE ; // always create
67+ }
68+ updatedPlaces ++;
69+
70+ final List <PhotonDoc > updatedDocs = exporter .getByPlaceId (place .getPlaceId ());
71+ boolean wasUseful = false ;
72+ for (PhotonDoc updatedDoc : updatedDocs ) {
73+ switch (indexedStatus ) {
74+ case CREATE :
75+ if (updatedDoc .isUsefulForIndex ()) {
76+ updater .create (updatedDoc );
77+ }
78+ break ;
79+ case UPDATE :
80+ if (updatedDoc .isUsefulForIndex ()) {
81+ updater .updateOrCreate (updatedDoc );
82+ wasUseful = true ;
83+ }
84+ break ;
85+ default :
86+ LOGGER .error (String .format ("Unknown index status %d" , indexedStatus ));
87+ break ;
88+ }
89+ }
90+ if (indexedStatus == UPDATE && !wasUseful ) {
91+ // only true when rank != 30
92+ // if no documents for the place id exist this will likely cause moaning
93+ updater .delete (placeId );
94+ updatedPlaces --;
95+ }
96+ }
6197 }
6298 }
63- }
6499
65- updater .finish ();
100+ LOGGER .info (String .format ("%d places created or updated, %d deleted" , updatedPlaces , deletedPlaces ));
101+
102+ // update documents generated from address interpolations
103+ // .isUsefulForIndex() should always return true for documents
104+ // created from interpolations so no need to check them
105+ LOGGER .info ("Starting interpolations" );
106+ int updatedInterpolations = 0 ;
107+ int deletedInterpolations = 0 ;
108+ int interpolationDocuments = 0 ;
109+ for (Map <String , Object > sector : template .queryForList (
110+ "select geometry_sector,count(*) from location_property_osmline where indexed_status > 0 group by geometry_sector order by geometry_sector;" )) {
111+ for (UpdateRow place : getIndexSectorInterpolations ((Integer ) sector .get ("geometry_sector" ))) {
112+ long placeId = place .getPlaceId ();
113+ template .update ("update location_property_osmline set indexed_status = 0 where place_id = ?;" , placeId );
114+
115+ Integer indexedStatus = place .getIndexdStatus ();
116+ if (indexedStatus != CREATE ) {
117+ updater .delete (placeId );
118+ if (indexedStatus == DELETE ) {
119+ deletedInterpolations ++;
120+ continue ;
121+ }
122+ }
123+ updatedInterpolations ++;
124+
125+ final List <PhotonDoc > updatedDocs = exporter .getInterpolationsByPlaceId (place .getPlaceId ());
126+ for (PhotonDoc updatedDoc : updatedDocs ) {
127+ updater .create (updatedDoc );
128+ interpolationDocuments ++;
129+ }
130+ }
131+ }
132+ LOGGER .info (String .format ("%d interpolations created or updated, %d deleted, %d documents added or updated" , updatedInterpolations ,
133+ deletedInterpolations , interpolationDocuments ));
134+ updater .finish ();
135+ template .update ("update import_status set indexed=true;" ); // indicate that we are finished
136+
137+ LOGGER .info ("Finished updating" );
138+ } finally {
139+ updateLock .unlock ();
140+ }
141+ } else {
142+ LOGGER .info ("Update already in progress" );
143+ }
66144 }
67145
68146 private List <Map <String , Object >> getIndexSectors (Integer rank ) {
69- return template .queryForList ("select geometry_sector,count(*) from placex where rank_search = ? " +
70- "and indexed_status > 0 group by geometry_sector order by geometry_sector;" , rank );
147+ return template .queryForList ("select geometry_sector,count(*) from placex where rank_search = ? "
148+ + "and indexed_status > 0 group by geometry_sector order by geometry_sector;" , rank );
71149 }
72150
73151 private List <UpdateRow > getIndexSectorPlaces (Integer rank , Integer geometrySector ) {
74- return template .query ("select place_id, indexed_status from placex where rank_search = ?" +
75- " and geometry_sector = ? and indexed_status > 0;" , new Object []{rank , geometrySector }, new RowMapper <UpdateRow >() {
76- @ Override
77- public UpdateRow mapRow (ResultSet rs , int rowNum ) throws SQLException {
78- UpdateRow updateRow = new UpdateRow ();
79- updateRow .setPlaceId (rs .getLong ("place_id" ));
80- updateRow .setIndexdStatus (rs .getInt ("indexed_status" ));
81- return updateRow ;
82- }
83- });
152+ return template .query ("select place_id, indexed_status from placex where rank_search = ?" + " and geometry_sector = ? and indexed_status > 0;" ,
153+ new Object [] { rank , geometrySector }, new RowMapper <UpdateRow >() {
154+ @ Override
155+ public UpdateRow mapRow (ResultSet rs , int rowNum ) throws SQLException {
156+ UpdateRow updateRow = new UpdateRow ();
157+ updateRow .setPlaceId (rs .getLong ("place_id" ));
158+ updateRow .setIndexdStatus (rs .getInt ("indexed_status" ));
159+ return updateRow ;
160+ }
161+ });
162+ }
163+
164+ private List <UpdateRow > getIndexSectorInterpolations (Integer geometrySector ) {
165+ return template .query ("select place_id, indexed_status from location_property_osmline where geometry_sector = ? and indexed_status > 0;" ,
166+ new Object [] { geometrySector }, new RowMapper <UpdateRow >() {
167+ @ Override
168+ public UpdateRow mapRow (ResultSet rs , int rowNum ) throws SQLException {
169+ UpdateRow updateRow = new UpdateRow ();
170+ updateRow .setPlaceId (rs .getLong ("place_id" ));
171+ updateRow .setIndexdStatus (rs .getInt ("indexed_status" ));
172+ return updateRow ;
173+ }
174+ });
84175 }
85176
86177 /**
178+ * Creates a new instance
179+ *
180+ * @param host Nominatim database host
181+ * @param port Nominatim database port
182+ * @param database Nominatim database name
183+ * @param username Nominatim database username
184+ * @param password Nominatim database password
87185 */
88186 public NominatimUpdater (String host , int port , String database , String username , String password ) {
89187 BasicDataSource dataSource = new BasicDataSource ();
@@ -92,7 +190,7 @@ public NominatimUpdater(String host, int port, String database, String username,
92190 dataSource .setUsername (username );
93191 dataSource .setPassword (password );
94192 dataSource .setDriverClassName (JtsWrapper .class .getCanonicalName ());
95- dataSource .setDefaultAutoCommit (false );
193+ dataSource .setDefaultAutoCommit (true );
96194
97195 exporter = new NominatimConnector (host , port , database , username , password );
98196 template = new JdbcTemplate (dataSource );
0 commit comments