Merge pull request #580 from lonvia/location-bias-and-query-structure

Rework location bias and query builder in general

Author: lonvia

README.md

@@ -122,10 +122,18 @@ http://localhost:2322/api?q=berlin
 http://localhost:2322/api?q=berlin&lon=10&lat=52
 ```
 
-Increase this bias (range is 0.1 to 10, default is 1.6)
+There are two optional parameters to influence the location bias. 'zoom'
+describes the radius around the center to focus on. This is a number that
+should correspond roughly to the map zoom parameter of a corresponding map.
+The default is `zoom=16`.
+
+The `location_bias_scale` describes how much the prominence of a result should
+still be taken into account. Sensible values go from 0.0 (ignore prominence
+almost completely) to 1.0 (prominence has approximately the same influence).
+The default is 0.2.
 
 ```
-http://localhost:2322/api?q=berlin&lon=10&lat=52&location_bias_scale=2
+http://localhost:2322/api?q=berlin&lon=10&lat=52&zoom=12&location_bias_scale=0.1
 ```
 
 #### Reverse geocode a coordinate

src/main/java/de/komoot/photon/elasticsearch/IndexMapping.java

@@ -39,8 +39,8 @@ public void putMapping(Client client, String indexName, String indexType) {
     public IndexMapping addLanguages(String[] languages) {
         // define collector json strings
         String copyToCollectorString = "{\"type\":\"text\",\"index\":false,\"copy_to\":[\"collector.{lang}\"]}";
-        String nameToCollectorString = "{\"type\":\"text\",\"index\":false,\"fields\":{\"ngrams\":{\"type\":\"text\",\"analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"text\",\"analyzer\":\"index_raw\"}},\"copy_to\":[\"collector.{lang}\"]}";
-        String collectorString = "{\"type\":\"text\",\"index\":false,\"fields\":{\"ngrams\":{\"type\":\"text\",\"analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"text\",\"analyzer\":\"index_raw\"}},\"copy_to\":[\"collector.{lang}\"]}";
+        String nameToCollectorString = "{\"type\":\"text\",\"index\":false,\"fields\":{\"ngrams\":{\"type\":\"text\",\"analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"text\",\"analyzer\":\"index_raw\",\"search_analyzer\":\"search_raw\"}},\"copy_to\":[\"collector.{lang}\"]}";
+        String collectorString = "{\"type\":\"text\",\"index\":false,\"fields\":{\"ngrams\":{\"type\":\"text\",\"analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"text\",\"analyzer\":\"index_raw\",\"search_analyzer\":\"search_raw\"}},\"copy_to\":[\"collector.{lang}\"]}";
 
         JSONObject placeObject = mappings.optJSONObject("place");
         JSONObject propertiesObject = placeObject == null ? null : placeObject.optJSONObject("properties");

src/main/java/de/komoot/photon/query/PhotonQueryBuilder.java

@@ -11,9 +11,7 @@
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
-import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.index.query.functionscore.WeightBuilder;
 
 import java.util.*;
 
@@ -34,6 +32,8 @@
  * Created by Sachin Dole on 2/12/2015.
  */
 public class PhotonQueryBuilder {
+    private static final String[] ALT_NAMES = new String[]{"alt", "int", "loc", "old", "reg", "housename"};
+
     private FunctionScoreQueryBuilder finalQueryWithoutTagFilterBuilder;
 
     private BoolQueryBuilder queryBuilderForTopLevelFilter;
@@ -50,27 +50,25 @@ public class PhotonQueryBuilder {
 
     protected ArrayList<FilterFunctionBuilder> alFilterFunction4QueryBuilder = new ArrayList<>(1);
 
-    protected BoolQueryBuilder query4QueryBuilder;
-
 
     private PhotonQueryBuilder(String query, String language, List<String> languages, boolean lenient) {
-        query4QueryBuilder = QueryBuilders.boolQuery();
+        BoolQueryBuilder query4QueryBuilder = QueryBuilders.boolQuery();
 
+        // 1. All terms of the quey must be contained in the place record somehow. Be more lenient on second try.
+        QueryBuilder collectorQuery;
         if (lenient) {
-            BoolQueryBuilder builder = QueryBuilders.boolQuery()
+            collectorQuery = QueryBuilders.boolQuery()
                     .should(QueryBuilders.matchQuery("collector.default", query)
-                                .fuzziness(Fuzziness.ONE)
-                                .prefixLength(2)
-                                .analyzer("search_ngram")
-                                .minimumShouldMatch("-1"))
+                            .fuzziness(Fuzziness.ONE)
+                            .prefixLength(2)
+                            .analyzer("search_ngram")
+                            .minimumShouldMatch("-1"))
                     .should(QueryBuilders.matchQuery(String.format("collector.%s.ngrams", language), query)
-                                .fuzziness(Fuzziness.ONE)
-                                .prefixLength(2)
-                                .analyzer("search_ngram")
-                                .minimumShouldMatch("-1"))
+                            .fuzziness(Fuzziness.ONE)
+                            .prefixLength(2)
+                            .analyzer("search_ngram")
+                            .minimumShouldMatch("-1"))
                     .minimumShouldMatch("1");
-
-            query4QueryBuilder.must(builder);
         } else {
             MultiMatchQueryBuilder builder =
                     QueryBuilders.multiMatchQuery(query).field("collector.default", 1.0f).type(MultiMatchQueryBuilder.Type.CROSS_FIELDS).prefixLength(2).analyzer("search_ngram").minimumShouldMatch("100%");
@@ -79,31 +77,65 @@ private PhotonQueryBuilder(String query, String language, List<String> languages
                 builder.field(String.format("collector.%s.ngrams", lang), lang.equals(language) ? 1.0f : 0.6f);
             }
 
-            query4QueryBuilder.must(builder);
+            collectorQuery = builder;
         }
 
-        query4QueryBuilder
-                .should(QueryBuilders.matchQuery(String.format("name.%s.raw", language), query).boost(200)
-                        .analyzer("search_raw"))
-                .should(QueryBuilders.matchQuery(String.format("collector.%s.raw", language), query).boost(100)
-                        .analyzer("search_raw"));
+        query4QueryBuilder.must(collectorQuery);
+
+        // 2. Prefer records that have the full names in. For address records with housenumbers this is the main
+        //    filter creterion because they have no name. Therefore boost the score in this case.
+        MultiMatchQueryBuilder hnrQuery = QueryBuilders.multiMatchQuery(query)
+                .field("collector.default.raw", 1.0f)
+                .type(MultiMatchQueryBuilder.Type.BEST_FIELDS);
+
+        for (String lang : languages) {
+            hnrQuery.field(String.format("collector.%s.raw", lang), lang.equals(language) ? 1.0f : 0.6f);
+        }
+
+        query4QueryBuilder.should(QueryBuilders.functionScoreQuery(hnrQuery.boost(0.3f), new FilterFunctionBuilder[]{
+                new FilterFunctionBuilder(QueryBuilders.matchQuery("housenumber", query).analyzer("standard"), new WeightBuilder().setWeight(10f))
+        }));
+
+        // 3. Either the name or housenumber must be in the query terms.
+        String defLang = "default".equals(language) ? languages.get(0) : language;
+        MultiMatchQueryBuilder nameNgramQuery = QueryBuilders.multiMatchQuery(query)
+                .type(MultiMatchQueryBuilder.Type.BEST_FIELDS)
+                .fuzziness(lenient ? Fuzziness.ONE : Fuzziness.ZERO)
+                .analyzer("search_ngram");
+
+        for (String lang: languages) {
+            nameNgramQuery.field(String.format("name.%s.ngrams", lang), lang.equals(defLang) ? 1.0f : 0.4f);
+        }
 
-        // this is former general-score, now inline
-        String strCode = "double score = 1 + doc['importance'].value * 100; score";
-        ScriptScoreFunctionBuilder functionBuilder4QueryBuilder =
-                ScoreFunctionBuilders.scriptFunction(new Script(ScriptType.INLINE, "painless", strCode, new HashMap<String, Object>()));
+        for (String alt: ALT_NAMES) {
+            nameNgramQuery.field(String.format("name.%s.raw", alt), 0.4f);
+        }
+
+        if (query.indexOf(',') < 0 && query.indexOf(' ') < 0) {
+            query4QueryBuilder.must(nameNgramQuery.boost(2f));
+        } else {
+            query4QueryBuilder.must(QueryBuilders.boolQuery()
+                                        .should(nameNgramQuery)
+                                        .should(QueryBuilders.matchQuery("housenumber", query).analyzer("standard"))
+                                        .minimumShouldMatch("1"));
+        }
+
+        // 4. Rerank results for having the full name in the default language.
+        query4QueryBuilder
+                .should(QueryBuilders.matchQuery(String.format("name.%s.raw", language), query));
 
-        alFilterFunction4QueryBuilder.add(new FilterFunctionBuilder(functionBuilder4QueryBuilder));
 
-        finalQueryWithoutTagFilterBuilder = new FunctionScoreQueryBuilder(query4QueryBuilder, alFilterFunction4QueryBuilder.toArray(new FilterFunctionBuilder[0]))
-                .boostMode(CombineFunction.MULTIPLY).scoreMode(ScoreMode.MULTIPLY);
+        // Weigh the resulting score by importance. Use a linear scale function that ensures that the weight
+        // never drops to 0 and cancels out the ES score.
+        finalQueryWithoutTagFilterBuilder = QueryBuilders.functionScoreQuery(query4QueryBuilder, new FilterFunctionBuilder[]{
+                new FilterFunctionBuilder(ScoreFunctionBuilders.linearDecayFunction("importance", "1.0", "0.6"))
+        });
 
-        // @formatter:off
+        // Filter for later: records that have a housenumber and no name must only appear when the housenumber matches.
         queryBuilderForTopLevelFilter = QueryBuilders.boolQuery()
                 .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("housenumber")))
                 .should(QueryBuilders.matchQuery("housenumber", query).analyzer("standard"))
                 .should(QueryBuilders.existsQuery(String.format("name.%s.raw", language)));
-        // @formatter:on
 
         state = State.PLAIN;
     }
@@ -120,21 +152,27 @@ public static PhotonQueryBuilder builder(String query, String language, List<Str
         return new PhotonQueryBuilder(query, language, languages, lenient);
     }
 
-    public PhotonQueryBuilder withLocationBias(Point point, double scale) {
-        if (point == null) return this;
+    public PhotonQueryBuilder withLocationBias(Point point, double scale, int zoom) {
+        if (point == null || zoom < 4) return this;
+
+        if (zoom > 18) {
+            zoom = 18;
+        }
+        double radius = (1 << (18 - zoom)) * 0.25;
+
+        if (scale <= 0.0) {
+            scale = 0.0000001;
+        }
+
         Map<String, Object> params = newHashMap();
         params.put("lon", point.getX());
         params.put("lat", point.getY());
 
-        scale = Math.abs(scale);
-        String strCode = "double dist = doc['coordinate'].planeDistance(params.lat, params.lon); " +
-                "double score = 0.1 + " + scale + " / (1.0 + dist * 0.001 / 10.0); " +
-                "score";
-        ScriptScoreFunctionBuilder builder = ScoreFunctionBuilders.scriptFunction(new Script(ScriptType.INLINE, "painless", strCode, params));
-        alFilterFunction4QueryBuilder.add(new FilterFunctionBuilder(builder));
         finalQueryWithoutTagFilterBuilder =
-                new FunctionScoreQueryBuilder(query4QueryBuilder, alFilterFunction4QueryBuilder.toArray(new FilterFunctionBuilder[0]))
-                        .boostMode(CombineFunction.MULTIPLY);
+                QueryBuilders.functionScoreQuery(finalQueryWithoutTagFilterBuilder, new FilterFunctionBuilder[] {
+                     new FilterFunctionBuilder(ScoreFunctionBuilders.exponentialDecayFunction("coordinate", params, radius + "km", radius / 10 + "km", 0.8)),
+                     new FilterFunctionBuilder(ScoreFunctionBuilders.linearDecayFunction("importance", "1.0", scale))
+                }).boostMode(CombineFunction.MULTIPLY).scoreMode(ScoreMode.MAX);
         return this;
     }
 

src/main/java/de/komoot/photon/query/PhotonRequest.java

@@ -18,6 +18,7 @@ public class PhotonRequest implements Serializable {
     private Point locationForBias;
     private String language;
     private final double scale;
+    private final int zoom;
     private Envelope bbox;
     private boolean debug;
 
@@ -30,11 +31,12 @@ public class PhotonRequest implements Serializable {
     private Map<String, Set<String>> excludeTagValues;
 
 
-    public PhotonRequest(String query, int limit, Envelope bbox, Point locationForBias, double scale, String language, boolean debug) {
+    public PhotonRequest(String query, int limit, Envelope bbox, Point locationForBias, double scale, int zoom, String language, boolean debug) {
         this.query = query;
         this.limit = limit;
         this.locationForBias = locationForBias;
         this.scale = scale;
+        this.zoom = zoom;
         this.language = language;
         this.bbox = bbox;
         this.debug = debug;
@@ -60,6 +62,10 @@ public double getScaleForBias() {
         return scale;
     }
 
+    public int getZoomForBias() {
+        return zoom;
+    }
+
     public String getLanguage() {
         return language;
     }

src/main/java/de/komoot/photon/query/PhotonRequestFactory.java

@@ -19,7 +19,7 @@ public class PhotonRequestFactory {
     private final BoundingBoxParamConverter bboxParamConverter;
 
     private static final HashSet<String> REQUEST_QUERY_PARAMS = new HashSet<>(Arrays.asList("lang", "q", "lon", "lat",
-            "limit", "osm_tag", "location_bias_scale", "bbox", "debug"));
+            "limit", "osm_tag", "location_bias_scale", "bbox", "debug", "zoom"));
 
     public PhotonRequestFactory(List<String> supportedLanguages, String defaultLanguage) {
         this.languageResolver = new RequestLanguageResolver(supportedLanguages, defaultLanguage);
@@ -47,8 +47,7 @@ public PhotonRequest create(Request webRequest) throws BadRequestException {
         Point locationForBias = optionalLocationParamConverter.apply(webRequest);
         Envelope bbox = bboxParamConverter.apply(webRequest);
 
-        // don't use too high default value, see #306
-        double scale = 1.6;
+        double scale = 0.2;
         String scaleStr = webRequest.queryParams("location_bias_scale");
         if (scaleStr != null && !scaleStr.isEmpty())
             try {
@@ -57,9 +56,19 @@ public PhotonRequest create(Request webRequest) throws BadRequestException {
                 throw new BadRequestException(400, "invalid parameter 'location_bias_scale' must be a number");
             }
 
+        int zoom = 16;
+        String zoomStr = webRequest.queryParams("zoom");
+        if (zoomStr != null && !zoomStr.isEmpty()) {
+            try {
+                zoom = Integer.parseInt(zoomStr);
+            } catch (NumberFormatException e) {
+                throw new BadRequestException(400, "Invalid parameter 'zoom'. Must be a number.");
+            }
+        }
+
         boolean debug = webRequest.queryParams("debug") != null;
 
-        PhotonRequest request = new PhotonRequest(query, limit, bbox, locationForBias, scale, language, debug);
+        PhotonRequest request = new PhotonRequest(query, limit, bbox, locationForBias, scale, zoom, language, debug);
 
         QueryParamsMap tagFiltersQueryMap = webRequest.queryMap("osm_tag");
         if (new CheckIfFilteredRequest().execute(tagFiltersQueryMap)) {

src/main/java/de/komoot/photon/searcher/PhotonRequestHandler.java

@@ -59,7 +59,7 @@ public PhotonQueryBuilder buildQuery(PhotonRequest photonRequest, boolean lenien
                 withoutKeys(photonRequest.notKeys()).
                 withoutValues(photonRequest.notValues()).
                 withTagsNotValues(photonRequest.tagNotValues()).
-                withLocationBias(photonRequest.getLocationForBias(), photonRequest.getScaleForBias()).
+                withLocationBias(photonRequest.getLocationForBias(), photonRequest.getScaleForBias(), photonRequest.getZoomForBias()).
                 withBoundingBox(photonRequest.getBbox());
     }
 }

src/test/java/de/komoot/photon/ApiIntegrationTest.java

@@ -27,8 +27,8 @@ public class ApiIntegrationTest extends ESBaseTester {
     public void setUp() throws Exception {
         setUpES();
         Importer instance = makeImporter();
-        instance.add(createDoc(13.38886, 52.51704, 1000, 1000, "place", "city"));
-        instance.add(createDoc(13.39026, 52.54714, 1001, 1001, "place", "town"));
+        instance.add(createDoc(13.38886, 52.51704, 1000, 1000, "place", "city").importance(0.6));
+        instance.add(createDoc(13.39026, 52.54714, 1001, 1001, "place", "town").importance(0.3));
         instance.finish();
         refresh();
     }
@@ -112,6 +112,27 @@ public void testApiWithLocationBias() throws Exception {
         assertEquals("berlin", properties.getString("name"));
     }
 
+    /**
+     * Search with large location bias
+     */
+    @Test
+    public void testApiWithLargerLocationBias() throws Exception {
+        App.main(new String[]{"-cluster", TEST_CLUSTER_NAME, "-listen-port", Integer.toString(LISTEN_PORT), "-transport-addresses", "127.0.0.1"});
+        awaitInitialization();
+        HttpURLConnection connection = (HttpURLConnection) new URL("http://127.0.0.1:" + port() + "/api?q=berlin&limit=1&lat=52.54714&lon=13.39026&zoom=12&location_bias_scale=0.6")
+                .openConnection();
+        JSONObject json = new JSONObject(
+                new BufferedReader(new InputStreamReader(connection.getInputStream())).lines().collect(Collectors.joining("\n")));
+        JSONArray features = json.getJSONArray("features");
+        assertEquals(1, features.length());
+        JSONObject feature = features.getJSONObject(0);
+        JSONObject properties = feature.getJSONObject("properties");
+        assertEquals("W", properties.getString("osm_type"));
+        assertEquals("place", properties.getString("osm_key"));
+        assertEquals("city", properties.getString("osm_value"));
+        assertEquals("berlin", properties.getString("name"));
+    }
+
     /**
      * Reverse geocode test
      */