diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..7351e91 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,27 @@ +version: 2.1 + +jobs: + test: + docker: + - image: cimg/clojure:1.10.1 + - image: mongo:3.6 + steps: + - checkout + - run: lein test + + release: + docker: + - image: cimg/clojure:1.10.1 + steps: + - checkout + - run: lein deploy + +workflows: + ci: + jobs: + - test + - release: + requires: [test] + filters: + branches: + only: master diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1d33494 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,6 @@ +version: '2' +services: + mongodb: + image: mongo:3.6 + ports: + - '127.0.0.1:27017:27017' diff --git a/project.clj b/project.clj index fb9b332..eacf37c 100755 --- a/project.clj +++ b/project.clj @@ -10,7 +10,7 @@ :distribution :repo} :min-lein-version "2.0.0" :dependencies [[org.clojure/data.json "0.2.7"] - [org.mongodb/mongo-java-driver "3.10.2"] + [org.mongodb/mongodb-driver-legacy "4.3.1"] [org.clojure/clojure "1.10.1" :scope "provided"]] :deploy-repositories {"releases" {:url "https://repo.clojars.org" :creds :gpg}} ;; if a :dev profile is added, remember to update :aliases below to diff --git a/src/somnium/congomongo.clj b/src/somnium/congomongo.clj index c83f767..0b13275 100644 --- a/src/somnium/congomongo.clj +++ b/src/somnium/congomongo.clj @@ -28,12 +28,11 @@ [somnium.congomongo.coerce :refer [coerce coerce-fields coerce-index-fields]]) (:import [com.mongodb MongoClient MongoClientOptions MongoClientOptions$Builder MongoClientURI MongoCredential - DB DBCollection DBObject DBRef ServerAddress ReadPreference WriteConcern Bytes - AggregationOptions AggregationOptions$OutputMode - GroupCommand + DB DBCollection CursorType DBObject DBRef + ServerAddress ReadPreference WriteConcern + AggregationOptions MapReduceCommand MapReduceCommand$OutputType] [com.mongodb.gridfs GridFS] - [com.mongodb.util JSON] [org.bson.types ObjectId] (java.util.concurrent TimeUnit))) @@ -97,10 +96,10 @@ (defn- make-mongo-client (^com.mongodb.MongoClient - [addresses creds options] - (if (> (count addresses) 1) - (MongoClient. ^java.util.List addresses creds options) - (MongoClient. ^ServerAddress (first addresses) creds options))) + [addresses cred options] + (if (> (count addresses) 1) + (MongoClient. ^java.util.List addresses cred options) + (MongoClient. ^ServerAddress (first addresses) cred options))) (^com.mongodb.MongoClient [addresses options] @@ -143,7 +142,7 @@ (MongoCredential/createCredential username db (.toCharArray password)))) mongo (if credential - (make-mongo-client addresses [credential] options) + (make-mongo-client addresses credential options) (make-mongo-client addresses options)) n-db (if db (.getDB mongo db) nil)] @@ -263,21 +262,10 @@ When with-mongo and set-connection! interact, last one wins" (def write-concern-map {:acknowledged WriteConcern/ACKNOWLEDGED - :fsynced WriteConcern/FSYNCED :journaled WriteConcern/JOURNALED :majority WriteConcern/MAJORITY - :replica-acknowledged WriteConcern/REPLICA_ACKNOWLEDGED - :unacknowledged WriteConcern/UNACKNOWLEDGED - ;; these are pre-2.10.x names for write concern: - :fsync-safe WriteConcern/FSYNC_SAFE ;; deprecated - use :fsynced - :journal-safe WriteConcern/JOURNAL_SAFE ;; deprecated - use :journaled - :normal WriteConcern/NORMAL ;; deprecated - use :unacknowledged - :replicas-safe WriteConcern/REPLICAS_SAFE ;; deprecated - use :replica-acknowledged - :safe WriteConcern/SAFE ;; deprecated - use :acknowledged - ;; these are left for backward compatibility but are deprecated: - :replica-safe WriteConcern/REPLICAS_SAFE - :strict WriteConcern/SAFE - }) + :replica-acknowledged WriteConcern/W2 + :unacknowledged WriteConcern/UNACKNOWLEDGED}) (defn set-write-concern "Sets the write concern on the connection. Setting is a key in the @@ -350,20 +338,6 @@ When with-mongo and set-connection! interact, last one wins" ^String (named collection) (coerce options [:clojure :mongo]))) -(def query-option-map - {:tailable Bytes/QUERYOPTION_TAILABLE - :slaveok Bytes/QUERYOPTION_SLAVEOK - :oplogreplay Bytes/QUERYOPTION_OPLOGREPLAY - :notimeout Bytes/QUERYOPTION_NOTIMEOUT - :awaitdata Bytes/QUERYOPTION_AWAITDATA}) - -(defn calculate-query-options - "Calculates the cursor's query option from a list of options" - [options] - (reduce bit-or 0 (map query-option-map (if (keyword? options) - (list options) - options)))) - (def ^:private read-preference-map "Private map of facory functions of ReadPreferences to aliases." {:nearest (fn nearest ([] (ReadPreference/nearest)) ([tags] (ReadPreference/nearest tags))) @@ -415,6 +389,20 @@ When with-mongo and set-connection! interact, last one wins" [collection] (.getReadPreference (get-coll collection))) +(defn set-options! + "sets the options on the cursor" + [cursor {:keys [tailable secondary-preferred slaveok oplog notimeout awaitdata]}] + (when tailable + (.cursorType cursor CursorType/Tailable)) + (when awaitdata + (.cursorType cursor CursorType/TailableAwait)) + (when (or secondary-preferred slaveok) + (.setReadPreference cursor (ReadPreference/secondaryPreferred))) + (when oplog + (.oplogReplay cursor true)) + (when notimeout + (.noCursorTimeout cursor true))) + (defn fetch "Fetches objects from a collection. Note that MongoDB always adds the _id and _ns @@ -483,13 +471,12 @@ You should use fetch with :limit 1 instead."))); one? and sort should NEVER be c ; (with negative limit) doesn't match expectations therefore changed to keep limit as is. n-limit (or limit 0) n-sort (when sort (coerce sort [from :mongo])) - n-options (calculate-query-options options) n-preferences (cond (nil? read-preferences) nil (instance? ReadPreference read-preferences) read-preferences :else (somnium.congomongo/read-preference read-preferences))] (cond - count? (.getCount n-col n-where n-only) + count? (.getCount n-col n-where) ;; The find command isn't documented so there's no nice way to build a ;; find command that adds read-preferences when necessary @@ -505,10 +492,21 @@ You should use fetch with :limit 1 instead."))); one? and sort should NEVER be c (.setReadPreference cursor n-preferences)) (when hint (if (string? hint) - (.hint cursor ^String hint) + ;; hint no longer supports strings + ;; .hintString exists for DBCollectionCountOptions but not + ;; for DBCursor. It would be possible to hack support by + ;; creating a DBCollectionCountOptions that is not used + ;; except for setting a hint string on it and getting the + ;; hint out as an object. For now follow the mongo + ;; deprecation and let this be a place where we don't have + ;; backwards compatibility. They have also added the string + ;; hint functionality back into the non-legacy client so it + ;; is possible that support will be restored via the java + ;; client. + (throw (IllegalArgumentException. "String hints are not currently supported. Use a seq instead.")) (.hint cursor ^DBObject (coerce-index-fields hint)))) - (when n-options - (.setOptions cursor n-options)) + (when options + (set-options! cursor options)) (when n-sort (.sort cursor n-sort)) (when skip @@ -726,7 +724,6 @@ You should use fetch with :limit 1 instead."))); one? and sort should NEVER be c cursor (.aggregate (get-coll coll) ^java.util.List (coerce (conj ops op) [from :mongo]) ^AggregationOptions (-> (AggregationOptions/builder) - (.outputMode AggregationOptions$OutputMode/CURSOR) (.build)))] {:serverUsed (.toString (.getServerAddress cursor)) :result (coerce cursor [:mongo to] :many true) diff --git a/src/somnium/congomongo/coerce.clj b/src/somnium/congomongo/coerce.clj index 71fb98f..9cf8036 100755 --- a/src/somnium/congomongo/coerce.clj +++ b/src/somnium/congomongo/coerce.clj @@ -20,11 +20,16 @@ (ns somnium.congomongo.coerce (:require [clojure.data.json :refer [write-str read-str]]) - (:import [clojure.lang IPersistentMap IPersistentVector Keyword] + (:import [clojure.lang IPersistentMap Keyword] [java.util Map List Set] [com.mongodb DBObject BasicDBObject BasicDBList] - [com.mongodb.gridfs GridFSFile] - [com.mongodb.util JSON])) + org.bson.json.JsonWriterSettings)) + +(def ^:private json-settings + ; The default output mode for JSON is RELAXED, which is what we require. + ; https://www.javadoc.io/doc/org.mongodb/mongo-java-driver/3.12.9/org/bson/json/JsonWriterSettings.Builder.html#maxLength(int) + ; https://github.com/mongodb/specifications/blob/df6be82f865e9b72444488fd62ae1eb5fca18569/source/extended-json.rst + (.build (JsonWriterSettings/builder))) (def ^{:dynamic true :doc "Set this to false to prevent coercion from setting string keys to keywords" @@ -47,6 +52,13 @@ (string? x) (instance? java.util.Map x)))) +(defn json->mongo [^String s] + (BasicDBObject/parse s)) + +(defn ^String mongo->json [^BasicDBObject dbo] + (.toJson dbo json-settings)) + + ;;; Converting data from mongo into Clojure data objects (defprotocol ConvertibleFromMongo @@ -126,11 +138,11 @@ *translations* {[:clojure :mongo ] #'clojure->mongo [:clojure :json ] #'write-str [:mongo :clojure] #(mongo->clojure ^DBObject % ^boolean *keywordize*) - [:mongo :json ] #(JSON/serialize %) + [:mongo :json ] #'mongo->json [:json :clojure] #(read-str % :key-fn (if *keywordize* keyword identity)) - [:json :mongo ] #(JSON/parse %)}) + [:json :mongo ] #'json->mongo}) (defn coerce "takes an object, a vector of keywords: @@ -150,8 +162,9 @@ (f obj)) (throw (RuntimeException. "unsupported keyword pair")))))) -(defn ^DBObject dbobject [& args] - "Create a DBObject from a sequence of key/value pairs, in order." +(defn ^DBObject dbobject + "Create a DBObject from a sequence of key/value pairs, in order." + [& args] (let [dbo (BasicDBObject.)] (doseq [[k v] (partition 2 args)] (.put dbo diff --git a/test/somnium/test/congomongo.clj b/test/somnium/test/congomongo.clj index 10ddee0..b2c2f06 100755 --- a/test/somnium/test/congomongo.clj +++ b/test/somnium/test/congomongo.clj @@ -246,7 +246,7 @@ uri (str "mongodb://" userpass test-db-host ":" test-db-port "/congomongotest-db-a?maxpoolsize=123&w=1&safe=true") a (make-connection uri) ^MongoClient m (:mongo a) - opts (.getMongoOptions m)] + opts (.getMongoClientOptions m)] (testing "make-connection parses options from URI" (is (= 123 (.getConnectionsPerHost opts))) (is (= WriteConcern/W1 (.getWriteConcern opts)))) @@ -285,15 +285,6 @@ (close-connection a) (is (= nil *mongo-config*))))))) -(deftest query-options - (are [x y] (= (calculate-query-options x) y) - nil 0 - [] 0 - [:tailable] 2 - [:tailable :slaveok] 6 - [:tailable :slaveok :notimeout] 22 - :notimeout 16)) - (deftest fetch-with-options (with-test-mongo (insert! :thingies {:foo 1}) @@ -501,7 +492,8 @@ (add-index! :test_col [[:key1 -1]]) ;; index3 (add-index! :test_col [:key1 [:key2 -1]]) ;; index 4 - (testing "index1" + ;; strings supplied as hints are not currently supported + #_(testing "index1" (let [plan (-> (fetch :test_col :where {:key1 1} :explain? true :hint "key1_1"))] (is (= "key1_1" (-> plan :queryPlanner :winningPlan :inputStage :indexName))))) @@ -509,7 +501,7 @@ (let [plan (-> (fetch :test_col :where {:key1 1} :explain? true :hint [:key1]))] (is (= "key1_1" (-> plan :queryPlanner :winningPlan :inputStage :indexName))))) - (testing "index2" + #_(testing "index2" (let [plan (-> (fetch :test_col :where {:key1 1} :explain? true :hint "key1_1_key2_1"))] (is (= "key1_1_key2_1" (-> plan :queryPlanner :winningPlan :inputStage :indexName))))) @@ -517,7 +509,7 @@ (let [plan (-> (fetch :test_col :where {:key1 1} :explain? true :hint [:key1 :key2]))] (is (= "key1_1_key2_1" (-> plan :queryPlanner :winningPlan :inputStage :indexName))))) - (testing "index3" + #_(testing "index3" (let [plan (-> (fetch :test_col :where {:key1 1} :explain? true :hint "key1_-1"))] (is (= "key1_-1" (-> plan :queryPlanner :winningPlan :inputStage :indexName))))) @@ -525,7 +517,7 @@ (let [plan (-> (fetch :test_col :where {:key1 1} :explain? true :hint [[:key1 -1]]))] (is (= "key1_-1" (-> plan :queryPlanner :winningPlan :inputStage :indexName))))) - (testing "index4" + #_(testing "index4" (let [plan (-> (fetch :test_col :where {:key1 1} :explain? true :hint "key1_1_key2_-1"))] (is (= "key1_1_key2_-1" (-> plan :queryPlanner :winningPlan :inputStage :indexName))))) @@ -649,6 +641,7 @@ "suffusion of yellow"))))) +;; TODO: this is failing on the json branch too (deftest test-distinct-values (with-test-mongo (insert! :distinct {:genus "Pan" :species "troglodytes" :common-name "chimpanzee"}) @@ -1097,3 +1090,30 @@ (is (= 3 (count index-info))) (is (set/subset? #{"key1_1" "key1_-1"} (set (map :name index-info)))))))) + + +(deftest test-json-serialization + (with-test-mongo + (drop-coll! :json-test) + (insert! :json-test {:fruit "bananas" :count 1}) + (let [bananas (fetch-one :json-test :as :json) + parsed (read-str bananas)] + (is (= {"fruit" "bananas" + "count" 1} + (select-keys parsed ["fruit" "count"])))) + (insert! :json-test + + (clojure.data.json/write-str + {:fruit "apples" :count 2}) + + :from :json) + (let [fruits (->> + (fetch :json-test :as :json) + (map read-str) + (map #(select-keys % ["fruit" "count"])) + (set))] + (is (= #{{"fruit" "bananas" + "count" 1} + {"fruit" "apples" + "count" 2}} + fruits)))))