Implement CBOR indexing and querying support

gradle/validation/versions-props-sorted.gradle

@@ -17,17 +17,18 @@
 
 // This ensures 'versions.props' file is sorted lexicographically.
 
+import java.util.stream.Collectors
+
 configure(rootProject) {
   task versionsPropsAreSorted() {
     doFirst {
       def versionsProps = file('versions.props')
-      def lines = versionsProps.readLines("UTF-8")
+      // remove # commented lines and blank lines
+      def lines = versionsProps.readLines("UTF-8").stream().filter(l -> !l.matches(/^(#.*|\s*)$/)).collect(Collectors.toList())
       def sorted = lines.toSorted()
 
       if (!Objects.equals(lines, sorted)) {
-        def sortedFile = file("${buildDir}/versions.props")
-        sortedFile.write(sorted.join("\n"), "UTF-8")
-        throw new GradleException("${versionsProps} file is not sorted lexicographically. I wrote a sorted file to ${sortedFile} - please review and commit.")
+        throw new GradleException("${versionsProps} file is not sorted lexicographically.")
       }
     }
   }

solr/core/build.gradle

@@ -121,6 +121,7 @@ dependencies {
   implementation 'com.fasterxml.jackson.core:jackson-core'
   implementation 'com.fasterxml.jackson.core:jackson-databind'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile'
+  implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor'
 
   implementation 'org.apache.httpcomponents:httpclient'
   implementation 'org.apache.httpcomponents:httpcore'

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -122,6 +122,7 @@
 import org.apache.solr.request.SolrRequestInfo;
 import org.apache.solr.response.BinaryResponseWriter;
 import org.apache.solr.response.CSVResponseWriter;
+import org.apache.solr.response.CborResponseWriter;
 import org.apache.solr.response.GeoJSONResponseWriter;
 import org.apache.solr.response.GraphMLResponseWriter;
 import org.apache.solr.response.JSONResponseWriter;
@@ -3051,6 +3052,7 @@ public PluginBag<QueryResponseWriter> getResponseWriters() {
     m.put("ruby", new RubyResponseWriter());
     m.put("raw", new RawResponseWriter());
     m.put(CommonParams.JAVABIN, new BinaryResponseWriter());
+    m.put("cbor", new CborResponseWriter());
     m.put("csv", new CSVResponseWriter());
     m.put("schema.xml", new SchemaXmlResponseWriter());
     m.put("smile", new SmileResponseWriter());

solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java

@@ -32,6 +32,7 @@
 import org.apache.solr.common.util.ContentStream;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.handler.loader.CSVLoader;
+import org.apache.solr.handler.loader.CborLoader;
 import org.apache.solr.handler.loader.ContentStreamLoader;
 import org.apache.solr.handler.loader.JavabinLoader;
 import org.apache.solr.handler.loader.JsonLoader;
@@ -173,6 +174,7 @@ protected Map<String, ContentStreamLoader> createDefaultLoaders(NamedList<?> arg
     registry.put("application/json", new JsonLoader().init(p));
     registry.put("application/csv", new CSVLoader().init(p));
     registry.put("application/javabin", new JavabinLoader(instance).init(p));
+    registry.put("application/cbor", CborLoader.createLoader(p));
     registry.put("text/csv", registry.get("application/csv"));
     registry.put("text/xml", registry.get("application/xml"));
     registry.put("text/json", registry.get("application/json"));
@@ -181,6 +183,7 @@ protected Map<String, ContentStreamLoader> createDefaultLoaders(NamedList<?> arg
     pathVsLoaders.put(DOC_PATH, registry.get("application/json"));
     pathVsLoaders.put(CSV_PATH, registry.get("application/csv"));
     pathVsLoaders.put(BIN_PATH, registry.get("application/javabin"));
+    pathVsLoaders.put(CBOR_PATH, registry.get("application/cbor"));
     return registry;
   }
 
@@ -211,4 +214,5 @@ public Category getCategory() {
   public static final String JSON_PATH = "/update/json";
   public static final String CSV_PATH = "/update/csv";
   public static final String BIN_PATH = "/update/bin";
+  public static final String CBOR_PATH = "/update/cbor";
 }

solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java

@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.loader;
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import com.fasterxml.jackson.dataformat.cbor.CBORGenerator;
+import com.fasterxml.jackson.dataformat.cbor.CBORParser;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+
+/**
+ * This class can load a single document or a stream of documents in CBOR format this is equivalent
+ * of loading a single json documet or an array of json documents
+ */
+public class CborLoader {
+  final CBORFactory cborFactory;
+  private final Consumer<SolrInputDocument> sink;
+
+  public CborLoader(CBORFactory cborFactory, Consumer<SolrInputDocument> sink) {
+    this.cborFactory =
+        cborFactory == null
+            ? CBORFactory.builder().enable(CBORGenerator.Feature.STRINGREF).build()
+            : cborFactory;
+    this.sink = sink;
+  }
+
+  public void stream(InputStream is) throws IOException {
+    ObjectMapper mapper = new ObjectMapper(new CBORFactory());
+    try (CBORParser parser = (CBORParser) mapper.getFactory().createParser(is)) {
+      JsonToken t;
+      while ((t = parser.nextToken()) != null) {
+
+        if (t == JsonToken.START_ARRAY) {
+          // this is an array of docs
+          t = parser.nextToken();
+          if (t == JsonToken.START_OBJECT) {
+            handleDoc(parser);
+          }
+
+        } else if (t == JsonToken.START_OBJECT) {
+          // this is just a single doc
+          handleDoc(parser);
+        }
+      }
+    }
+  }
+
+  private void handleDoc(CBORParser p) throws IOException {
+    SolrInputDocument doc = new SolrInputDocument();
+    for (; ; ) {
+      JsonToken t = p.nextToken();
+      if (t == JsonToken.END_OBJECT) {
+        if (!doc.isEmpty()) {
+          sink.accept(doc);
+        }
+        return;
+      }
+      String name;
+      if (t == JsonToken.FIELD_NAME) {
+        name = p.getCurrentName();
+        t = p.nextToken();
+        if (t == JsonToken.START_ARRAY) {
+
+          List<Object> l = new ArrayList<>();
+          for (; ; ) {
+            t = p.nextToken();
+            if (t == JsonToken.END_ARRAY) break;
+            else {
+              l.add(readVal(t, p));
+            }
+          }
+          if (!l.isEmpty()) {
+            doc.addField(name, l);
+          }
+
+        } else {
+          doc.addField(name, readVal(t, p));
+        }
+      }
+    }
+  }
+
+  private Object readVal(JsonToken t, CBORParser p) throws IOException {
+    if (t == JsonToken.VALUE_NULL) {
+      return null;
+    }
+    if (t == JsonToken.VALUE_STRING) {
+      return p.getValueAsString();
+    }
+    if (t == JsonToken.VALUE_TRUE) {
+      return Boolean.TRUE;
+    }
+    if (t == JsonToken.VALUE_FALSE) {
+      return Boolean.FALSE;
+    }
+    if (t == JsonToken.VALUE_NUMBER_INT || t == JsonToken.VALUE_NUMBER_FLOAT) {
+      return p.getNumberValue();
+    }
+    throw new RuntimeException("Unknown type :" + t);
+  }
+
+  public static ContentStreamLoader createLoader(SolrParams p) {
+    CBORFactory factory = new CBORFactory();
+    return new ContentStreamLoader() {
+      @Override
+      public void load(
+          SolrQueryRequest req,
+          SolrQueryResponse rsp,
+          ContentStream stream,
+          UpdateRequestProcessor processor)
+          throws IOException {
+        int commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1);
+        boolean overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true);
+        new CborLoader(
+                factory,
+                doc -> {
+                  AddUpdateCommand add = new AddUpdateCommand(req);
+                  add.commitWithin = commitWithin;
+                  add.solrDoc = doc;
+                  add.overwrite = overwrite;
+                  try {
+                    processor.processAdd(add);
+                  } catch (IOException e) {
+                    throw new RuntimeException(e);
+                  }
+                })
+            .stream(stream.getStream());
+      }
+    }.init(p);
+  }
+}