backport SOLR-17343: Live-updating Global Circuit Breakers via ClusterProperties

solr/bin/solr

@@ -1886,6 +1886,28 @@ if [ "${SOLR_ENABLE_STREAM_BODY:-false}" == "true" ]; then
   SOLR_OPTS+=("-Dsolr.enableStreamBody=true")
 fi
 
+# Parse global circuit breaker env vars and convert to dot separated, lowercase properties
+if [ -n "${SOLR_CIRCUITBREAKER_UPDATE_CPU:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.update.cpu=$SOLR_CIRCUITBREAKER_UPDATE_CPU")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_UPDATE_MEM:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.update.mem=$SOLR_CIRCUITBREAKER_UPDATE_MEM")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_UPDATE_LOADAVG:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.update.loadavg=$SOLR_CIRCUITBREAKER_UPDATE_LOADAVG")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_QUERY_CPU:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.query.cpu=$SOLR_CIRCUITBREAKER_QUERY_CPU")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_QUERY_MEM:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.query.mem=$SOLR_CIRCUITBREAKER_QUERY_MEM")
+fi
+if [ -n "${SOLR_CIRCUITBREAKER_QUERY_LOADAVG:-}" ]; then
+  SOLR_OPTS+=("-Dsolr.circuitbreaker.query.loadavg=$SOLR_CIRCUITBREAKER_QUERY_LOADAVG")
+fi
+
+echo "SOLR_OPTS is now: ${SOLR_OPTS[*]}"
+
 : ${SOLR_SERVER_DIR:=$DEFAULT_SERVER_DIR}
 
 if [ ! -e "$SOLR_SERVER_DIR" ]; then

solr/core/src/java/org/apache/solr/core/GlobalCircuitBreakerConfig.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.solr.common.annotation.JsonProperty;
+import org.apache.solr.common.util.ReflectMapWriter;
+
+public class GlobalCircuitBreakerConfig implements ReflectMapWriter {
+  public static final String CIRCUIT_BREAKER_CLUSTER_PROPS_KEY = "circuit-breakers";
+  @JsonProperty public Map<String, CircuitBreakerConfig> configs = new HashMap<>();
+
+  @JsonProperty
+  public Map<String, Map<String, CircuitBreakerConfig>> hostOverrides = new HashMap<>();
+
+  public static class CircuitBreakerConfig implements ReflectMapWriter {
+    @JsonProperty public Boolean enabled = false;
+    @JsonProperty public Boolean warnOnly = false;
+    @JsonProperty public Double updateThreshold;
+    @JsonProperty public Double queryThreshold;
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(enabled, warnOnly, updateThreshold, queryThreshold);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj instanceof CircuitBreakerConfig) {
+        CircuitBreakerConfig that = (CircuitBreakerConfig) obj;
+        return that.enabled.equals(this.enabled)
+            && that.warnOnly.equals(this.warnOnly)
+            && that.updateThreshold.equals(this.updateThreshold)
+            && that.queryThreshold.equals(this.queryThreshold);
+      }
+      return false;
+    }
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o instanceof GlobalCircuitBreakerConfig) {
+      GlobalCircuitBreakerConfig that = (GlobalCircuitBreakerConfig) o;
+      return that.configs.equals(this.configs) && that.hostOverrides.equals(this.hostOverrides);
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(configs, hostOverrides);
+  }
+}

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -251,7 +251,7 @@ public class SolrCore implements SolrInfoBean, Closeable {
   private final ConfigSet configSet;
   // singleton listener for all packages used in schema
 
-  private final CircuitBreakerRegistry circuitBreakerRegistry = new CircuitBreakerRegistry();
+  private final CircuitBreakerRegistry circuitBreakerRegistry;
 
   private final List<Runnable> confListeners = new CopyOnWriteArrayList<>();
 
@@ -1073,6 +1073,7 @@ private SolrCore(
     final CountDownLatch latch = new CountDownLatch(1);
     try {
       this.coreContainer = coreContainer;
+      this.circuitBreakerRegistry = new CircuitBreakerRegistry(coreContainer);
       this.configSet = configSet;
       this.coreDescriptor = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null");
       this.name = Objects.requireNonNull(coreDescriptor.getName());
@@ -3184,6 +3185,9 @@ public <T> T initPlugins(
             type.getSimpleName() + "." + info.name, (SolrMetricProducer) o);
       }
       if (o instanceof CircuitBreaker) {
+        if (o instanceof SolrCoreAware) {
+          ((SolrCoreAware) o).inform(this);
+        }
         circuitBreakerRegistry.register((CircuitBreaker) o);
       }
       if (info.isDefault()) {

solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java

@@ -72,6 +72,7 @@
 import org.apache.solr.schema.SimilarityFactory;
 import org.apache.solr.search.QParserPlugin;
 import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.apache.solr.util.circuitbreaker.CircuitBreaker;
 import org.apache.solr.util.plugin.SolrCoreAware;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -813,7 +814,8 @@ public Path getInstancePath() {
         SolrCoreAware.class,
         new Class<?>[] {
           // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
-          // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+          // VIA RUNTIME APIS -- UNTIL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+          CircuitBreaker.class,
           CodecFactory.class,
           DirectoryFactory.class,
           ManagedIndexSchemaFactory.class,
@@ -829,7 +831,7 @@ public Path getInstancePath() {
         ResourceLoaderAware.class,
         new Class<?>[] {
           // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
-          // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+          // VIA RUNTIME APIS -- UNTIL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
           // evaluate if this must go into schemaResourceLoaderComponents
           CharFilterFactory.class,
           TokenFilterFactory.class,
@@ -919,7 +921,7 @@ private <T> T _classLookup(
   }
 
   /**
-   * Create a n instance of a class using an appropriate {@link SolrResourceLoader} depending on the
+   * Create an instance of a class using an appropriate {@link SolrResourceLoader} depending on the
    * package of that class
    *
    * @param registerCoreReloadListener register a listener for the package and reload the core if

solr/core/src/java/org/apache/solr/handler/ClusterAPI.java

@@ -28,6 +28,7 @@
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.OVERSEERSTATUS;
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.REQUESTSTATUS;
+import static org.apache.solr.core.GlobalCircuitBreakerConfig.CIRCUIT_BREAKER_CLUSTER_PROPS_KEY;
 import static org.apache.solr.core.RateLimiterConfig.RL_CONFIG_KEY;
 import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM;
 import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM;
@@ -56,6 +57,7 @@
 import org.apache.solr.common.util.ReflectMapWriter;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.GlobalCircuitBreakerConfig;
 import org.apache.solr.core.NodeRoles;
 import org.apache.solr.handler.admin.CollectionsHandler;
 import org.apache.solr.handler.admin.ConfigSetsHandler;
@@ -307,6 +309,19 @@ public void setRateLimiters(PayloadObj<RateLimiterPayload> payLoad) {
         throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error in API", e);
       }
     }
+
+    @Command(name = "set-circuit-breakers")
+    public void setCircuitBreakers(PayloadObj<GlobalCircuitBreakerConfig> payLoad) {
+      GlobalCircuitBreakerConfig circuitBreakerConfig = payLoad.get();
+      ClusterProperties clusterProperties =
+          new ClusterProperties(getCoreContainer().getZkController().getZkClient());
+
+      try {
+        clusterProperties.update(circuitBreakerConfig, CIRCUIT_BREAKER_CLUSTER_PROPS_KEY);
+      } catch (Exception e) {
+        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error in API", e);
+      }
+    }
   }
 
   public static class RoleInfo implements ReflectMapWriter {

solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java

@@ -16,9 +16,6 @@
  */
 package org.apache.solr.handler;
 
-import static org.apache.solr.common.params.CommonParams.FAILURE;
-import static org.apache.solr.common.params.CommonParams.STATUS;
-
 import java.lang.invoke.MethodHandles;
 import java.util.List;
 import org.apache.solr.client.solrj.SolrRequest.SolrRequestType;
@@ -34,6 +31,7 @@
 import org.apache.solr.update.processor.UpdateRequestProcessorChain;
 import org.apache.solr.util.circuitbreaker.CircuitBreaker;
 import org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry;
+import org.apache.solr.util.circuitbreaker.CircuitBreakerUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -134,15 +132,7 @@ protected boolean checkCircuitBreakers(SolrQueryRequest req, SolrQueryResponse r
     if (circuitBreakerRegistry.isEnabled(SolrRequestType.UPDATE)) {
       List<CircuitBreaker> trippedCircuitBreakers =
           circuitBreakerRegistry.checkTripped(SolrRequestType.UPDATE);
-      if (trippedCircuitBreakers != null) {
-        String errorMessage = CircuitBreakerRegistry.toErrorMessage(trippedCircuitBreakers);
-        rsp.add(STATUS, FAILURE);
-        rsp.setException(
-            new SolrException(
-                CircuitBreaker.getErrorCode(trippedCircuitBreakers),
-                "Circuit Breakers tripped " + errorMessage));
-        return true;
-      }
+      return CircuitBreakerUtils.reportErrorIfBreakersTripped(rsp, trippedCircuitBreakers);
     }
     return false;
   }

solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java

@@ -28,8 +28,8 @@
 import org.apache.solr.api.ApiBag;
 import org.apache.solr.api.ApiSupport;
 import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.ShardParams;
 import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ShardParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SuppressForbidden;

solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java

@@ -17,9 +17,7 @@
 package org.apache.solr.handler.component;
 
 import static org.apache.solr.common.params.CommonParams.DISTRIB;
-import static org.apache.solr.common.params.CommonParams.FAILURE;
 import static org.apache.solr.common.params.CommonParams.PATH;
-import static org.apache.solr.common.params.CommonParams.STATUS;
 
 import com.codahale.metrics.Counter;
 import java.io.PrintWriter;
@@ -81,6 +79,7 @@
 import org.apache.solr.util.SolrPluginUtils;
 import org.apache.solr.util.circuitbreaker.CircuitBreaker;
 import org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry;
+import org.apache.solr.util.circuitbreaker.CircuitBreakerUtils;
 import org.apache.solr.util.plugin.PluginInfoInitialized;
 import org.apache.solr.util.plugin.SolrCoreAware;
 import org.slf4j.Logger;
@@ -376,15 +375,7 @@ protected boolean checkCircuitBreakers(
         trippedCircuitBreakers = circuitBreakerRegistry.checkTripped(SolrRequestType.QUERY);
       }
 
-      if (trippedCircuitBreakers != null) {
-        String errorMessage = CircuitBreakerRegistry.toErrorMessage(trippedCircuitBreakers);
-        rsp.add(STATUS, FAILURE);
-        rsp.setException(
-            new SolrException(
-                CircuitBreaker.getErrorCode(trippedCircuitBreakers),
-                "Circuit Breakers tripped " + errorMessage));
-        return true;
-      }
+      return CircuitBreakerUtils.reportErrorIfBreakersTripped(rsp, trippedCircuitBreakers);
     }
     return false;
   }