Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Set;
import java.util.function.Supplier;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CommonParams.DISTRIB;
import static org.apache.solr.update.PeerSync.MissedUpdatesRequest;
import static org.apache.solr.update.PeerSync.absComparator;
import static org.apache.solr.update.PeerSync.percentile;
public class PeerSyncWithLeader implements SolrMetricProducer {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private boolean debug = log.isDebugEnabled();
private String leaderUrl;
private int nUpdates;
private UpdateHandler uhandler;
private UpdateLog ulog;
private HttpSolrClient clientToLeader;
private boolean doFingerprint;
private SolrCore core;
private PeerSync.Updater updater;
private MissedUpdatesFinder missedUpdatesFinder;
private Set bufferedUpdates;
// metrics
private Timer syncTime;
private Counter syncErrors;
private Counter syncSkipped;
public PeerSyncWithLeader(SolrCore core, String leaderUrl, int nUpdates) {
this.core = core;
this.leaderUrl = leaderUrl;
this.nUpdates = nUpdates;
this.doFingerprint = !"true".equals(System.getProperty("solr.disableFingerprint"));
this.uhandler = core.getUpdateHandler();
this.ulog = uhandler.getUpdateLog();
HttpClient httpClient = core.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient();
this.clientToLeader = new HttpSolrClient.Builder(leaderUrl).withHttpClient(httpClient).build();
this.updater = new PeerSync.Updater(msg(), core);
core.getCoreMetricManager().registerMetricProducer(SolrInfoBean.Category.REPLICATION.toString(), this);
}
public static final String METRIC_SCOPE = "peerSync";
@Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) {
syncTime = manager.timer(null, registry, "time", scope, METRIC_SCOPE);
syncErrors = manager.counter(null, registry, "errors", scope, METRIC_SCOPE);
syncSkipped = manager.counter(null, registry, "skipped", scope, METRIC_SCOPE);
}
// start of peersync related debug messages. includes the core name for correlation.
private String msg() {
ZkController zkController = uhandler.core.getCoreContainer().getZkController();
String myURL = "";
if (zkController != null) {
myURL = zkController.getBaseUrl();
}
return "PeerSync: core="+uhandler.core.getName()+ " url="+myURL +" ";
}
/**
* Sync with leader
* @param startingVersions : recent versions on startup
* @return result of PeerSync with leader
*/
public PeerSync.PeerSyncResult sync(List startingVersions){
if (ulog == null) {
syncErrors.inc();
return PeerSync.PeerSyncResult.failure();
}
if (startingVersions.isEmpty()) {
log.warn("no frame of reference to tell if we've missed updates");
syncErrors.inc();
return PeerSync.PeerSyncResult.failure();
}
Timer.Context timerContext = null;
try {
if (log.isInfoEnabled()) {
log.info("{} START leader={} nUpdates={}", msg(), leaderUrl, nUpdates);
}
if (debug) {
log.debug("{} startingVersions={} {}", msg(), startingVersions.size(), startingVersions);
}
// check if we already in sync to begin with
if(doFingerprint && alreadyInSync()) {
syncSkipped.inc();
return PeerSync.PeerSyncResult.success();
}
// measure only when actual sync is performed
timerContext = syncTime.time();
List ourUpdates;
try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) {
ourUpdates = recentUpdates.getVersions(nUpdates);
bufferedUpdates = recentUpdates.getBufferUpdates();
}
ourUpdates.sort(absComparator);
startingVersions.sort(absComparator);
long ourLowThreshold = percentile(startingVersions, 0.8f);
long ourHighThreshold = percentile(startingVersions, 0.2f);
// now make sure that the starting updates overlap our updates
// there shouldn't be reorders, so any overlap will do.
long smallestNewUpdate = Math.abs(ourUpdates.get(ourUpdates.size() - 1));
if (Math.abs(startingVersions.get(0)) < smallestNewUpdate) {
log.warn("{} too many updates received since start - startingUpdates no longer overlaps with our currentUpdates", msg());
syncErrors.inc();
return PeerSync.PeerSyncResult.failure();
}
// let's merge the lists
for (Long ver : startingVersions) {
if (Math.abs(ver) < smallestNewUpdate) {
ourUpdates.add(ver);
}
}
boolean success = doSync(ourUpdates, ourLowThreshold, ourHighThreshold);
if (log.isInfoEnabled()) {
log.info("{} DONE. sync {}", msg(), (success ? "succeeded" : "failed"));
}
if (!success) {
syncErrors.inc();
}
return success ? PeerSync.PeerSyncResult.success() : PeerSync.PeerSyncResult.failure();
} finally {
if (timerContext != null) {
timerContext.close();
}
try {
clientToLeader.close();
} catch (IOException e) {
log.warn("{} unable to close client to leader", msg(), e);
}
}
}
private boolean doSync(List ourUpdates, long ourLowThreshold, long ourHighThreshold) {
// get leader's recent versions and fingerprint
// note: by getting leader's versions later, we guarantee that leader's versions always super set of {@link bufferedUpdates}
NamedList