Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.rocketmq.broker.offset.ConsumerOffsetManager Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.offset;
import com.google.common.base.Strings;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.BrokerPathConfigHelper;
import org.apache.rocketmq.common.ConfigManager;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.protocol.DataVersion;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class ConsumerOffsetManager extends ConfigManager {
private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
public static final String TOPIC_GROUP_SEPARATOR = "@";
private DataVersion dataVersion = new DataVersion();
private ConcurrentMap> offsetTable =
new ConcurrentHashMap<>(512);
private final ConcurrentMap> resetOffsetTable =
new ConcurrentHashMap<>(512);
private final ConcurrentMap> pullOffsetTable =
new ConcurrentHashMap<>(512);
protected transient BrokerController brokerController;
private final transient AtomicLong versionChangeCounter = new AtomicLong(0);
public ConsumerOffsetManager() {
}
public ConsumerOffsetManager(BrokerController brokerController) {
this.brokerController = brokerController;
}
public void cleanOffset(String group) {
Iterator>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
Entry> next = it.next();
String topicAtGroup = next.getKey();
if (topicAtGroup.contains(group)) {
String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
if (arrays.length == 2 && group.equals(arrays[1])) {
it.remove();
LOG.warn("Clean group's offset, {}, {}", topicAtGroup, next.getValue());
}
}
}
}
public void cleanOffsetByTopic(String topic) {
Iterator>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
Entry> next = it.next();
String topicAtGroup = next.getKey();
if (topicAtGroup.contains(topic)) {
String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
if (arrays.length == 2 && topic.equals(arrays[0])) {
it.remove();
LOG.warn("Clean topic's offset, {}, {}", topicAtGroup, next.getValue());
}
}
}
}
public void scanUnsubscribedTopic() {
Iterator>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
Entry> next = it.next();
String topicAtGroup = next.getKey();
String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
if (arrays.length == 2) {
String topic = arrays[0];
String group = arrays[1];
if (null == brokerController.getConsumerManager().findSubscriptionData(group, topic)
&& this.offsetBehindMuchThanData(topic, next.getValue())) {
it.remove();
LOG.warn("remove topic offset, {}", topicAtGroup);
}
}
}
}
private boolean offsetBehindMuchThanData(final String topic, ConcurrentMap table) {
Iterator> it = table.entrySet().iterator();
boolean result = !table.isEmpty();
while (it.hasNext() && result) {
Entry next = it.next();
long minOffsetInStore = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, next.getKey());
long offsetInPersist = next.getValue();
result = offsetInPersist <= minOffsetInStore;
}
return result;
}
public Set whichTopicByConsumer(final String group) {
Set topics = new HashSet<>();
Iterator>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
Entry> next = it.next();
String topicAtGroup = next.getKey();
String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
if (arrays.length == 2) {
if (group.equals(arrays[1])) {
topics.add(arrays[0]);
}
}
}
return topics;
}
public Set whichGroupByTopic(final String topic) {
Set groups = new HashSet<>();
Iterator>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
Entry> next = it.next();
String topicAtGroup = next.getKey();
String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
if (arrays.length == 2) {
if (topic.equals(arrays[0])) {
groups.add(arrays[1]);
}
}
}
return groups;
}
public Map> getGroupTopicMap() {
Map> retMap = new HashMap<>(128);
for (String key : this.offsetTable.keySet()) {
String[] arr = key.split(TOPIC_GROUP_SEPARATOR);
if (arr.length == 2) {
String topic = arr[0];
String group = arr[1];
Set topics = retMap.get(group);
if (topics == null) {
topics = new HashSet<>(8);
retMap.put(group, topics);
}
topics.add(topic);
}
}
return retMap;
}
public void commitOffset(final String clientHost, final String group, final String topic, final int queueId,
final long offset) {
// topic@group
String key = topic + TOPIC_GROUP_SEPARATOR + group;
this.commitOffset(clientHost, key, queueId, offset);
}
private void commitOffset(final String clientHost, final String key, final int queueId, final long offset) {
ConcurrentMap map = this.offsetTable.get(key);
if (null == map) {
map = new ConcurrentHashMap<>(32);
map.put(queueId, offset);
this.offsetTable.put(key, map);
} else {
Long storeOffset = map.put(queueId, offset);
if (storeOffset != null && offset < storeOffset) {
LOG.warn("[NOTIFYME]update consumer offset less than store. clientHost={}, key={}, queueId={}, requestOffset={}, storeOffset={}", clientHost, key, queueId, offset, storeOffset);
}
}
if (versionChangeCounter.incrementAndGet() % brokerController.getBrokerConfig().getConsumerOffsetUpdateVersionStep() == 0) {
long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
dataVersion.nextVersion(stateMachineVersion);
}
}
public void commitPullOffset(final String clientHost, final String group, final String topic, final int queueId,
final long offset) {
// topic@group
String key = topic + TOPIC_GROUP_SEPARATOR + group;
ConcurrentMap map = this.pullOffsetTable.computeIfAbsent(
key, k -> new ConcurrentHashMap<>(32));
map.put(queueId, offset);
}
/**
* If the target queue has temporary reset offset, return the reset-offset.
* Otherwise, return the current consume offset in the offset store.
* @param group Consumer group
* @param topic Topic
* @param queueId Queue ID
* @return current consume offset or reset offset if there were one.
*/
public long queryOffset(final String group, final String topic, final int queueId) {
// topic@group
String key = topic + TOPIC_GROUP_SEPARATOR + group;
if (this.brokerController.getBrokerConfig().isUseServerSideResetOffset()) {
Map reset = resetOffsetTable.get(key);
if (null != reset && reset.containsKey(queueId)) {
return reset.get(queueId);
}
}
ConcurrentMap map = this.offsetTable.get(key);
if (null != map) {
Long offset = map.get(queueId);
if (offset != null) {
return offset;
}
}
return -1L;
}
/**
* Query pull offset in pullOffsetTable
* @param group Consumer group
* @param topic Topic
* @param queueId Queue ID
* @return latest pull offset of consumer group
*/
public long queryPullOffset(final String group, final String topic, final int queueId) {
// topic@group
String key = topic + TOPIC_GROUP_SEPARATOR + group;
Long offset = null;
ConcurrentMap map = this.pullOffsetTable.get(key);
if (null != map) {
offset = map.get(queueId);
}
if (offset == null) {
offset = queryOffset(group, topic, queueId);
}
return offset;
}
@Override
public String encode() {
return this.encode(false);
}
@Override
public String configFilePath() {
return BrokerPathConfigHelper.getConsumerOffsetPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
}
@Override
public void decode(String jsonString) {
if (jsonString != null) {
ConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class);
if (obj != null) {
this.setOffsetTable(obj.getOffsetTable());
this.dataVersion = obj.dataVersion;
}
}
}
@Override
public String encode(final boolean prettyFormat) {
return RemotingSerializable.toJson(this, prettyFormat);
}
public ConcurrentMap> getOffsetTable() {
return offsetTable;
}
public void setOffsetTable(ConcurrentMap> offsetTable) {
this.offsetTable = offsetTable;
}
public Map queryMinOffsetInAllGroup(final String topic, final String filterGroups) {
Map queueMinOffset = new HashMap<>();
Set topicGroups = this.offsetTable.keySet();
if (!UtilAll.isBlank(filterGroups)) {
for (String group : filterGroups.split(",")) {
Iterator it = topicGroups.iterator();
while (it.hasNext()) {
if (group.equals(it.next().split(TOPIC_GROUP_SEPARATOR)[1])) {
it.remove();
}
}
}
}
for (Map.Entry> offSetEntry : this.offsetTable.entrySet()) {
String topicGroup = offSetEntry.getKey();
String[] topicGroupArr = topicGroup.split(TOPIC_GROUP_SEPARATOR);
if (topic.equals(topicGroupArr[0])) {
for (Entry entry : offSetEntry.getValue().entrySet()) {
long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, entry.getKey());
if (entry.getValue() >= minOffset) {
Long offset = queueMinOffset.get(entry.getKey());
if (offset == null) {
queueMinOffset.put(entry.getKey(), Math.min(Long.MAX_VALUE, entry.getValue()));
} else {
queueMinOffset.put(entry.getKey(), Math.min(entry.getValue(), offset));
}
}
}
}
}
return queueMinOffset;
}
public Map queryOffset(final String group, final String topic) {
// topic@group
String key = topic + TOPIC_GROUP_SEPARATOR + group;
return this.offsetTable.get(key);
}
public void cloneOffset(final String srcGroup, final String destGroup, final String topic) {
ConcurrentMap offsets = this.offsetTable.get(topic + TOPIC_GROUP_SEPARATOR + srcGroup);
if (offsets != null) {
this.offsetTable.put(topic + TOPIC_GROUP_SEPARATOR + destGroup, new ConcurrentHashMap<>(offsets));
}
}
public DataVersion getDataVersion() {
return dataVersion;
}
public void setDataVersion(DataVersion dataVersion) {
this.dataVersion = dataVersion;
}
public void removeOffset(final String group) {
Iterator>> it = this.offsetTable.entrySet().iterator();
while (it.hasNext()) {
Entry> next = it.next();
String topicAtGroup = next.getKey();
if (topicAtGroup.contains(group)) {
String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
if (arrays.length == 2 && group.equals(arrays[1])) {
it.remove();
LOG.warn("clean group offset {}", topicAtGroup);
}
}
}
}
public void assignResetOffset(String topic, String group, int queueId, long offset) {
if (Strings.isNullOrEmpty(topic) || Strings.isNullOrEmpty(group) || queueId < 0 || offset < 0) {
LOG.warn("Illegal arguments when assigning reset offset. Topic={}, group={}, queueId={}, offset={}",
topic, group, queueId, offset);
return;
}
String key = topic + TOPIC_GROUP_SEPARATOR + group;
ConcurrentMap map = resetOffsetTable.get(key);
if (null == map) {
map = new ConcurrentHashMap();
ConcurrentMap previous = resetOffsetTable.putIfAbsent(key, map);
if (null != previous) {
map = previous;
}
}
map.put(queueId, offset);
LOG.debug("Reset offset OK. Topic={}, group={}, queueId={}, resetOffset={}",
topic, group, queueId, offset);
// Two things are important here:
// 1, currentOffsetMap might be null if there is no previous records;
// 2, Our overriding here may get overridden by the client instantly in concurrent cases; But it still makes
// sense in cases like clients are offline.
ConcurrentMap currentOffsetMap = offsetTable.get(key);
if (null != currentOffsetMap) {
currentOffsetMap.put(queueId, offset);
}
}
public boolean hasOffsetReset(String topic, String group, int queueId) {
String key = topic + TOPIC_GROUP_SEPARATOR + group;
ConcurrentMap map = resetOffsetTable.get(key);
if (null == map) {
return false;
}
return map.containsKey(queueId);
}
public Long queryThenEraseResetOffset(String topic, String group, Integer queueId) {
String key = topic + TOPIC_GROUP_SEPARATOR + group;
ConcurrentMap map = resetOffsetTable.get(key);
if (null == map) {
return null;
} else {
return map.remove(queueId);
}
}
}
