Skip to content

M
midjourney-proxy
Commit 82a53d5a by 胡青青
Options

feat:[none]添加热点key采集日志，增长快类型key日志

parent 5b201992
Showing with 139 additions and 362 deletions
pcloud-common-core/src/main/java/com/pcloud/common/core/aspect/JedisClusterAspect.java
package com.pcloud.common.core.aspect;
import com.pcloud.common.utils.DateUtils;
import com.pcloud.common.utils.string.StringUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomUtils;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
......@@ -16,7 +16,6 @@ import java.io.*;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.LongAdder;
/**
* 添加监控，查看
......@@ -26,63 +25,12 @@ import java.util.concurrent.atomic.LongAdder;
@Slf4j
public class JedisClusterAspect {
//需要统计的方法调用次数
private LongAdder allGetLongAdder = new LongAdder();
private LongAdder allRpopLongAdder = new LongAdder();
private LongAdder allHgetAllLongAdder = new LongAdder();
private LongAdder missGetLongAdder = new LongAdder();
private LongAdder missRpopLongAdder = new LongAdder();
private LongAdder missHgetAllLongAdder = new LongAdder();
private ConcurrentHashMap<String, AtomicInteger> missKeyMap = new ConcurrentHashMap<>();
private ConcurrentHashMap<String, AtomicInteger> tempMap = new ConcurrentHashMap<>();
private LongAdder getSetLongAdder = new LongAdder();
private LongAdder missGetSetLongAdder = new LongAdder();
private LongAdder getSetRLongAdder = new LongAdder();
private LongAdder missgetSetRLongAdder = new LongAdder();
private LongAdder setnxLongAdder = new LongAdder();
private LongAdder missSetnxLongAdder = new LongAdder();
private LongAdder hmgetLongAdder = new LongAdder();
private LongAdder misshmgetLongAdder = new LongAdder();
private LongAdder hgetLongAdder = new LongAdder();
private LongAdder misshgetLongAdder = new LongAdder();
private LongAdder hlenLongAdder = new LongAdder();
private LongAdder misshlenLongAdder = new LongAdder();
private LongAdder hkeysLongAdder = new LongAdder();
private LongAdder misshkeysLongAdder = new LongAdder();
private LongAdder lpopLongAdder = new LongAdder();
private LongAdder misslpopLongAdder = new LongAdder();
private LongAdder zcardLongAdder = new LongAdder();
private LongAdder misszcardLongAdder = new LongAdder();
private LongAdder zrevrangeLongAdder = new LongAdder();
private LongAdder misszrevrangeLongAdder = new LongAdder();
private LongAdder ttlLongAdder = new LongAdder();
private LongAdder misshttlLongAdder = new LongAdder();
private LongAdder zrankLongAdder = new LongAdder();
private LongAdder misszrankLongAdder = new LongAdder();
private LongAdder lrangeLongAdder = new LongAdder();
private LongAdder misslrangeLongAdder = new LongAdder();
private LongAdder llenLongAdder = new LongAdder();
private LongAdder missllenLongAdder = new LongAdder();
private Map<String, ConcurrentHashMap<String, AtomicInteger>> solfDistributedMap = new ConcurrentHashMap<>();
private Map<String, ConcurrentHashMap<String, AtomicInteger>> tempMap = new ConcurrentHashMap<>();
private LFU<String, String> cache = new LFU<>(10000);
public JedisClusterAspect() {
solfDistributedMap.put("firstNode", new ConcurrentHashMap<String, AtomicInteger>());
solfDistributedMap.put("secondNode", new ConcurrentHashMap<String, AtomicInteger>());
solfDistributedMap.put("thirdNode", new ConcurrentHashMap<String, AtomicInteger>());
solfDistributedMap.put("fourthNode", new ConcurrentHashMap<String, AtomicInteger>());
//初始化redis操作
new PringLogThread().start();
}
......@@ -102,58 +50,23 @@ public class JedisClusterAspect {
while (true) {
BufferedWriter out = null;
try {
Thread.sleep( 5*60 * 1000);
tempMap.clear();
tempMap.putAll(missKeyMap);
missKeyMap.clear();
StringBuffer logStr = new StringBuffer();
logStr.append("get调用：" + allGetLongAdder.intValue() + "次，未命中：" + missGetLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("rpop调用：" + allRpopLongAdder.intValue() + "次，未命中：" + missRpopLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hgetall调用：" + allHgetAllLongAdder.intValue() + "次，未命中：" + missHgetAllLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("getSet调用：" + getSetLongAdder.intValue() + "次，未命中：" + missGetSetLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hget调用：" + hgetLongAdder.intValue() + "次，未命中：" + misshgetLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hlen调用：" + hlenLongAdder.intValue() + "次，未命中：" + misshlenLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("lrange调用：" + lrangeLongAdder.intValue() + "次，未命中：" + misslrangeLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("llen调用：" + llenLongAdder.intValue() + "次，未命中：" + missllenLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hkeys调用：" + hkeysLongAdder.intValue() + "次，未命中：" + misshkeysLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("lpop调用：" + lpopLongAdder.intValue() + "次，未命中：" + misslpopLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("zcard调用：" + zcardLongAdder.intValue() + "次，未命中：" + misszcardLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("zrevrange调用：" + zrevrangeLongAdder.intValue() + "次，未命中：" + misszrevrangeLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("ttl调用：" + ttlLongAdder.intValue() + "次，未命中：" + misshttlLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("zrank调用：" + zrankLongAdder.intValue() + "次，未命中：" + misszrankLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("getSetR调用：" + getSetRLongAdder.intValue() + "次，未命中：" + missgetSetRLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("setnx调用：" + setnxLongAdder.intValue() + "次，未命中：" + missSetnxLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hmget调用：" + hmgetLongAdder.intValue() + "次，未命中：" + misshmgetLongAdder.intValue() + "次");
logStr.append("\n");
Thread.sleep(60 * 1000);
tempMap = solfDistributedMap;
solfDistributedMap.clear();
String fileName = "hotkey" + DateUtils.getYmdHmsTime() + ".log";
out = new BufferedWriter(new FileWriter(path + fileName));
for (Map.Entry<String, ConcurrentHashMap<String, AtomicInteger>> entry : solfDistributedMap.entrySet()) {
for (Map.Entry<String, AtomicInteger> childEntry : entry.getValue().entrySet()) {
logStr.append(entry.getKey() + "节点上 key：" + childEntry.getKey());
logStr.append("，调用次数：" + childEntry.getValue());
logStr.append("\n");
out.write("key前缀："+entry.getKey() + "，节点：" + childEntry.getKey());
out.write("，调用次数：" + childEntry.getValue());
out.newLine();
}
}
String fileName = "redisOperate" + DateUtils.getYmdHmsTime() + ".log";
out = new BufferedWriter(new FileWriter(path + fileName));
out.write(logStr.toString());
out.newLine();
for (Map.Entry<String, AtomicInteger> entry : tempMap.entrySet()) {
out.write(entry.getKey() + "未命中：" + entry.getValue().intValue());
tempMap.clear();
Map<String, LFU<String, String>.HitRate> hotMap = cache.getHotKeyMap();
for (Map.Entry<String, LFU<String, String>.HitRate> entry : hotMap.entrySet()) {
LFU<String, String>.HitRate hitRate = entry.getValue();
out.write("key：" + hitRate.getKey() + "，槽位：" + hitRate.getV() + "，次数：" + hitRate.getHitCount());
out.newLine();
}
} catch (Exception e) {
......@@ -173,147 +86,31 @@ public class JedisClusterAspect {
String methodName = joinPoint.getSignature().getName();
Object[] obj = joinPoint.getArgs();
String key = obj[0].toString();
switch (methodName) {
case "get":
allGetLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
missGetLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "rpop":
allRpopLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
missRpopLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "hgetAll":
allHgetAllLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
missHgetAllLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "getSet":
getSetLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
missGetSetLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "getSetR":
getSetRLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
missgetSetRLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "setnx":
setnxLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
missSetnxLongAdder.add(1);
addMissCount(missKeyMap, key);
cache.put(key, getNodeName(key));
int randomInt = RandomUtils.nextInt(1,101);
//100里面1个数，小于等于10的概率就是10%
if(randomInt<10){
log.warn("采样hotkey methodName {} key {}",methodName,key);
}
break;
case "hmget":
hmgetLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
misshmgetLongAdder.add(1);
addMissCount(missKeyMap, key);
if(key.startsWith("readercenter")){
log.warn("统计增长比较快的key methodName {} key {}",methodName,key);
}
break;
case "hget":
hgetLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
misshgetLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "hlen":
hlenLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
misshlenLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "lrange":
lrangeLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
misslrangeLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "llen":
llenLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
missllenLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "hkeys":
hkeysLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
misshkeysLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "lpop":
lpopLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
misslpopLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "zcard":
zcardLongAdder.add(1);
setSoltDistributed(key);
if (result == null || StringUtil.isEmpty(result.toString())) {
misszcardLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "zrevrange":
zrevrangeLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misszrevrangeLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "ttl":
setSoltDistributed(key);
ttlLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misshttlLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "zrank":
setSoltDistributed(key);
zrankLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misszrankLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
default:
private String getNodeName(String key){
int slot = JedisClusterCRC16.getSlot(SafeEncoder.encode(key));
String nodeName = "";
if (slot >= 0 && slot <= 4095) {
nodeName = "firstNode";
} else if (slot >= 4096 && slot <= 8191) {
nodeName = "secondNode";
} else if (slot >= 8192 && slot <= 12287) {
nodeName = "thirdNode";
} else if (slot > 12288 && slot <= 16383) {
nodeName = "fourthNode";
}
return nodeName;
}
/**
* 计算key的分布
*
......@@ -325,25 +122,19 @@ public class JedisClusterAspect {
return;
}
String keyPrefix = key.substring(0, key.lastIndexOf(":"));
int slot = JedisClusterCRC16.getSlot(SafeEncoder.encode(key));
String mapkey = "";
if (slot >= 0 && slot <= 4095) {
mapkey = "firstNode";
} else if (slot >= 4096 && slot <= 8191) {
mapkey = "secondNode";
} else if (slot >= 8192 && slot <= 12287) {
mapkey = "thirdNode";
} else if (slot > 12288 && slot <= 16383) {
mapkey = "fourthNode";
}
ConcurrentHashMap<String, AtomicInteger> map = solfDistributedMap.get(mapkey);
AtomicInteger missCount = map.get(keyPrefix);
if (missCount == null) {
missCount = new AtomicInteger(1);
String nodeName = getNodeName(key);
ConcurrentHashMap<String, AtomicInteger> map = solfDistributedMap.get(keyPrefix);
if (map == null) {
map = new ConcurrentHashMap<String, AtomicInteger>();
solfDistributedMap.put(keyPrefix,map);
}
AtomicInteger count = map.get(nodeName);
if (count == null) {
count = new AtomicInteger(1);
} else {
missCount.getAndIncrement();
count.getAndIncrement();
}
map.put(keyPrefix, missCount);
map.put(nodeName, count);
}
private static String getResourceBasePath() {
......
pcloud-common-core/src/main/java/com/pcloud/common/core/aspect/LFU.java 0 → 100644
package com.pcloud.common.core.aspect;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class LFU<K, V> {
private final int capcity;
private Map<K, V> cache = new ConcurrentHashMap<>();
private Map<K, HitRate> count = new ConcurrentHashMap<>();
public LFU(int capcity) {
this.capcity = capcity;
}
public Map<K, HitRate> getHotKeyMap() {
Map<K, HitRate> count = this.count;
return count;
}
public void put(K key, V value) {
V v = cache.get(key);
if (v == null) {
if (cache.size() == capcity) {
removeElement();
}
count.put(key, new HitRate(key, 1, System.nanoTime(), value));
} else {
addHitCount(key);
}
cache.put(key, value);
}
public V get(K key) {
V value = cache.get(key);
if (value != null) {
addHitCount(key);
return value;
}
return null;
}
//移除元素
private void removeElement() {
HitRate hr = Collections.min(count.values());
cache.remove(hr.key);
count.remove(hr.key);
}
//更新访问元素状态
private void addHitCount(K key) {
HitRate hitRate = count.get(key);
hitRate.hitCount = hitRate.hitCount + 1;
hitRate.lastTime = System.nanoTime();
}
//内部类
class HitRate implements Comparable<HitRate> {
private K key;
private V v;
private int hitCount;
private long lastTime;
private HitRate(K key, int hitCount, long lastTime, V v) {
this.key = key;
this.hitCount = hitCount;
this.lastTime = lastTime;
this.v = v;
}
@Override
public int compareTo(HitRate o) {
int compare = Integer.compare(this.hitCount, o.hitCount);
return compare == 0 ? Long.compare(this.lastTime, o.lastTime) : compare;
}
public K getKey() {
return key;
}
public V getV() {
return v;
}
public int getHitCount() {
return hitCount;
}
}
}
pcloud-common-core/src/main/java/com/pcloud/common/core/aspect/ParamLogAspect.java deleted 100644 → 0
/**
*
*/
package com.pcloud.common.core.aspect;
import java.lang.reflect.Method;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.Signature;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Configuration;
import com.alibaba.fastjson.JSON;
import com.pcloud.common.utils.string.StringUtil;
/**
* @author：songx
* @date：2018年4月26日,下午2:32:36
*/
@Aspect
@Configuration
public class ParamLogAspect {
/**
*
*/
private final static Logger LOGGER = LoggerFactory.getLogger("");
@Pointcut("execution(* com.pcloud..*(..))")
public void bizPoint() {
}
/**
* 方法执行前以后执行
*
* @param joinPoint
*/
@Before("bizPoint()")
public void doBefore(JoinPoint joinPoint) {
ParamLog paramLog = checkAnnotation(joinPoint);
if (paramLog == null) {
return;
}
Signature signature = joinPoint.getSignature();
String methodName = signature.getName();
String description = paramLog.description();
description = StringUtil.isEmpty(description) ? paramLog.value() : description;
StringBuffer msg = new StringBuffer(signature.getDeclaringTypeName()).append(" 【" + methodName + " before】");
if (!StringUtil.isEmpty(description)) {
msg.append(description);
}
if (!paramLog.isBefore()) {
LOGGER.info(msg.toString());
} else {
String argsJson = JSON.toJSONString(joinPoint.getArgs());
LOGGER.info(msg.append(",[Args]=").append(argsJson).toString());
}
}
/**
* 方法执行完以后执行
*
* @param joinPoint
* @param result
*/
@AfterReturning(pointcut = "bizPoint()", returning = "result")
public void doAfterReturn(JoinPoint joinPoint, Object result) {
ParamLog paramLog = checkAnnotation(joinPoint);
if (paramLog == null) {
return;
}
Signature signature = joinPoint.getSignature();
String methodName = signature.getName();
String description = paramLog.description();
description = StringUtil.isEmpty(description) ? paramLog.value() : description;
StringBuffer msg = new StringBuffer(signature.getDeclaringTypeName())
.append(" 【" + methodName + " afterReturn】");
if (!StringUtil.isEmpty(description)) {
msg.append(description);
}
if (!paramLog.isAfterReturn()) {
LOGGER.info(msg.toString());
} else {
String resultJson = JSON.toJSONString(result);
LOGGER.info(msg.append(",[result]=").append(resultJson).toString());
}
}
private static ParamLog checkAnnotation(JoinPoint joinPoint) {
// 获取方法签名
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
// java reflect相关类，通过反射得到注解
Method method = signature.getMethod();
if (!method.isAnnotationPresent(ParamLog.class)) {
return null;
}
return method.getAnnotation(ParamLog.class);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment