Skip to content

M
midjourney-proxy
Commit bb6ca641 by 田超
Options

Merge branch 'featur/redisCacheHitRate' into 'master' 

feat:[none]添加热点key采集日志，增长快类型key日志，修改日志刷新频率

See merge request rays/pcloud-common-parent!148
parents c8f7949d 75ba6ebe
Showing with 162 additions and 322 deletions
pcloud-common-core/src/main/java/com/pcloud/common/core/aspect/JedisClusterAspect.java
package com.pcloud.common.core.aspect; package com.pcloud.common.core.aspect;
import com.pcloud.common.utils.DateUtils; import com.pcloud.common.utils.DateUtils;
import com.pcloud.common.utils.string.StringUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomUtils;
import org.aspectj.lang.JoinPoint; import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.AfterReturning; import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut; import org.aspectj.lang.annotation.Pointcut;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.ResourceUtils; import org.springframework.util.ResourceUtils;
import redis.clients.util.JedisClusterCRC16;
import redis.clients.util.SafeEncoder;
import java.io.*; import java.io.*;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.LongAdder;
/** /**
* 添加监控，查看 * 添加监控，查看
...@@ -24,57 +25,12 @@ import java.util.concurrent.atomic.LongAdder; ...@@ -24,57 +25,12 @@ import java.util.concurrent.atomic.LongAdder;
@Slf4j @Slf4j
public class JedisClusterAspect { public class JedisClusterAspect {
//需要统计的方法调用次数 //需要统计的方法调用次数
private LongAdder allGetLongAdder = new LongAdder(); private Map<String, ConcurrentHashMap<String, AtomicInteger>> solfDistributedMap = new ConcurrentHashMap<>();
private LongAdder allRpopLongAdder = new LongAdder(); private Map<String, ConcurrentHashMap<String, AtomicInteger>> tempMap = new ConcurrentHashMap<>();
private LongAdder allHgetAllLongAdder = new LongAdder(); private LFU<String, String> cache = new LFU<>(10000);
private LongAdder missGetLongAdder = new LongAdder();
private LongAdder missRpopLongAdder = new LongAdder();
private LongAdder missHgetAllLongAdder = new LongAdder();
private ConcurrentHashMap<String, AtomicInteger> missKeyMap = new ConcurrentHashMap<>();
private ConcurrentHashMap<String, AtomicInteger> tempMap = new ConcurrentHashMap<>();
private LongAdder getSetLongAdder = new LongAdder();
private LongAdder missGetSetLongAdder = new LongAdder();
private LongAdder getSetRLongAdder = new LongAdder();
private LongAdder missgetSetRLongAdder = new LongAdder();
private LongAdder setnxLongAdder = new LongAdder();
private LongAdder missSetnxLongAdder = new LongAdder();
private LongAdder hmgetLongAdder = new LongAdder();
private LongAdder misshmgetLongAdder = new LongAdder();
private LongAdder hgetLongAdder = new LongAdder();
private LongAdder misshgetLongAdder = new LongAdder();
private LongAdder hlenLongAdder = new LongAdder();
private LongAdder misshlenLongAdder = new LongAdder();
private LongAdder hkeysLongAdder = new LongAdder();
private LongAdder misshkeysLongAdder = new LongAdder();
private LongAdder lpopLongAdder = new LongAdder();
private LongAdder misslpopLongAdder = new LongAdder();
private LongAdder zcardLongAdder = new LongAdder();
private LongAdder misszcardLongAdder = new LongAdder();
private LongAdder zrevrangeLongAdder = new LongAdder();
private LongAdder misszrevrangeLongAdder = new LongAdder();
private LongAdder ttlLongAdder = new LongAdder();
private LongAdder misshttlLongAdder = new LongAdder();
private LongAdder zrankLongAdder = new LongAdder();
private LongAdder misszrankLongAdder = new LongAdder();
private LongAdder lrangeLongAdder = new LongAdder();
private LongAdder misslrangeLongAdder = new LongAdder();
private LongAdder llenLongAdder = new LongAdder();
private LongAdder missllenLongAdder = new LongAdder();
public JedisClusterAspect() { public JedisClusterAspect() {
//初始化redis操作
new PringLogThread().start(); new PringLogThread().start();
} }
...@@ -94,51 +50,23 @@ public class JedisClusterAspect { ...@@ -94,51 +50,23 @@ public class JedisClusterAspect {
while (true) { while (true) {
BufferedWriter out = null; BufferedWriter out = null;
try { try {
Thread.sleep(60 * 60 * 1000); Thread.sleep(60*60 * 1000);
tempMap.clear(); tempMap = solfDistributedMap;
tempMap.putAll(missKeyMap); solfDistributedMap.clear();
missKeyMap.clear(); String fileName = "hotkey" + DateUtils.getYmdHmsTime() + ".log";
StringBuffer logStr = new StringBuffer("\n");
logStr.append("get调用：" + allGetLongAdder.intValue() + "次，未命中：" + missGetLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("rpop调用：" + allRpopLongAdder.intValue() + "次，未命中：" + missRpopLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hgetall调用：" + allHgetAllLongAdder.intValue() + "次，未命中：" + missHgetAllLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("getSet调用：" + getSetLongAdder.intValue() + "次，未命中：" + missGetSetLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hget调用：" + hgetLongAdder.intValue() + "次，未命中：" + misshgetLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hlen调用：" + hlenLongAdder.intValue() + "次，未命中：" + misshlenLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("lrange调用：" + lrangeLongAdder.intValue() + "次，未命中：" + misslrangeLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("llen调用：" + llenLongAdder.intValue() + "次，未命中：" + missllenLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hkeys调用：" + hkeysLongAdder.intValue() + "次，未命中：" + misshkeysLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("lpop调用：" + lpopLongAdder.intValue() + "次，未命中：" + misslpopLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("zcard调用：" + zcardLongAdder.intValue() + "次，未命中：" + misszcardLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("zrevrange调用：" + zrevrangeLongAdder.intValue() + "次，未命中：" + misszrevrangeLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("ttl调用：" + ttlLongAdder.intValue() + "次，未命中：" + misshttlLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("zrank调用：" + zrankLongAdder.intValue() + "次，未命中：" + misszrankLongAdder.intValue() + "次");
logStr.append("getSetR调用：" + getSetRLongAdder.intValue() + "次，未命中：" + missgetSetRLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("setnx调用：" + setnxLongAdder.intValue() + "次，未命中：" + missSetnxLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("hmget调用：" + hmgetLongAdder.intValue() + "次，未命中：" + misshmgetLongAdder.intValue() + "次");
logStr.append("\n");
logStr.append("\n");
String fileName = "redisOperate" + DateUtils.getYmdHmsTime() + ".log";
out = new BufferedWriter(new FileWriter(path + fileName)); out = new BufferedWriter(new FileWriter(path + fileName));
out.write(logStr.toString()); for (Map.Entry<String, ConcurrentHashMap<String, AtomicInteger>> entry : solfDistributedMap.entrySet()) {
for (Map.Entry<String, AtomicInteger> childEntry : entry.getValue().entrySet()) {
out.write("key前缀："+entry.getKey() + "，节点：" + childEntry.getKey());
out.write("，调用次数：" + childEntry.getValue());
out.newLine(); out.newLine();
for (Map.Entry<String, AtomicInteger> entry : tempMap.entrySet()) { }
out.write(entry.getKey() + "未命中：" + entry.getValue().intValue()); }
tempMap.clear();
Map<String, LFU<String, String>.HitRate> hotMap = cache.getHotKeyMap();
for (Map.Entry<String, LFU<String, String>.HitRate> entry : hotMap.entrySet()) {
LFU<String, String>.HitRate hitRate = entry.getValue();
out.write("key：" + hitRate.getKey() + "，槽位：" + hitRate.getV() + "，次数：" + hitRate.getHitCount());
out.newLine(); out.newLine();
} }
} catch (Exception e) { } catch (Exception e) {
...@@ -158,129 +86,55 @@ public class JedisClusterAspect { ...@@ -158,129 +86,55 @@ public class JedisClusterAspect {
String methodName = joinPoint.getSignature().getName(); String methodName = joinPoint.getSignature().getName();
Object[] obj = joinPoint.getArgs(); Object[] obj = joinPoint.getArgs();
String key = obj[0].toString(); String key = obj[0].toString();
switch (methodName) { setSoltDistributed(key);
case "get": cache.put(key, getNodeName(key));
allGetLongAdder.add(1); int randomInt = RandomUtils.nextInt(1,101);
if (result == null || StringUtil.isEmpty(result.toString())) { //100里面1个数，小于等于10的概率就是10%
missGetLongAdder.add(1); if(randomInt<10){
addMissCount(missKeyMap, key); log.warn("采样hotkey methodName {} key {}",methodName,key);
} }
break; if(key.startsWith("readercenter")){
case "rpop": log.warn("统计增长比较快的key methodName {} key {}",methodName,key);
allRpopLongAdder.add(1); }
if (result == null || StringUtil.isEmpty(result.toString())) { }
missRpopLongAdder.add(1); private String getNodeName(String key){
addMissCount(missKeyMap, key); int slot = JedisClusterCRC16.getSlot(SafeEncoder.encode(key));
} String nodeName = "";
break; if (slot >= 0 && slot <= 4095) {
case "hgetAll": nodeName = "firstNode";
allHgetAllLongAdder.add(1); } else if (slot >= 4096 && slot <= 8191) {
if (result == null || StringUtil.isEmpty(result.toString())) { nodeName = "secondNode";
missHgetAllLongAdder.add(1); } else if (slot >= 8192 && slot <= 12287) {
addMissCount(missKeyMap, key); nodeName = "thirdNode";
} } else if (slot > 12288 && slot <= 16383) {
break; nodeName = "fourthNode";
case "getSet": }
getSetLongAdder.add(1); return nodeName;
if (result == null || StringUtil.isEmpty(result.toString())) { }
missGetSetLongAdder.add(1); /**
addMissCount(missKeyMap, key); * 计算key的分布
} *
break; * @param key
case "getSetR": */
getSetRLongAdder.add(1); private void setSoltDistributed(String key) {
if (result == null || StringUtil.isEmpty(result.toString())) { String[] keys = key.split(":");
missgetSetRLongAdder.add(1); if (keys.length < 2) {
addMissCount(missKeyMap, key); return;
} }
break; String keyPrefix = key.substring(0, key.lastIndexOf(":"));
case "setnx": String nodeName = getNodeName(key);
setnxLongAdder.add(1); ConcurrentHashMap<String, AtomicInteger> map = solfDistributedMap.get(keyPrefix);
if (result == null || StringUtil.isEmpty(result.toString())) { if (map == null) {
missSetnxLongAdder.add(1); map = new ConcurrentHashMap<String, AtomicInteger>();
addMissCount(missKeyMap, key); solfDistributedMap.put(keyPrefix,map);
} }
break; AtomicInteger count = map.get(nodeName);
case "hmget": if (count == null) {
hmgetLongAdder.add(1); count = new AtomicInteger(1);
if (result == null || StringUtil.isEmpty(result.toString())) { } else {
misshmgetLongAdder.add(1); count.getAndIncrement();
addMissCount(missKeyMap, key);
}
break;
case "hget":
hgetLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misshgetLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "hlen":
hlenLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misshlenLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "lrange":
lrangeLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misslrangeLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "llen":
llenLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
missllenLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "hkeys":
hkeysLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misshkeysLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "lpop":
lpopLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misslpopLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "zcard":
zcardLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misszcardLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "zrevrange":
zrevrangeLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misszrevrangeLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "ttl":
ttlLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misshttlLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
case "zrank":
zrankLongAdder.add(1);
if (result == null || StringUtil.isEmpty(result.toString())) {
misszrankLongAdder.add(1);
addMissCount(missKeyMap, key);
}
break;
default:
} }
map.put(nodeName, count);
} }
private static String getResourceBasePath() { private static String getResourceBasePath() {
......
pcloud-common-core/src/main/java/com/pcloud/common/core/aspect/LFU.java 0 → 100644
package com.pcloud.common.core.aspect;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class LFU<K, V> {
private final int capcity;
private Map<K, V> cache = new ConcurrentHashMap<>();
private Map<K, HitRate> count = new ConcurrentHashMap<>();
public LFU(int capcity) {
this.capcity = capcity;
}
public Map<K, HitRate> getHotKeyMap() {
Map<K, HitRate> count = this.count;
return count;
}
public void put(K key, V value) {
V v = cache.get(key);
if (v == null) {
if (cache.size() == capcity) {
removeElement();
}
count.put(key, new HitRate(key, 1, System.nanoTime(), value));
} else {
addHitCount(key);
}
cache.put(key, value);
}
public V get(K key) {
V value = cache.get(key);
if (value != null) {
addHitCount(key);
return value;
}
return null;
}
//移除元素
private void removeElement() {
HitRate hr = Collections.min(count.values());
cache.remove(hr.key);
count.remove(hr.key);
}
//更新访问元素状态
private void addHitCount(K key) {
HitRate hitRate = count.get(key);
hitRate.hitCount = hitRate.hitCount + 1;
hitRate.lastTime = System.nanoTime();
}
//内部类
class HitRate implements Comparable<HitRate> {
private K key;
private V v;
private int hitCount;
private long lastTime;
private HitRate(K key, int hitCount, long lastTime, V v) {
this.key = key;
this.hitCount = hitCount;
this.lastTime = lastTime;
this.v = v;
}
@Override
public int compareTo(HitRate o) {
int compare = Integer.compare(this.hitCount, o.hitCount);
return compare == 0 ? Long.compare(this.lastTime, o.lastTime) : compare;
}
public K getKey() {
return key;
}
public V getV() {
return v;
}
public int getHitCount() {
return hitCount;
}
}
}
pcloud-common-core/src/main/java/com/pcloud/common/core/aspect/ParamLogAspect.java deleted 100644 → 0
/**
*
*/
package com.pcloud.common.core.aspect;
import java.lang.reflect.Method;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.Signature;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Configuration;
import com.alibaba.fastjson.JSON;
import com.pcloud.common.utils.string.StringUtil;
/**
* @author：songx
* @date：2018年4月26日,下午2:32:36
*/
@Aspect
@Configuration
public class ParamLogAspect {
/**
*
*/
private final static Logger LOGGER = LoggerFactory.getLogger("");
@Pointcut("execution(* com.pcloud..*(..))")
public void bizPoint() {
}
/**
* 方法执行前以后执行
*
* @param joinPoint
*/
@Before("bizPoint()")
public void doBefore(JoinPoint joinPoint) {
ParamLog paramLog = checkAnnotation(joinPoint);
if (paramLog == null) {
return;
}
Signature signature = joinPoint.getSignature();
String methodName = signature.getName();
String description = paramLog.description();
description = StringUtil.isEmpty(description) ? paramLog.value() : description;
StringBuffer msg = new StringBuffer(signature.getDeclaringTypeName()).append(" 【" + methodName + " before】");
if (!StringUtil.isEmpty(description)) {
msg.append(description);
}
if (!paramLog.isBefore()) {
LOGGER.info(msg.toString());
} else {
String argsJson = JSON.toJSONString(joinPoint.getArgs());
LOGGER.info(msg.append(",[Args]=").append(argsJson).toString());
}
}
/**
* 方法执行完以后执行
*
* @param joinPoint
* @param result
*/
@AfterReturning(pointcut = "bizPoint()", returning = "result")
public void doAfterReturn(JoinPoint joinPoint, Object result) {
ParamLog paramLog = checkAnnotation(joinPoint);
if (paramLog == null) {
return;
}
Signature signature = joinPoint.getSignature();
String methodName = signature.getName();
String description = paramLog.description();
description = StringUtil.isEmpty(description) ? paramLog.value() : description;
StringBuffer msg = new StringBuffer(signature.getDeclaringTypeName())
.append(" 【" + methodName + " afterReturn】");
if (!StringUtil.isEmpty(description)) {
msg.append(description);
}
if (!paramLog.isAfterReturn()) {
LOGGER.info(msg.toString());
} else {
String resultJson = JSON.toJSONString(result);
LOGGER.info(msg.append(",[result]=").append(resultJson).toString());
}
}
private static ParamLog checkAnnotation(JoinPoint joinPoint) {
// 获取方法签名
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
// java reflect相关类，通过反射得到注解
Method method = signature.getMethod();
if (!method.isAnnotationPresent(ParamLog.class)) {
return null;
}
return method.getAnnotation(ParamLog.class);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment