Commit 05bccfea authored by lepdou's avatar lepdou

Merge pull request #199 from lepdou/adminservice_healthcheck

remove active env cache
parents a2bf6d3a d6a65c5e
...@@ -40,18 +40,13 @@ public class PortalSettings { ...@@ -40,18 +40,13 @@ public class PortalSettings {
private List<Env> allEnvs = new ArrayList<Env>(); private List<Env> allEnvs = new ArrayList<Env>();
private volatile boolean updatedFromLastHealthCheck = true; private List<Env> activeEnvs;
//for cache
private List<Env> activeEnvs = new LinkedList<>();
//mark env up or down //mark env up or down
private Map<Env, Boolean> envStatusMark = new ConcurrentHashMap<>(); private Map<Env, Boolean> envStatusMark = new ConcurrentHashMap<>();
private ScheduledExecutorService healthCheckService; private ScheduledExecutorService healthCheckService;
private Lock lock = new ReentrantLock();
@PostConstruct @PostConstruct
private void postConstruct() { private void postConstruct() {
//init origin config envs //init origin config envs
...@@ -63,9 +58,6 @@ public class PortalSettings { ...@@ -63,9 +58,6 @@ public class PortalSettings {
envStatusMark.put(env, true); envStatusMark.put(env, true);
} }
//init active envs
activeEnvs = refreshActiveEnvs();
healthCheckService = Executors.newScheduledThreadPool(1); healthCheckService = Executors.newScheduledThreadPool(1);
healthCheckService healthCheckService
...@@ -75,24 +67,14 @@ public class PortalSettings { ...@@ -75,24 +67,14 @@ public class PortalSettings {
} }
public List<Env> getActiveEnvs() { public List<Env> getActiveEnvs() {
if (updatedFromLastHealthCheck) { List<Env> activeEnvs = new LinkedList<>();
lock.lock();
//maybe refresh many times but not create a bad impression.
activeEnvs = refreshActiveEnvs();
lock.unlock();
}
return activeEnvs;
}
private List<Env> refreshActiveEnvs() {
List<Env> envs = new LinkedList<>();
for (Env env : allEnvs) { for (Env env : allEnvs) {
if (envStatusMark.get(env)) { if (envStatusMark.get(env)) {
envs.add(env); activeEnvs.add(env);
} }
} }
logger.info("refresh active envs"); this.activeEnvs = activeEnvs;
return envs; return activeEnvs;
} }
public Env getFirstAliveEnv() { public Env getFirstAliveEnv() {
...@@ -117,7 +99,6 @@ public class PortalSettings { ...@@ -117,7 +99,6 @@ public class PortalSettings {
public void run() { public void run() {
logger.info("admin server health check start..."); logger.info("admin server health check start...");
boolean hasUpdateStatus = false;
for (Env env : allEnvs) { for (Env env : allEnvs) {
try { try {
...@@ -126,25 +107,20 @@ public class PortalSettings { ...@@ -126,25 +107,20 @@ public class PortalSettings {
if (!envStatusMark.get(env)) { if (!envStatusMark.get(env)) {
envStatusMark.put(env, true); envStatusMark.put(env, true);
healthCheckFailCnt.put(env, 0l); healthCheckFailCnt.put(env, 0l);
hasUpdateStatus = true;
logger.info("env up again [env:{}]", env); logger.info("env up again [env:{}]", env);
} }
} else { } else {
//maybe meta server up but admin server down //maybe meta server up but admin server down
hasUpdateStatus = handleEnvDown(env); handleEnvDown(env);
} }
} catch (Exception e) { } catch (Exception e) {
//maybe meta server down //maybe meta server down
logger.warn("health check fail. [env:{}]", env, e.getMessage()); logger.warn("health check fail. [env:{}]", env, e.getMessage());
hasUpdateStatus = handleEnvDown(env); handleEnvDown(env);
} }
} }
if (!hasUpdateStatus) {
logger.info("admin server health check OK");
}
updatedFromLastHealthCheck = hasUpdateStatus;
} }
private boolean isUp(Env env) { private boolean isUp(Env env) {
...@@ -152,17 +128,15 @@ public class PortalSettings { ...@@ -152,17 +128,15 @@ public class PortalSettings {
return "UP".equals(health.getStatus().getCode()); return "UP".equals(health.getStatus().getCode());
} }
private boolean handleEnvDown(Env env) { private void handleEnvDown(Env env) {
long failCnt = healthCheckFailCnt.get(env); long failCnt = healthCheckFailCnt.get(env);
healthCheckFailCnt.put(env, ++failCnt); healthCheckFailCnt.put(env, ++failCnt);
if (envStatusMark.get(env) && failCnt >= ENV_DIED_THREADHOLD){ if (envStatusMark.get(env) && failCnt >= ENV_DIED_THREADHOLD){
envStatusMark.put(env, false); envStatusMark.put(env, false);
logger.error("env turn to down [env:{}]", env); logger.error("env turn to down [env:{}]", env);
return true;
}else { }else {
logger.warn("[env:{}] down yet.", env); logger.warn("[env:{}] down yet.", env);
return false;
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment