feat(apollo-portal): read dynamic environment from key...

feat(apollo-portal): read dynamic environment from key "apollo.portal.meta.servers" in PortalDB.ServerConfig (#2953)

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/component/AdminServiceAddressLocator.java

 package com.ctrip.framework.apollo.portal.component;
 
-import com.ctrip.framework.apollo.portal.environment.PortalMetaDomainConsts;
+import com.ctrip.framework.apollo.portal.environment.PortalMetaDomainService;
 import com.ctrip.framework.apollo.core.dto.ServiceDTO;
 import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.core.utils.ApolloThreadFactory;
@@ -39,13 +39,17 @@ public class AdminServiceAddressLocator {
 
   private final PortalSettings portalSettings;
   private final RestTemplateFactory restTemplateFactory;
+  private final PortalMetaDomainService portalMetaDomainService;
 
   public AdminServiceAddressLocator(
       final HttpMessageConverters httpMessageConverters,
       final PortalSettings portalSettings,
-      final RestTemplateFactory restTemplateFactory) {
+      final RestTemplateFactory restTemplateFactory,
+      final PortalMetaDomainService portalMetaDomainService
+  ) {
     this.portalSettings = portalSettings;
     this.restTemplateFactory = restTemplateFactory;
+    this.portalMetaDomainService = portalMetaDomainService;
   }
 
   @PostConstruct
@@ -106,17 +110,17 @@ public class AdminServiceAddressLocator {
         return true;
       } catch (Throwable e) {
         logger.error(String.format("Get admin server address from meta server failed. env: %s, meta server address:%s",
-                                   env, PortalMetaDomainConsts.getDomain(env)), e);
+                                   env, portalMetaDomainService.getDomain(env)), e);
         Tracer
             .logError(String.format("Get admin server address from meta server failed. env: %s, meta server address:%s",
-                                    env, PortalMetaDomainConsts.getDomain(env)), e);
+                                    env, portalMetaDomainService.getDomain(env)), e);
       }
     }
     return false;
   }
 
   private ServiceDTO[] getAdminServerAddress(Env env) {
-    String domainName = PortalMetaDomainConsts.getDomain(env);
+    String domainName = portalMetaDomainService.getDomain(env);
     String url = domainName + ADMIN_SERVICE_URL_PATH;
     return restTemplate.getForObject(url, ServiceDTO[].class);
   }

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/component/PortalSettings.java

 package com.ctrip.framework.apollo.portal.component;
 
 
-import com.ctrip.framework.apollo.portal.environment.PortalMetaDomainConsts;
+import com.ctrip.framework.apollo.portal.environment.PortalMetaDomainService;
 import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.core.utils.ApolloThreadFactory;
 import com.ctrip.framework.apollo.portal.api.AdminServiceAPI;
@@ -31,15 +31,21 @@ public class PortalSettings {
 
   private final ApplicationContext applicationContext;
   private final PortalConfig portalConfig;
+  private final PortalMetaDomainService portalMetaDomainService;
 
   private List<Env> allEnvs = new ArrayList<>();
 
   //mark env up or down
   private Map<Env, Boolean> envStatusMark = new ConcurrentHashMap<>();
 
-  public PortalSettings(final ApplicationContext applicationContext, final PortalConfig portalConfig) {
+  public PortalSettings(
+          final ApplicationContext applicationContext,
+          final PortalConfig portalConfig,
+          final PortalMetaDomainService portalMetaDomainService
+  ) {
     this.applicationContext = applicationContext;
     this.portalConfig = portalConfig;
+    this.portalMetaDomainService = portalMetaDomainService;
   }
 
   @PostConstruct
@@ -108,14 +114,14 @@ public class PortalSettings {
             }
           } else {
             logger.error("Env health check failed, maybe because of admin server down. env: {}, meta server address: {}", env,
-                        PortalMetaDomainConsts.getDomain(env));
+                    portalMetaDomainService.getDomain(env));
             handleEnvDown(env);
           }
 
         } catch (Exception e) {
           logger.error("Env health check failed, maybe because of meta server down "
                        + "or configure wrong meta server address. env: {}, meta server address: {}", env,
-                       PortalMetaDomainConsts.getDomain(env), e);
+                  portalMetaDomainService.getDomain(env), e);
           handleEnvDown(env);
         }
       }
@@ -133,17 +139,17 @@ public class PortalSettings {
 
       if (!envStatusMark.get(env)) {
         logger.error("Env is down. env: {}, failed times: {}, meta server address: {}", env, failedTimes,
-                     PortalMetaDomainConsts.getDomain(env));
+                portalMetaDomainService.getDomain(env));
       } else {
         if (failedTimes >= ENV_DOWN_THRESHOLD) {
           envStatusMark.put(env, false);
           logger.error("Env is down because health check failed for {} times, "
                        + "which equals to down threshold. env: {}, meta server address: {}", ENV_DOWN_THRESHOLD, env,
-                       PortalMetaDomainConsts.getDomain(env));
+                  portalMetaDomainService.getDomain(env));
         } else {
           logger.error(
               "Env health check failed for {} times which less than down threshold. down threshold:{}, env: {}, meta server address: {}",
-              failedTimes, ENV_DOWN_THRESHOLD, env, PortalMetaDomainConsts.getDomain(env));
+              failedTimes, ENV_DOWN_THRESHOLD, env, portalMetaDomainService.getDomain(env));
         }
       }
 

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/component/RetryableRestTemplate.java

 package com.ctrip.framework.apollo.portal.component;
 
 import com.ctrip.framework.apollo.common.exception.ServiceException;
-import com.ctrip.framework.apollo.portal.environment.PortalMetaDomainConsts;
+import com.ctrip.framework.apollo.portal.environment.PortalMetaDomainService;
 import com.ctrip.framework.apollo.core.dto.ServiceDTO;
 import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.constant.TracerEventType;
@@ -40,12 +40,16 @@ public class RetryableRestTemplate {
 
   private final RestTemplateFactory restTemplateFactory;
   private final AdminServiceAddressLocator adminServiceAddressLocator;
+  private final PortalMetaDomainService portalMetaDomainService;
 
   public RetryableRestTemplate(
       final @Lazy RestTemplateFactory restTemplateFactory,
-      final @Lazy AdminServiceAddressLocator adminServiceAddressLocator) {
+      final @Lazy AdminServiceAddressLocator adminServiceAddressLocator,
+      final PortalMetaDomainService portalMetaDomainService
+  ) {
     this.restTemplateFactory = restTemplateFactory;
     this.adminServiceAddressLocator = adminServiceAddressLocator;
+    this.portalMetaDomainService = portalMetaDomainService;
   }
 
 
@@ -116,7 +120,7 @@ public class RetryableRestTemplate {
     //all admin server down
     ServiceException e =
         new ServiceException(String.format("Admin servers are unresponsive. meta server address: %s, admin servers: %s",
-                PortalMetaDomainConsts.getDomain(env), services));
+                portalMetaDomainService.getDomain(env), services));
     ct.setStatus(e);
     ct.complete();
     throw e;
@@ -160,7 +164,7 @@ public class RetryableRestTemplate {
     //all admin server down
     ServiceException e =
         new ServiceException(String.format("Admin servers are unresponsive. meta server address: %s, admin servers: %s",
-                PortalMetaDomainConsts.getDomain(env), services));
+                portalMetaDomainService.getDomain(env), services));
     ct.setStatus(e);
     ct.complete();
     throw e;
@@ -175,7 +179,7 @@ public class RetryableRestTemplate {
       ServiceException e = new ServiceException(String.format("No available admin server."
                                                               + " Maybe because of meta server down or all admin server down. "
                                                               + "Meta server address: %s",
-              PortalMetaDomainConsts.getDomain(env)));
+              portalMetaDomainService.getDomain(env)));
       ct.setStatus(e);
       ct.complete();
       throw e;

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/component/config/PortalConfig.java

@@ -3,8 +3,8 @@ package com.ctrip.framework.apollo.portal.component.config;
 
 import com.ctrip.framework.apollo.common.config.RefreshableConfig;
 import com.ctrip.framework.apollo.common.config.RefreshablePropertySource;
-import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.entity.vo.Organization;
+import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.service.PortalDBPropertySource;
 import com.ctrip.framework.apollo.portal.service.SystemRoleManagerService;
 import com.google.common.base.Strings;
@@ -12,21 +12,27 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
 import java.lang.reflect.Type;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 
 @Component
 public class PortalConfig extends RefreshableConfig {
 
+  private static final Logger logger = LoggerFactory.getLogger(PortalConfig.class);
+
   private Gson gson = new Gson();
   private static final Type ORGANIZATION = new TypeToken<List<Organization>>() {
   }.getType();
 
+  /**
+   * meta servers config in "PortalDB.ServerConfig"
+   */
+  private static final Type META_SERVERS = new TypeToken<Map<String, String>>(){}.getType();
+
   private final PortalDBPropertySource portalDBPropertySource;
 
   public PortalConfig(final PortalDBPropertySource portalDBPropertySource) {
@@ -52,6 +58,28 @@ public class PortalConfig extends RefreshableConfig {
     return envs;
   }
 
+  /**
+   * @return the relationship between environment and its meta server. empty if meet exception
+   */
+  public Map<String, String> getMetaServers() {
+    final String key = "apollo.portal.meta.servers";
+    String jsonContent = getValue(key);
+    if(null == jsonContent) {
+      return Collections.emptyMap();
+    }
+
+    // watch out that the format of content may be wrong
+    // that will cause exception
+    Map<String, String> map = Collections.emptyMap();
+    try {
+      // try to parse
+      map = gson.fromJson(jsonContent, META_SERVERS);
+    } catch (Exception e) {
+      logger.error("wrong format with key: {}", key);
+    }
+    return map;
+  }
+
   public List<String> superAdmins() {
     String superAdminConfig = getValue("superAdmin", "");
     if (Strings.isNullOrEmpty(superAdminConfig)) {

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/controller/SystemInfoController.java

 package com.ctrip.framework.apollo.portal.controller;
 
 import com.ctrip.framework.apollo.Apollo;
-import com.ctrip.framework.apollo.portal.environment.PortalMetaDomainConsts;
+import com.ctrip.framework.apollo.portal.environment.PortalMetaDomainService;
 import com.ctrip.framework.apollo.core.dto.ServiceDTO;
 import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.component.PortalSettings;
@@ -32,12 +32,16 @@ public class SystemInfoController {
   private RestTemplate restTemplate;
   private final PortalSettings portalSettings;
   private final RestTemplateFactory restTemplateFactory;
+  private final PortalMetaDomainService portalMetaDomainService;
 
   public SystemInfoController(
       final PortalSettings portalSettings,
-      final RestTemplateFactory restTemplateFactory) {
+      final RestTemplateFactory restTemplateFactory,
+      final PortalMetaDomainService portalMetaDomainService
+  ) {
     this.portalSettings = portalSettings;
     this.restTemplateFactory = restTemplateFactory;
+    this.portalMetaDomainService = portalMetaDomainService;
   }
 
   @PostConstruct
@@ -101,13 +105,13 @@ public class SystemInfoController {
 
   private EnvironmentInfo adaptEnv2EnvironmentInfo(final Env env) {
     EnvironmentInfo environmentInfo = new EnvironmentInfo();
-    String metaServerAddresses = PortalMetaDomainConsts.getMetaServerAddress(env);
+    String metaServerAddresses = portalMetaDomainService.getMetaServerAddress(env);
 
     environmentInfo.setEnv(env);
     environmentInfo.setActive(portalSettings.isEnvActive(env));
     environmentInfo.setMetaServerAddress(metaServerAddresses);
 
-    String selectedMetaServerAddress = PortalMetaDomainConsts.getDomain(env);
+    String selectedMetaServerAddress = portalMetaDomainService.getDomain(env);
     try {
       environmentInfo.setConfigServices(getServerAddress(selectedMetaServerAddress, CONFIG_SERVICE_URL_PATH));
 

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/environment/DatabasePortalMetaServerProvider.java

+package com.ctrip.framework.apollo.portal.environment;
+
+import com.ctrip.framework.apollo.portal.component.config.PortalConfig;
+import java.util.Map;
+
+/**
+ * load meta server addressed from database.
+ * PortalDB.ServerConfig
+ */
+class DatabasePortalMetaServerProvider implements PortalMetaServerProvider {
+
+  /**
+   * read config from database
+   */
+  private final PortalConfig portalConfig;
+
+  private volatile Map<Env, String> addresses;
+
+  public DatabasePortalMetaServerProvider(
+      final PortalConfig portalConfig
+  ) {
+    this.portalConfig = portalConfig;
+    // will cause NullPointException if portalConfig is null
+    Map<String, String> map =  portalConfig.getMetaServers();
+    addresses = Env.conversionKey(map);
+  }
+
+  @Override
+  public String getMetaServerAddress(Env targetEnv) {
+    return addresses.get(targetEnv);
+  }
+
+  @Override
+  public boolean exists(Env targetEnv) {
+    return addresses.containsKey(targetEnv);
+  }
+
+  @Override
+  public void reload() {
+    Map<String, String> map =  portalConfig.getMetaServers();
+    addresses = Env.conversionKey(map);
+  }
+
+}

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/environment/DefaultPortalMetaServerProvider.java

+package com.ctrip.framework.apollo.portal.environment;
+
+import static com.ctrip.framework.apollo.portal.environment.Env.conversionKey;
+
+import com.ctrip.framework.apollo.core.utils.ResourceUtils;
+import com.ctrip.framework.apollo.portal.util.KeyValueUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Only use in apollo-portal
+ * load all meta server address from
+ *  - System Property           [key ends with "_meta" (case insensitive)]
+ *  - OS environment variable   [key ends with "_meta" (case insensitive)]
+ *  - user's configuration file [key ends with ".meta" (case insensitive)]
+ * when apollo-portal start up.
+ * @see com.ctrip.framework.apollo.core.internals.LegacyMetaServerProvider
+ * @author wxq
+ */
+class DefaultPortalMetaServerProvider implements PortalMetaServerProvider {
+
+    private static final Logger logger = LoggerFactory.getLogger(DefaultPortalMetaServerProvider.class);
+
+    /**
+     * environments and their meta server address
+     * properties file path
+     */
+    private static final String APOLLO_ENV_PROPERTIES_FILE_PATH = "apollo-env.properties";
+
+    private static final DefaultPortalMetaServerProvider INSTANCE = new DefaultPortalMetaServerProvider();
+
+
+    public static DefaultPortalMetaServerProvider getInstance() {
+        return INSTANCE;
+    }
+
+    private volatile Map<Env, String> domains;
+
+    private DefaultPortalMetaServerProvider() {
+        domains = initializeDomains();
+    }
+
+    @Override
+    public String getMetaServerAddress(Env targetEnv) {
+        String metaServerAddress = domains.get(targetEnv);
+        return metaServerAddress == null ? null : metaServerAddress.trim();
+    }
+
+    @Override
+    public boolean exists(Env targetEnv) {
+        return domains.containsKey(targetEnv);
+    }
+
+    @Override
+    public void reload() {
+        domains = initializeDomains();
+    }
+
+    /**
+     * load all environment's meta address dynamically when this class loaded by JVM
+     */
+    private static Map<Env, String> initializeDomains() {
+
+        // add to domain
+        Map<Env, String> map = new ConcurrentHashMap<>();
+        // lower priority add first
+        map.putAll(getDomainsFromPropertiesFile());
+        map.putAll(getDomainsFromOSEnvironment());
+        map.putAll(getDomainsFromSystemProperty());
+
+        // log all
+        logger.info("Loaded meta server addresses: {}", map);
+        return map;
+    }
+
+    static Map<Env, String> getDomainsFromSystemProperty() {
+        // find key-value from System Property which key ends with "_meta" (case insensitive)
+        Map<String, String> metaServerAddressesFromSystemProperty = KeyValueUtils.filterWithKeyIgnoreCaseEndsWith(System.getProperties(), "_meta");
+        // remove key's suffix "_meta" (case insensitive)
+        metaServerAddressesFromSystemProperty = KeyValueUtils.removeKeySuffix(metaServerAddressesFromSystemProperty, "_meta".length());
+        return conversionKey(metaServerAddressesFromSystemProperty);
+    }
+
+    static Map<Env, String> getDomainsFromOSEnvironment() {
+        // find key-value from OS environment variable which key ends with "_meta" (case insensitive)
+        Map<String, String> metaServerAddressesFromOSEnvironment = KeyValueUtils.filterWithKeyIgnoreCaseEndsWith(System.getenv(), "_meta");
+        // remove key's suffix "_meta" (case insensitive)
+        metaServerAddressesFromOSEnvironment = KeyValueUtils.removeKeySuffix(metaServerAddressesFromOSEnvironment, "_meta".length());
+        return conversionKey(metaServerAddressesFromOSEnvironment);
+    }
+
+    static Map<Env, String> getDomainsFromPropertiesFile() {
+        // find key-value from properties file which key ends with ".meta" (case insensitive)
+        Properties properties = new Properties();
+        properties = ResourceUtils.readConfigFile(APOLLO_ENV_PROPERTIES_FILE_PATH, properties);
+        Map<String, String> metaServerAddressesFromPropertiesFile = KeyValueUtils.filterWithKeyIgnoreCaseEndsWith(properties, ".meta");
+        // remove key's suffix ".meta" (case insensitive)
+        metaServerAddressesFromPropertiesFile = KeyValueUtils.removeKeySuffix(metaServerAddressesFromPropertiesFile, ".meta".length());
+        return conversionKey(metaServerAddressesFromPropertiesFile);
+    }
+
+}

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/environment/Env.java

@@ -192,5 +192,23 @@ public class Env {
     public String getName() {
         return name;
     }
-    
+
+    /**
+     * conversion key from {@link String} to {@link Env}
+     * @param metaServerAddresses key is environment, value is environment's meta server address
+     * @return relationship between {@link Env} and meta server address
+     */
+    public static Map<Env, String> conversionKey(Map<String, String> metaServerAddresses) {
+        // add to domain
+        Map<Env, String> map = new ConcurrentHashMap<>();
+        for(Map.Entry<String, String> entry : metaServerAddresses.entrySet()) {
+            // add new environment
+            Env env = Env.addEnvironment(entry.getKey());
+            // get meta server address value
+            String value = entry.getValue();
+            // put pair (Env, meta server address)
+            map.put(env, value);
+        }
+        return map;
+    }
 }

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/environment/PortalMetaDomainConsts.java → apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/environment/PortalMetaDomainService.java

@@ -2,14 +2,14 @@ package com.ctrip.framework.apollo.portal.environment;
 
 import com.ctrip.framework.apollo.core.utils.ApolloThreadFactory;
 import com.ctrip.framework.apollo.core.utils.NetUtil;
+import com.ctrip.framework.apollo.portal.component.config.PortalConfig;
 import com.ctrip.framework.apollo.tracer.Tracer;
 import com.ctrip.framework.apollo.tracer.spi.Transaction;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -17,6 +17,9 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
 
 /**
  * Only use in apollo-portal
@@ -26,26 +29,38 @@ import java.util.concurrent.atomic.AtomicBoolean;
  * @see com.ctrip.framework.apollo.core.MetaDomainConsts
  * @author wxq
  */
-public class PortalMetaDomainConsts {
+@Service
+public class PortalMetaDomainService {
 
     public static final String DEFAULT_META_URL = "http://apollo.meta";
 
     // env -> meta server address cache
     private static final Map<Env, String> metaServerAddressCache = Maps.newConcurrentMap();
 
-    // initialize meta server provider without cache
-    private static final PortalMetaServerProvider metaServerProvider = PortalMetaServerProvider.getInstance();
+    /**
+     * initialize meta server provider without cache.
+     * Multiple {@link PortalMetaServerProvider}
+     */
+    private final List<PortalMetaServerProvider> portalMetaServerProviders = new ArrayList<>();
 
     private static final long REFRESH_INTERVAL_IN_SECOND = 60;// 1 min
-    private static final Logger logger = LoggerFactory.getLogger(PortalMetaDomainConsts.class);
+    private static final Logger logger = LoggerFactory.getLogger(PortalMetaDomainService.class);
     // comma separated meta server address -> selected single meta server address cache
     private static final Map<String, String> selectedMetaServerAddressCache = Maps.newConcurrentMap();
     private static final AtomicBoolean periodicRefreshStarted = new AtomicBoolean(false);
 
+    public PortalMetaDomainService(final PortalConfig portalConfig) {
+        // high priority with data in database
+        portalMetaServerProviders.add(new DatabasePortalMetaServerProvider(portalConfig));
+
+        // System properties, OS environment, configuration file
+        portalMetaServerProviders.add(DefaultPortalMetaServerProvider.getInstance());
+    }
+
     /**
      * Return one meta server address. If multiple meta server addresses are configured, will select one.
      */
-    public static String getDomain(Env env) {
+    synchronized public String getDomain(Env env) {
         String metaServerAddress = getMetaServerAddress(env);
         // if there is more than one address, need to select one
         if (metaServerAddress.contains(",")) {
@@ -57,13 +72,13 @@ public class PortalMetaDomainConsts {
     /**
      * Return meta server address. If multiple meta server addresses are configured, will return the comma separated string.
      */
-    public static String getMetaServerAddress(Env env) {
+    synchronized public String getMetaServerAddress(Env env) {
         // in cache?
         if (!metaServerAddressCache.containsKey(env)) {
             // put it to cache
             metaServerAddressCache.put(
                     env,
-                    getMetaServerAddressCacheValue(metaServerProvider, env)
+                    getMetaServerAddressCacheValue(portalMetaServerProviders, env)
             );
         }
 
@@ -75,14 +90,25 @@ public class PortalMetaDomainConsts {
      * Get the meta server from provider by given environment.
      * If there is no available meta server url for the given environment,
      * the default meta server url will be used(http://apollo.meta).
-     * @param provider
-     * @param env
-     * @return
+     * @param providers provide environment's meta server address
+     * @param env environment
+     * @return  meta server address
      */
-    private static String getMetaServerAddressCacheValue(PortalMetaServerProvider provider, Env env) {
-        String metaAddress = provider.getMetaServerAddress(env);
-        logger.info("Located meta server address [{}] for env [{}]", metaAddress, env);
+    private static String getMetaServerAddressCacheValue(
+        Collection<PortalMetaServerProvider> providers, Env env) {
 
+        // null value
+        String metaAddress = null;
+
+        for(PortalMetaServerProvider portalMetaServerProvider : providers) {
+            if(portalMetaServerProvider.exists(env)) {
+                metaAddress = portalMetaServerProvider.getMetaServerAddress(env);
+                logger.info("Located meta server address [{}] for env [{}]", metaAddress, env);
+                break;
+            }
+        }
+
+        // check find it or not
         if (Strings.isNullOrEmpty(metaAddress)) {
             // Fallback to default meta address
             metaAddress = DEFAULT_META_URL;
@@ -93,6 +119,17 @@ public class PortalMetaDomainConsts {
         return metaAddress.trim();
     }
 
+    /**
+     * reload all {@link PortalMetaServerProvider}.
+     * clear cache {@link this#metaServerAddressCache}
+     */
+    public void reload() {
+        for(PortalMetaServerProvider portalMetaServerProvider : portalMetaServerProviders) {
+            portalMetaServerProvider.reload();
+        }
+        metaServerAddressCache.clear();
+    }
+
     /**
      * Select one available meta server from the comma separated meta server addresses, e.g.
      * http://1.2.3.4:8080,http://2.3.4.5:8080

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/environment/PortalMetaServerProvider.java

 package com.ctrip.framework.apollo.portal.environment;
 
-import com.ctrip.framework.apollo.core.utils.ResourceUtils;
-import com.ctrip.framework.apollo.portal.util.KeyValueUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-
 /**
- * Only use in apollo-portal
- * load all meta server address from
- *  - System Property           [key ends with "_meta" (case insensitive)]
- *  - OS environment variable   [key ends with "_meta" (case insensitive)]
- *  - user's configuration file [key ends with ".meta" (case insensitive)]
- * when apollo-portal start up.
- * @see com.ctrip.framework.apollo.core.internals.LegacyMetaServerProvider
+ * For the supporting of multiple meta server address providers.
+ * From configuration file,
+ * from OS environment,
+ * From database,
+ * ...
+ * Just implement this interface
  * @author wxq
  */
-public class PortalMetaServerProvider {
-
-    private static final Logger logger = LoggerFactory.getLogger(PortalMetaServerProvider.class);
-
-    /**
-     * environments and their meta server address
-     * properties file path
-     */
-    private static final String APOLLO_ENV_PROPERTIES_FILE_PATH = "apollo-env.properties";
-
-    private static final PortalMetaServerProvider instance = new PortalMetaServerProvider();
-
-    private Map<Env, String> domains;
-
-    private PortalMetaServerProvider() {
-        domains = initializeDomains();
-    }
-
-    public static PortalMetaServerProvider getInstance() {
-        return instance;
-    }
-
-    String getMetaServerAddress(Env targetEnv) {
-        String metaServerAddress = domains.get(targetEnv);
-        return metaServerAddress == null ? null : metaServerAddress.trim();
-    }
-
-    /**
-     * load all environment's meta address dynamically when this class loaded by JVM
-     */
-    private Map<Env, String> initializeDomains() {
-        // find key-value from System Property which key ends with "_meta" (case insensitive)
-        Map<String, String> metaServerAddressesFromSystemProperty = KeyValueUtils.filterWithKeyIgnoreCaseEndsWith(System.getProperties(), "_meta");
-        // remove key's suffix "_meta" (case insensitive)
-        metaServerAddressesFromSystemProperty = KeyValueUtils.removeKeySuffix(metaServerAddressesFromSystemProperty, "_meta".length());
-
-        // find key-value from OS environment variable which key ends with "_meta" (case insensitive)
-        Map<String, String> metaServerAddressesFromOSEnvironment = KeyValueUtils.filterWithKeyIgnoreCaseEndsWith(System.getenv(), "_meta");
-        // remove key's suffix "_meta" (case insensitive)
-        metaServerAddressesFromOSEnvironment = KeyValueUtils.removeKeySuffix(metaServerAddressesFromOSEnvironment, "_meta".length());
-
-        // find key-value from properties file which key ends with ".meta" (case insensitive)
-        Properties properties = new Properties();
-        properties = ResourceUtils.readConfigFile(APOLLO_ENV_PROPERTIES_FILE_PATH, properties);
-        Map<String, String> metaServerAddressesFromPropertiesFile = KeyValueUtils.filterWithKeyIgnoreCaseEndsWith(properties, ".meta");
-        // remove key's suffix ".meta" (case insensitive)
-        metaServerAddressesFromPropertiesFile = KeyValueUtils.removeKeySuffix(metaServerAddressesFromPropertiesFile, ".meta".length());
-
-        // begin to add key-value, key is environment, value is meta server address matched
-        Map<String, String> metaServerAddresses = new HashMap<>();
-        // lower priority add first
-        metaServerAddresses.putAll(metaServerAddressesFromPropertiesFile);
-        metaServerAddresses.putAll(metaServerAddressesFromOSEnvironment);
-        metaServerAddresses.putAll(metaServerAddressesFromSystemProperty);
+public interface PortalMetaServerProvider {
 
-        // add to domain
-        Map<Env, String> map = new ConcurrentHashMap<>();
-        for(Map.Entry<String, String> entry : metaServerAddresses.entrySet()) {
-            // add new environment
-            Env env = Env.addEnvironment(entry.getKey());
-            // get meta server address value
-            String value = entry.getValue();
-            // put pair (Env, meta server address)
-            map.put(env, value);
-        }
+  /**
+   * @param targetEnv environment
+   * @return meta server address matched environment
+   */
+  String getMetaServerAddress(Env targetEnv);
 
-        // log all
-        logger.info("Loaded meta server addresses: {}", map);
-        return map;
-    }
+  /**
+   * @param targetEnv environment
+   * @return environment's meta server address exists or not
+   */
+  boolean exists(Env targetEnv);
 
-    /**
-     * add a environment's meta server address
-     * for the feature: add self-define environment in the web ui
-     */
-    void mockMetaServerAddress(Env env, String metaServerAddress) {
-        domains.put(env, metaServerAddress);
-    }
+  /**
+   * reload the meta server address in runtime
+   */
+  void reload();
 
-    /**
-     * only for test
-     * reload all environments and meta server addresses
-     */
-    void reset() {
-        domains = initializeDomains();
-    }
 }

apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/RetryableRestTemplateTest.java

 package com.ctrip.framework.apollo.portal;
 
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
 import com.ctrip.framework.apollo.common.exception.ServiceException;
 import com.ctrip.framework.apollo.core.dto.ServiceDTO;
-import com.ctrip.framework.apollo.portal.environment.Env;
 import com.ctrip.framework.apollo.portal.component.AdminServiceAddressLocator;
 import com.ctrip.framework.apollo.portal.component.RetryableRestTemplate;
-
+import com.ctrip.framework.apollo.portal.environment.Env;
+import com.ctrip.framework.apollo.portal.environment.PortalMetaDomainService;
+import java.net.SocketTimeoutException;
+import java.util.Arrays;
+import java.util.Collections;
 import org.apache.http.HttpHost;
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.conn.HttpHostConnectException;
@@ -19,21 +27,14 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.client.ResourceAccessException;
 import org.springframework.web.client.RestTemplate;
 
-import java.net.SocketTimeoutException;
-import java.util.Arrays;
-import java.util.Collections;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
 public class RetryableRestTemplateTest extends AbstractUnitTest {
 
   @Mock
   private AdminServiceAddressLocator serviceAddressLocator;
   @Mock
   private RestTemplate restTemplate;
+  @Mock
+  private PortalMetaDomainService portalMetaDomainService;
   @InjectMocks
   private RetryableRestTemplate retryableRestTemplate;
 

apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/environment/DatabasePortalMetaServerProviderTest.java

+package com.ctrip.framework.apollo.portal.environment;
+
+import static org.junit.Assert.*;
+
+import com.ctrip.framework.apollo.portal.component.config.PortalConfig;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DatabasePortalMetaServerProviderTest {
+
+  private DatabasePortalMetaServerProvider databasePortalMetaServerProvider;
+
+  @Before
+  public void init() {
+    MockitoAnnotations.initMocks(this);
+    // mock it
+    PortalConfig portalConfig = Mockito.mock(PortalConfig.class);
+    final Map<String, String> map = new HashMap<>();
+    map.put("nothing", "http://unknown.com");
+    map.put("dev", "http://server.com:8080");
+    Mockito.when(portalConfig.getMetaServers()).thenReturn(map);
+
+    // use mocked object to construct
+    databasePortalMetaServerProvider = new DatabasePortalMetaServerProvider(portalConfig);
+  }
+
+  @Test
+  public void getMetaServerAddress() {
+    String address = databasePortalMetaServerProvider.getMetaServerAddress(Env.DEV);
+    assertEquals("http://server.com:8080", address);
+  }
+}
\ No newline at end of file

apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/environment/PortalMetaServerProviderTest.java → apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/environment/DefaultPortalMetaServerProviderTest.java

 package com.ctrip.framework.apollo.portal.environment;
 
+import static org.junit.Assert.assertEquals;
+
+import com.ctrip.framework.apollo.portal.AbstractUnitTest;
 import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
+public class DefaultPortalMetaServerProviderTest extends AbstractUnitTest {
 
-public class PortalMetaServerProviderTest {
+  private final DefaultPortalMetaServerProvider defaultPortalMetaServerProvider = DefaultPortalMetaServerProvider.getInstance();
 
-  private PortalMetaServerProvider portalMetaServerProvider;
-
-  @Before
-  public void setUp() throws Exception {
-    portalMetaServerProvider = PortalMetaServerProvider.getInstance();
-  }
 
   @After
   public void tearDown() throws Exception {
     System.clearProperty("dev_meta");
     System.clearProperty("fat_meta");
-    PortalMetaServerProvider.getInstance().reset();
+    defaultPortalMetaServerProvider.reload();
   }
 
   @Test
   public void testFromPropertyFile() {
-    assertEquals("http://localhost:8080", portalMetaServerProvider.getMetaServerAddress(Env.LOCAL));
-    assertEquals("${dev_meta}", portalMetaServerProvider.getMetaServerAddress(Env.DEV));
-    assertEquals("${pro_meta}", portalMetaServerProvider.getMetaServerAddress(Env.PRO));
+    assertEquals("http://localhost:8080", defaultPortalMetaServerProvider.getMetaServerAddress(Env.LOCAL));
+    assertEquals("${dev_meta}", defaultPortalMetaServerProvider.getMetaServerAddress(Env.DEV));
+    assertEquals("${pro_meta}", defaultPortalMetaServerProvider.getMetaServerAddress(Env.PRO));
   }
 
   /**
@@ -39,17 +35,17 @@ public class PortalMetaServerProviderTest {
     System.setProperty("dev_meta", someDevMetaAddress);
     System.setProperty("fat_meta", someFatMetaAddress);
     // reload above added
-    portalMetaServerProvider.reset();
-    assertEquals(someDevMetaAddress, portalMetaServerProvider.getMetaServerAddress(Env.DEV));
-    assertEquals(someFatMetaAddress, portalMetaServerProvider.getMetaServerAddress(Env.FAT));
+    defaultPortalMetaServerProvider.reload();
+    assertEquals(someDevMetaAddress, defaultPortalMetaServerProvider.getMetaServerAddress(Env.DEV));
+    assertEquals(someFatMetaAddress, defaultPortalMetaServerProvider.getMetaServerAddress(Env.FAT));
 
     String randomAddress = "randomAddress";
     String randomEnvironment = "randomEnvironment";
     System.setProperty(randomEnvironment + "_meta", randomAddress);
     // reload above added
-    portalMetaServerProvider.reset();
+    defaultPortalMetaServerProvider.reload();
     assertEquals(randomAddress,
-        portalMetaServerProvider.getMetaServerAddress(Env.valueOf(randomEnvironment)));
+        defaultPortalMetaServerProvider.getMetaServerAddress(Env.valueOf(randomEnvironment)));
 
     // clear the property
     System.clearProperty(randomEnvironment + "_meta");

apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/environment/PortalMetaDomainConstsTest.java → apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/environment/PortalMetaDomainServiceTest.java

 package com.ctrip.framework.apollo.portal.environment;
 
-import org.junit.After;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
+import com.ctrip.framework.apollo.portal.component.config.PortalConfig;
+import java.util.HashMap;
+import java.util.Map;
 import javax.servlet.http.HttpServletResponse;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+public class PortalMetaDomainServiceTest extends BaseIntegrationTest {
+
+    @Mock
+    private PortalMetaDomainService portalMetaDomainService;
 
-public class PortalMetaDomainConstsTest extends BaseIntegrationTest {
+    @Before
+    public void init() {
+        MockitoAnnotations.initMocks(this);
+        // mock it
+        PortalConfig portalConfig = Mockito.mock(PortalConfig.class);
+        final Map<String, String> map = new HashMap<>();
+        map.put("nothing", "http://unknown.com");
+        Mockito.when(portalConfig.getMetaServers()).thenReturn(map);
 
-    @Override
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-        clear();
+        portalMetaDomainService = new PortalMetaDomainService(portalConfig);
     }
 
     @Test
@@ -22,12 +35,12 @@ public class PortalMetaDomainConstsTest extends BaseIntegrationTest {
         // local
         String localMetaServerAddress = "http://localhost:8080";
         mockMetaServerAddress(Env.LOCAL, localMetaServerAddress);
-        assertEquals(localMetaServerAddress, PortalMetaDomainConsts.getDomain(Env.LOCAL));
+        assertEquals(localMetaServerAddress, portalMetaDomainService.getDomain(Env.LOCAL));
 
         // add this environment without meta server address
         String randomEnvironment = "randomEnvironment";
         Env.addEnvironment(randomEnvironment);
-        assertEquals(PortalMetaDomainConsts.DEFAULT_META_URL, PortalMetaDomainConsts.getDomain(Env.valueOf(randomEnvironment)));
+        assertEquals(PortalMetaDomainService.DEFAULT_META_URL, portalMetaDomainService.getDomain(Env.valueOf(randomEnvironment)));
     }
 
     @Test
@@ -40,9 +53,10 @@ public class PortalMetaDomainConstsTest extends BaseIntegrationTest {
 
         mockMetaServerAddress(Env.FAT, validServer + "," + invalidServer);
         mockMetaServerAddress(Env.UAT, invalidServer + "," + validServer);
+        portalMetaDomainService.reload();
 
-        assertEquals(validServer.trim(), PortalMetaDomainConsts.getDomain(Env.FAT));
-        assertEquals(validServer.trim(), PortalMetaDomainConsts.getDomain(Env.UAT));
+        assertEquals(validServer.trim(), portalMetaDomainService.getDomain(Env.FAT));
+        assertEquals(validServer.trim(), portalMetaDomainService.getDomain(Env.UAT));
     }
 
     @Test
@@ -52,16 +66,16 @@ public class PortalMetaDomainConstsTest extends BaseIntegrationTest {
 
         mockMetaServerAddress(Env.LPT, invalidServer + "," + anotherInvalidServer);
 
-        String metaServer = PortalMetaDomainConsts.getDomain(Env.LPT);
+        portalMetaDomainService.reload();
+
+        String metaServer = portalMetaDomainService.getDomain(Env.LPT);
 
         assertTrue(metaServer.equals(invalidServer.trim()) || metaServer.equals(anotherInvalidServer.trim()));
     }
 
     private void mockMetaServerAddress(Env env, String metaServerAddress) {
-        PortalMetaServerProvider.getInstance().mockMetaServerAddress(env, metaServerAddress);
+        // add it to system's property
+        System.setProperty(env.getName() + "_meta", metaServerAddress);
     }
 
-    private void clear() {
-        PortalMetaServerProvider.getInstance().reset();
-    }
 }
\ No newline at end of file