Merge branch 'staging' into 'master'

Staging

See merge request !92

src/main/java/com/edgec/browserbackend/account/service/impl/PaymentServiceImpl.java

@@ -367,8 +367,8 @@ public class PaymentServiceImpl implements PaymentService {
         Account byName = accountService.findByName(username);
         if (byName == null )
             throw new ClientRequestException(AccountErrorCode.NAMENOTEXIST, "account does not exist: " + username);
-//        if (byName.getPermission() < 8)
-//            throw new ClientRequestException(AccountErrorCode.NOPERMISSION, "account does not have permission: " + username);
+        if (byName.getPermission() < 4)
+            throw new ClientRequestException(AccountErrorCode.NOPERMISSION, "account does not have permission: " + username);
         boolean isVpsClient = true;
 
         UserPayment internalOrder = new UserPayment();
@@ -421,8 +421,8 @@ public class PaymentServiceImpl implements PaymentService {
         Account byName = accountService.findByName(username);
         if (byName == null)
             throw new ClientRequestException(AccountErrorCode.NAMENOTEXIST, "account does not exist: " + username);
-//        if (byName.getPermission() < 8)
-//            throw new ClientRequestException(AccountErrorCode.NOPERMISSION, "account does not have permission: " + username);
+        if (byName.getPermission() < 4)
+            throw new ClientRequestException(AccountErrorCode.NOPERMISSION, "account does not have permission: " + username);
 
         boolean isVpsClient = true;
 

src/main/java/com/edgec/browserbackend/browser/domain/IpResource.java

@@ -37,6 +37,7 @@ public class IpResource implements Serializable {
     private double price;
 
     private boolean specialLine; //是否使用专线
+    private boolean usingSpecialLine; //是否正在使用专线
     //专线数据
     private String proxyUsername; //专线的代理用户名
     private String proxyPassword; //专线的代理密码
@@ -46,6 +47,7 @@ public class IpResource implements Serializable {
 
     private boolean isLocked;
     private long lockTimestamp;
+    private long healthLockTimestamp;
 
     private String unit;
     private int period;
@@ -285,4 +287,19 @@ public class IpResource implements Serializable {
         this.specialLine = specialLine;
     }
 
+    public long getHealthLockTimestamp() {
+        return healthLockTimestamp;
+    }
+
+    public void setHealthLockTimestamp(long healthLockTimestamp) {
+        this.healthLockTimestamp = healthLockTimestamp;
+    }
+
+    public boolean isUsingSpecialLine() {
+        return usingSpecialLine;
+    }
+
+    public void setUsingSpecialLine(boolean usingSpecialLine) {
+        this.usingSpecialLine = usingSpecialLine;
+    }
 }

src/main/java/com/edgec/browserbackend/browser/repository/IpResourceRepositoryCustom.java

@@ -10,6 +10,10 @@ public interface IpResourceRepositoryCustom {
 
     boolean unLockTask(String id);
 
+    boolean healthLock(IpResource ipResource);
+
+    boolean unLockHealth(String id);
+
     List<IpResource> sampleTasks(int status, long timestamp);
 
     List<IpResource> sampleTasks(List<Integer> status);

src/main/java/com/edgec/browserbackend/browser/repository/IpResourceRepositoryCustomImpl.java

@@ -34,7 +34,7 @@ public class IpResourceRepositoryCustomImpl implements IpResourceRepositoryCusto
         BasicQuery basicQuery = new BasicQuery(doc);
         Criteria criteria = new Criteria();
         criteria.orOperator(where("id").is(ipResource.getId()).and("isLocked").is(false).and("status").is(ipResource.getStatus()).and("isDeleted").is(false),
-                where("lockTimestamp").lte(Instant.now().minusSeconds(300).toEpochMilli()).and("status").is(ipResource.getStatus())).and("isDeleted").is(false);
+                where("lockTimestamp").lte(Instant.now().minusSeconds(300).toEpochMilli()).and("status").is(ipResource.getStatus()).and("isDeleted").is(false));
         basicQuery.addCriteria(criteria);
         Update update = new Update();
         update.set("isLocked", true).set("lockTimestamp", Instant.now().toEpochMilli());
@@ -61,11 +61,44 @@ public class IpResourceRepositoryCustomImpl implements IpResourceRepositoryCusto
             return true;
     }
 
+    @Override
+    public boolean healthLock(IpResource ipResource) {
+        Document doc = new Document();
+        BasicQuery basicQuery = new BasicQuery(doc);
+        Criteria criteria = new Criteria();
+        criteria.orOperator(where("id").is(ipResource.getId()).and("isLocked").is(false).and("status").is(ipResource.getStatus()).and("isDeleted").is(false),
+                where("healthLockTimestamp").lte(Instant.now().minusSeconds(300).toEpochMilli()).and("status").is(ipResource.getStatus()).and("isDeleted").is(false));
+        basicQuery.addCriteria(criteria);
+        Update update = new Update();
+        update.set("isLocked", true).set("healthLockTimestamp", Instant.now().toEpochMilli());
+        UpdateResult result = mongoTemplate.updateFirst(basicQuery, update, IpResource.class);
+
+        if (result.getModifiedCount() < 1)
+            return false;
+        else
+            return true;
+    }
+
+    @Override
+    public boolean unLockHealth(String id) {
+        Document doc = new Document();
+        BasicQuery basicQuery = new BasicQuery(doc);
+        basicQuery.addCriteria(where("id").is(id));
+        Update update = new Update();
+        update.set("isLocked", false).set("healthLockTimestamp", Instant.now().toEpochMilli());
+        UpdateResult result = mongoTemplate.updateFirst(basicQuery, update, IpResource.class);
+
+        if (result.getModifiedCount() < 1)
+            return false;
+        else
+            return true;
+    }
+
     @Override
     public List<IpResource> sampleTasks(int status, long timestamp) {
         Criteria matchCriteria  = new Criteria();
         matchCriteria.orOperator(where("status").is(status).and("isLocked").is(false).and("isDeleted").is(false),
-                where("status").is(status).and("isLocked").is(true).and("lockTimestamp").lte(timestamp)).and("isDeleted").is(false);
+                where("status").is(status).and("isLocked").is(true).and("lockTimestamp").lte(timestamp).and("isDeleted").is(false));
 
         MatchOperation match = Aggregation.match(matchCriteria);
 
@@ -79,12 +112,12 @@ public class IpResourceRepositoryCustomImpl implements IpResourceRepositoryCusto
     @Override
     public List<IpResource> sampleTasks(List<Integer> status) {
         Criteria matchCriteria  = new Criteria();
-        matchCriteria.orOperator(where("status").in(status).and("isLocked").is(false).and("isDeleted").is(false).and("lockTimestamp").lte(Instant.now().toEpochMilli()),
-                where("status").in(status).and("isLocked").is(true).and("lockTimestamp").lte((Instant.now().toEpochMilli()) - 60*1000*30)).and("isDeleted").is(false);
+        matchCriteria.orOperator(where("status").in(status).and("isLocked").is(false).and("isDeleted").is(false).and("healthLockTimestamp").lte(Instant.now().minusSeconds(60*30).toEpochMilli()),
+                where("status").in(status).and("isLocked").is(true).and("healthLockTimestamp").lte(Instant.now().minusSeconds(300).toEpochMilli()).and("isDeleted").is(false));
 
         MatchOperation match = Aggregation.match(matchCriteria);
 
-        SampleOperation sample = Aggregation.sample(20);
+        SampleOperation sample = Aggregation.sample(100);
         AggregationResults<IpResource> results = mongoTemplate.aggregate(Aggregation.newAggregation(match, sample), IpResource.class, IpResource.class);
         List<IpResource> mappedResults = results.getMappedResults();
 

src/main/java/com/edgec/browserbackend/browser/service/Impl/IpResourceServiceImpl.java

@@ -198,7 +198,7 @@ public class IpResourceServiceImpl implements IpResourceService {
         Account account = accountRepository.findByName(username);
         if (account == null)
             throw new ClientRequestException(AccountErrorCode.NAMENOTEXIST);
-        if (account.getPermission() < 8)
+        if (account.getPermission() < 4)
             throw new ClientRequestException(AccountErrorCode.NOPERMISSION);
 
         Map<String, List<String>> priceList = ipOptionsRepository.findAll().get(0).getIpPlatForm();
@@ -279,7 +279,10 @@ public class IpResourceServiceImpl implements IpResourceService {
                         break;
                 }
                 ipResource.setStatus(6);
-                ipResource.setValidTime(Instant.now().plusSeconds(3600*24*30).toEpochMilli());
+                if (ipResourceRequestDto.getUnit().equals("week"))
+                    ipResource.setValidTime(Instant.now().atZone(ZoneOffset.UTC).plusWeeks(ipResource.getPeriod()).toInstant().toEpochMilli());
+                else
+                    ipResource.setValidTime(Instant.now().atZone(ZoneOffset.UTC).plusMonths(ipResource.getPeriod()).toInstant().toEpochMilli());
                 ipResource.setUsername(USERNAME);
                 ipResource.setPort(port);
             }

src/main/java/com/edgec/browserbackend/browser/service/Impl/ShopServiceImpl.java

@@ -416,8 +416,6 @@ public class ShopServiceImpl implements ShopService {
         if (shops == null || shops.getNumberOfElements() < 1)
             return new ShopPageResultDto();
         List<ShopResultDto> shopResultDtos = new ArrayList<>();
-        logger.error("shops.size" + shops.getNumberOfElements());
-        logger.error("shops.content.size " + shops.getContent().size());
         shops.getContent().stream().forEach(x -> {
             IpResource ipResource = ipResourceRepository.findFirstByShopIdAndIsDeleted(x.getShopId(), false);
             if (ipResource == null)

src/main/java/com/edgec/browserbackend/browser/task/BrowserTask.java

@@ -219,36 +219,60 @@ public class BrowserTask {
         }
     }
 
-//    @Scheduled(cron = "0 0/1 * * * ?")
+    @Scheduled(cron = "0 0/5 * * * ?")
     public void healthCheck() {
-        String URL = (profiles.equals("dev") || profiles.equals("staging")) ? TESTURL : CLOUDAMURL;
         List<IpResource> ipResources = ipResourceRepository.sampleTasks(Arrays.asList(0, 2));
         for (IpResource ipResource : ipResources) {
             long start = System.currentTimeMillis();
-            CompletableFuture.runAsync(() -> {
-                if (ipResourceRepository.lockTask(ipResource)) {
-                    try {
-                        QueryIpUrlList queryIpUrlList = queryIpUrlListRepository.findAll().get(0);
-                        Trans trans;
-                        if (ipResource.isSpecialLine()) {
-                            trans = new Trans();
+            if (ipResourceRepository.healthLock(ipResource)) {
+                try {
+                    QueryIpUrlList queryIpUrlList = queryIpUrlListRepository.findAll().get(0);
+                    if (ipResource.isSpecialLine()) {
+                        Trans trans = new Trans(ipResource.getProxyUsername(), ipResource.getProxyPassword());
+                        String sp_result = trans.get(queryIpUrlList.getUrl(), true);
+                        int failTime = 0;
+                        while (!sp_result.contains(ipResource.getAddr())) {
+                            if (failTime > 5) {
+                                NotifyUtils.sendMessage("防关联浏览器 ip " + ipResource.getAddr() + " 专线代理异常", NotifyUtils.MsgType.WEBHOOK);
+                                log.error("防关联浏览器 ip " + ipResource.getAddr() + " 专线代理异常 " + sp_result);
+                                break;
+                            }
+                            failTime ++;
+                            Thread.sleep(2000);
+                            sp_result = trans.get(queryIpUrlList.getUrl(), true);
                         }
-
-                    } catch (Exception e) {
-                        log.error(e.getMessage(), e);
-                    } finally {
-                        long end = System.currentTimeMillis();
-                        log.debug("queryIpTask {} execution time is: " + (end - start) / 1000 + "s", ipResource.getId());
-                        try {
-                            ipResourceRepository.unLockTask(ipResource.getId());
-                        } catch (Throwable th) {
-                            log.error("unlock failed", th);
-                            //try again
-                            ipResourceRepository.unLockTask(ipResource.getId());
+                    }
+                    String result;
+                    Trans trans = new Trans(ipResource.getAddr(), Integer.valueOf(ipResource.getPort().size() > 1?ipResource.getPort().get(1):ipResource.getPort().get(0)), ipResource.getUsername(), ipResource.getPassword());
+                    result = trans.get(queryIpUrlList.getUrl(), false);
+                    if (!result.contains(ipResource.getAddr())) {
+                        int failTime = 0;
+                        while (!result.contains(ipResource.getAddr())) {
+                            if (failTime > 5) {
+                                NotifyUtils.sendMessage("防关联浏览器 ip " + ipResource.getAddr() + " 代理异常", NotifyUtils.MsgType.WEBHOOK);
+                                log.error("防关联浏览器 ip " + ipResource.getAddr() + " 代理异常 " + result);
+                                break;
+                            }
+                            failTime ++;
+                            Thread.sleep(2000);
+                            result = trans.get(queryIpUrlList.getUrl(), true);
                         }
                     }
+                } catch (Exception e) {
+                    NotifyUtils.sendMessage("防关联浏览器 ip " + ipResource.getAddr() + " 代理异常", e, NotifyUtils.MsgType.WEBHOOK);
+                    log.error(e.getMessage(), e);
+                } finally {
+                    long end = System.currentTimeMillis();
+                    log.debug("queryIpTask {} execution time is: " + (end - start) / 1000 + "s", ipResource.getId());
+                    try {
+                        ipResourceRepository.unLockHealth(ipResource.getId());
+                    } catch (Throwable th) {
+                        log.error("unlock failed", th);
+                        //try again
+                        ipResourceRepository.unLockHealth(ipResource.getId());
+                    }
                 }
-            }, ThreadPoolUtils.queryIpTasksPool);
+            }
         }
     }
 

src/main/java/com/edgec/browserbackend/browser/task/Set3proxyTask.java

@@ -47,7 +47,6 @@ public class Set3proxyTask {
     @Scheduled(cron = "0 0/10 * * * ?")
     @SchedulerLock(name = "proxyTask", lockAtLeastFor = 60*1000*5, lockAtMostFor = 60*1000*9)
     public void set3proxy() {
-        long validTime = Instant.now().minusSeconds(43200).toEpochMilli();
         long nowtime = Instant.now().toEpochMilli();
 //        List<String> tokenUsernames = mongoOAuth2AccessTokenRepository.findByCreatedAtGreaterThan(validTime).stream().map(MongoOAuth2AccessToken::getUsername).collect(Collectors.toList());
 //        List<String> accountParents = accountRepository.findByNameIn(tokenUsernames).stream().map(x -> x.getParent() == null ? x.getName() : x.getParent()).distinct().collect(Collectors.toList());
@@ -71,6 +70,7 @@ public class Set3proxyTask {
             for (IpResource ipResource : ipResources) {
                 if (StringUtils.isNotBlank(ipResource.getAddr()))
                     bw.write("users \"" + ipResource.getProxyUsername() + ":CL:" + ipResource.getProxyPassword() + "\"\n");
+                ipResource.setUsingSpecialLine(true);
             }
 
             bw.write("\nauth strong\n");
@@ -102,6 +102,7 @@ public class Set3proxyTask {
                 proxyConfigRepository.updateProxy(file, nowtime);
                 file.delete();
             }
+
         } catch (Exception e) {
             log.error("出错了");
             log.error(e.getMessage(), e);

src/main/java/com/edgec/browserbackend/common/utils/ThreadPoolUtils.java

@@ -38,14 +38,14 @@ public abstract class ThreadPoolUtils {
     });
 
 
-    private static final int IPTRANSACTION_POOL_COUNT = 10;
+    private static final int IPTRANSACTION_POOL_COUNT = 20;
 
-    public static final ScheduledExecutorService queryIpTransactionPool = Executors.newScheduledThreadPool(IPTRANSACTION_POOL_COUNT, new ThreadFactory() {
+    public static final ExecutorService queryIpHealth = Executors.newFixedThreadPool(IPTRANSACTION_POOL_COUNT, new ThreadFactory() {
         int count = 1;
 
         @Override
         public Thread newThread(Runnable runnable) {
-            return new Thread(runnable, "browser-ipTransaction-task-" + count++);
+            return new Thread(runnable, "browser-ipHealthCheck-task-" + count++);
         }
     });
 }

src/main/java/com/edgec/browserbackend/common/utils/Trans.java

@@ -34,8 +34,11 @@ public class Trans {
         this.username = username;
     }
 
-    public Trans() {
-
+    public Trans(String username, String password) {
+        this.host = specialHost;
+        this.port = specialPort;
+        this.password = password;
+        this.username = username;
     }
 
     /**
@@ -88,11 +91,12 @@ public class Trans {
         } catch (ClientProtocolException e) {
             NotifyUtils.sendMessage("防关联浏览器 ip " + host + " 代理异常", e, NotifyUtils.MsgType.WEBHOOK);
             logger.error(e.getMessage(), e);
+            return "";
         } catch (IOException e) {
             NotifyUtils.sendMessage("防关联浏览器 ip " + host + " 代理异常", e, NotifyUtils.MsgType.WEBHOOK);
             logger.error(e.getMessage(), e);
+            return "";
         }
-        System.out.println(sb.toString());
         return sb.toString();
     }
 }
\ No newline at end of file