Merge branch 'master' into 'staging'

Master

See merge request !145

src/main/java/com/edgec/browserbackend/account/domain/QueryIpUrlList.java

@@ -9,6 +9,14 @@ public class QueryIpUrlList {
     private String id;
     private String url;
 
+    public QueryIpUrlList() {
+    }
+
+    public QueryIpUrlList(String id, String url) {
+        this.id = id;
+        this.url = url;
+    }
+
     public String getId() {
         return id;
     }
@@ -24,4 +32,6 @@ public class QueryIpUrlList {
     public void setUrl(String url) {
         this.url = url;
     }
+
+
 }

src/main/java/com/edgec/browserbackend/auth/service/MongoTokenStore.java

@@ -4,13 +4,11 @@ import com.edgec.browserbackend.auth.domain.mongo.MongoOAuth2AccessToken;
 import com.edgec.browserbackend.auth.domain.mongo.MongoOAuth2RefreshToken;
 import com.edgec.browserbackend.auth.repository.mongo.MongoOAuth2AccessTokenRepository;
 import com.edgec.browserbackend.auth.repository.mongo.MongoOAuth2RefreshTokenRepository;
-import org.springframework.context.annotation.Bean;
 import org.springframework.security.oauth2.common.OAuth2AccessToken;
 import org.springframework.security.oauth2.common.OAuth2RefreshToken;
 import org.springframework.security.oauth2.common.util.SerializationUtils;
 import org.springframework.security.oauth2.provider.OAuth2Authentication;
 import org.springframework.security.oauth2.provider.token.AuthenticationKeyGenerator;
-import org.springframework.security.oauth2.provider.token.DefaultAuthenticationKeyGenerator;
 import org.springframework.security.oauth2.provider.token.TokenStore;
 import org.springframework.stereotype.Component;
 
@@ -27,6 +25,7 @@ import static java.util.Objects.nonNull;
 import static org.springframework.security.oauth2.common.util.SerializationUtils.deserialize;
 import static org.springframework.security.oauth2.common.util.SerializationUtils.serialize;
 
+
 @Component
 public class MongoTokenStore implements TokenStore {
 

src/main/java/com/edgec/browserbackend/browser/service/Impl/IpResourceServiceImpl.java

@@ -14,13 +14,9 @@ import com.edgec.browserbackend.browser.dto.*;
 import com.edgec.browserbackend.browser.repository.*;
 import com.edgec.browserbackend.browser.service.IpAndShopService;
 import com.edgec.browserbackend.browser.service.IpResourceService;
-import com.edgec.browserbackend.browser.service.ShopService;
 import com.edgec.browserbackend.common.commons.error.ClientRequestException;
 import com.edgec.browserbackend.common.commons.utils.NotifyUtils;
 import com.edgec.browserbackend.common.utils.FileUtil;
-import com.edgec.browserbackend.common.utils.PollerUtils;
-import com.edgec.browserbackend.common.utils.ThreadPoolUtils;
-import com.sun.org.apache.bcel.internal.generic.IF_ACMPEQ;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,9 +34,6 @@ import java.io.File;
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 

src/main/java/com/edgec/browserbackend/browser/service/Impl/ShopServiceImpl.java

 package com.edgec.browserbackend.browser.service.Impl;
 
 import com.edgec.browserbackend.account.domain.Account;
-import com.edgec.browserbackend.account.domain.User;
 import com.edgec.browserbackend.account.exception.AccountErrorCode;
 import com.edgec.browserbackend.account.repository.AccountRepository;
 import com.edgec.browserbackend.browser.ErrorCode.BrowserErrorCode;
@@ -14,7 +13,6 @@ import com.edgec.browserbackend.browser.service.ShopService;
 import com.edgec.browserbackend.common.commons.error.ClientRequestException;
 import com.edgec.browserbackend.common.utils.FileUtil;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.poi.ss.usermodel.Workbook;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -22,18 +20,12 @@ import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageImpl;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
-import org.springframework.security.core.parameters.P;
-import org.springframework.security.task.DelegatingSecurityContextAsyncTaskExecutor;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.IOException;
-import java.text.SimpleDateFormat;
 import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;

src/main/java/com/edgec/browserbackend/browser/task/BrowserTask.java

@@ -20,12 +20,12 @@ import com.edgec.browserbackend.common.commons.utils.NotifyUtils;
 import com.edgec.browserbackend.common.utils.ThreadPoolUtils;
 import com.edgec.browserbackend.common.utils.Trans;
 import okhttp3.*;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.data.mongodb.core.aggregation.ArrayOperators;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
@@ -35,10 +35,7 @@ import org.springframework.web.client.RestTemplate;
 
 import java.io.IOException;
 import java.time.Instant;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -241,40 +238,43 @@ public class BrowserTask {
             long start = System.currentTimeMillis();
             if (ipResourceRepository.healthLock(ipResource)) {
                 try {
-                    QueryIpUrlList queryIpUrlList = queryIpUrlListRepository.findAll().get(0);
+                    int failTime = 0;
+                    List<QueryIpUrlList> queryIpUrlLists = queryIpUrlListRepository.findAll();
+                    final int maxRetry = Math.max(queryIpUrlLists.size(), 5);
                     if (ipResource.getValidTime() <= Instant.now().toEpochMilli() && ipResource.getStatus() != 3 && ipResource.getStatus() != 6) {
                         ipResource.setStatus(1);
                         ipResourceRepository.save(ipResource);
-                    }
-                    else if (ipResource.isSpecialLine()) {
+                    } else if (ipResource.isSpecialLine() || (ipResource.getIpType() == IpType.OWN && ipResource.isUsingSpecialLine())) {
+
                         Trans trans = new Trans(ipResource.getProxyUsername(), ipResource.getProxyPassword());
-                        String sp_result = trans.get(queryIpUrlList.getUrl());
-                        int failTime = 0;
+                        String sp_result = trans.get(getNextUrl(queryIpUrlLists, failTime).getUrl());
                         while (!sp_result.contains(ipResource.getAddr())) {
-                            if (failTime > 5) {
-                                NotifyUtils.sendMessage("防关联浏览器 ip " + ipResource.getAddr() + " 专线代理异常", NotifyUtils.MsgType.WEBHOOK);
+                            if (failTime > maxRetry) {
+                                NotifyUtils.sendMessage("防关联浏览器 ip " + ipResource.getAddr() + " 专线代理异常:" + ipResource.getIpType(),
+                                        NotifyUtils.MsgType.WEBHOOK);
                                 log.error("防关联浏览器 ip " + ipResource.getAddr() + " 专线代理异常 " + sp_result);
                                 break;
                             }
-                            failTime ++;
+                            failTime++;
                             Thread.sleep(2000);
-                            sp_result = trans.get(queryIpUrlList.getUrl());
+                            sp_result = trans.get(getNextUrl(queryIpUrlLists, failTime).getUrl());
                         }
-                    } else if (ipResource.getIpType() == IpType.VENDOR){
+                    } else if (ipResource.getIpType() == IpType.VENDOR
+                            || (ipResource.getIpType() == IpType.OWN && !ipResource.isUsingSpecialLine())) {
                         String result;
-                        Trans trans = new Trans(ipResource.getAddr(), Integer.valueOf(ipResource.getPort().size() > 1?ipResource.getPort().get(1):ipResource.getPort().get(0)), ipResource.getUsername(), ipResource.getPassword());
-                        result = trans.get(queryIpUrlList.getUrl());
+                        Trans trans = new Trans(ipResource.getAddr(), Integer.valueOf(ipResource.getPort().size() > 1 ? ipResource.getPort().get(1) : ipResource.getPort().get(0)), ipResource.getUsername(), ipResource.getPassword());
+                        result = trans.get(getNextUrl(queryIpUrlLists, failTime).getUrl());
                         if (!result.contains(ipResource.getAddr())) {
-                            int failTime = 0;
                             while (!result.contains(ipResource.getAddr())) {
-                                if (failTime > 5) {
-                                    NotifyUtils.sendMessage("防关联浏览器 ip " + ipResource.getAddr() + " 代理异常", NotifyUtils.MsgType.WEBHOOK);
+                                if (failTime > maxRetry) {
+                                    NotifyUtils.sendMessage("防关联浏览器 ip " + ipResource.getAddr() + " 代理异常:" + ipResource.getIpType(),
+                                            NotifyUtils.MsgType.WEBHOOK);
                                     log.error("防关联浏览器 ip " + ipResource.getAddr() + " 代理异常 " + result);
                                     break;
                                 }
-                                failTime ++;
+                                failTime++;
                                 Thread.sleep(2000);
-                                result = trans.get(queryIpUrlList.getUrl());
+                                result = trans.get(getNextUrl(queryIpUrlLists, failTime).getUrl());
                             }
                         }
                     }
@@ -296,6 +296,17 @@ public class BrowserTask {
         }
     }
 
+
+    private static QueryIpUrlList getNextUrl(List<QueryIpUrlList> queryIpUrlLists, int count) {
+        if (CollectionUtils.isEmpty(queryIpUrlLists)) {
+            return null;
+        }
+        int size = queryIpUrlLists.size();
+        int index = count % size;
+        return queryIpUrlLists.get(index);
+    }
+
+
     @Scheduled(cron = "0 0 0 * * ?")
     public void countIp() {
         List<Account> accounts = accountRepository.findAll();
@@ -351,23 +362,13 @@ public class BrowserTask {
 
 
     public static void main(String[] args) throws IOException {
-//        String url = CLOUDAMURL + "/ecc/ipinfo?accountId=browser&ip=" + "100.25.101.201";
-//        String profiles = "prod";
-//        Map<String, String> headers = new HashMap<>();
-//        headers.put("Content-Type", "application/json");
-//        if (profiles.equals("dev") || profiles.equals("staging"))
-//            headers.put("Authorization", "Bearer oq5tg3gMsflHcK5iZ2741G5R30XYd9blyOqH9qeBItKtrzfTsGIoy8AsxqqNXdcm");
-//        else if (profiles.equals("prod"))
-//            headers.put("Authorization", "Bearer tKWsuHzcngf0RQPMss70f9jgymDIwgQ9zbLfESJdcou3pZSNWl7lNTzto8VQgwaO");
-//
-//        String rs = HttpClientutils.doGet(url, headers);
-//        System.out.println(rs);
-//        IpInfoResultDto ipInfoResultDto = JSONObject.parseObject(rs, IpInfoResultDto.class);
-//        System.out.println(ipInfoResultDto);
-
-        Trans trans = new Trans("8.208.28.192", "962MrDCCd2pvaA1");
-        String sp_result = trans.get("http://pv.sohu.com/cityjson");
-        System.out.println(sp_result);
+        List<QueryIpUrlList> queryIpUrlLists = new ArrayList<>();
+        queryIpUrlLists.add(new QueryIpUrlList("1", "baidu"));
+        queryIpUrlLists.add(new QueryIpUrlList("2", "google"));
+        for (int i = 0; i < 10; i++) {
+            System.out.println(getNextUrl(queryIpUrlLists, i).getUrl());
+        }
+
     }
 
 

src/main/java/com/edgec/browserbackend/browser/task/Set3proxyTask.java

 package com.edgec.browserbackend.browser.task;
 
 import com.edgec.browserbackend.account.repository.AccountRepository;
-import com.edgec.browserbackend.account.repository.CompanyAuthorizeRepository;
-import com.edgec.browserbackend.auth.domain.mongo.MongoOAuth2AccessToken;
 import com.edgec.browserbackend.auth.repository.mongo.MongoOAuth2AccessTokenRepository;
 import com.edgec.browserbackend.browser.domain.IpResource;
+import com.edgec.browserbackend.browser.domain.IpType;
 import com.edgec.browserbackend.browser.domain.ProxyConfig;
-import com.edgec.browserbackend.browser.domain.SpecialLine;
 import com.edgec.browserbackend.browser.repository.IpResourceRepository;
 import com.edgec.browserbackend.browser.repository.ProxyConfigRepository;
 import com.edgec.browserbackend.browser.repository.SpecialLineRepository;
 import net.javacrumbs.shedlock.core.SchedulerLock;
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -19,10 +16,11 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
-import java.io.*;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
 import java.time.Instant;
 import java.util.List;
-import java.util.stream.Collectors;
 
 @Component
 public class Set3proxyTask {
@@ -45,7 +43,7 @@ public class Set3proxyTask {
     private ProxyConfigRepository proxyConfigRepository;
 
     @Scheduled(cron = "0 0/10 * * * ?")
-    @SchedulerLock(name = "proxyTask", lockAtLeastFor = 60*1000*5, lockAtMostFor = 60*1000*9)
+    @SchedulerLock(name = "proxyTask", lockAtLeastFor = 60 * 1000 * 5, lockAtMostFor = 60 * 1000 * 9)
     public void set3proxy() {
         long nowtime = Instant.now().toEpochMilli();
 //        List<String> tokenUsernames = mongoOAuth2AccessTokenRepository.findByCreatedAtGreaterThan(validTime).stream().map(MongoOAuth2AccessToken::getUsername).collect(Collectors.toList());
@@ -61,7 +59,7 @@ public class Set3proxyTask {
                 log.error(t1.getMessage(), t1);
             }
 
-            FileWriter fileWriter =new FileWriter(file, true);
+            FileWriter fileWriter = new FileWriter(file, true);
             BufferedWriter bw = new BufferedWriter(fileWriter);
             bw.write("daemon\nlog /var/log/3proxy.log D\nrotate 30\n");
             bw.write("maxconn 10000\n");
@@ -75,14 +73,21 @@ public class Set3proxyTask {
             }
 
             bw.write("\nauth strong\n");
-           a: for (IpResource ipResource : ipResources) {
+            a:
+            for (IpResource ipResource : ipResources) {
                 if (StringUtils.isNotBlank(ipResource.getAddr())) {
                     if (ipResource.getPort() == null || ipResource.getPort().size() == 0)
                         continue a;
 
                     bw.write("allow " + ipResource.getProxyUsername() + "\n");
-                    bw.write("parent 1000 http " + ipResource.getAddr() + " " +
-                            (ipResource.getPort().size() > 1?ipResource.getPort().get(1):ipResource.getPort().get(0)) + " fangguanlianbrowser " + ipResource.getPassword() + "\n");
+                    if (ipResource.getIpType() != IpType.OWN) {
+                        bw.write("parent 1000 http " + ipResource.getAddr() + " " +
+                                (ipResource.getPort().size() > 1 ? ipResource.getPort().get(1) : ipResource.getPort().get(0)) + " fangguanlianbrowser " + ipResource.getPassword() + "\n");
+                    } else {
+                        bw.write("parent 1000 http " + ipResource.getAddr() + " " +
+                                (ipResource.getPort().size() > 1 ? ipResource.getPort().get(1) : ipResource.getPort().get(0)) + " " + ipResource.getUsername() + " " + ipResource.getPassword() + "\n");
+
+                    }
                 }
             }
             bw.write("\nallow none\nproxy -p20004\nsocks -p20005\n");

src/main/java/com/edgec/browserbackend/common/utils/Aes.java

 package com.edgec.browserbackend.common.utils;
 
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
 import org.springframework.util.StringUtils;
 import sun.misc.BASE64Decoder;
 
-import javax.crypto.*;
-import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.Cipher;
+import javax.crypto.KeyGenerator;
 import javax.crypto.spec.SecretKeySpec;
-import java.io.UnsupportedEncodingException;
 import java.math.BigInteger;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.Security;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.KeySpec;
 import java.util.Base64;
 
+
 public class Aes {
     //密钥 (需要前端和后端保持一致)
     private static final String KEY = "iefWFOAjfwefnWEI";

src/main/java/com/edgec/browserbackend/common/utils/Trans.java

 package com.edgec.browserbackend.common.utils;
 
-import com.edgec.browserbackend.common.commons.utils.NotifyUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
@@ -8,11 +7,11 @@ import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.HttpClient;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.conn.params.ConnRouteParams;
-import org.apache.http.impl.client.*;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -46,11 +45,14 @@ public class Trans {
 
     /**
      * 主入口方法
+     *
      * @param args
      */
     public static void main(String[] args) {
-        String url = "http://pv.sohu.com/cityjson";
-        Trans trans = new Trans("120.76.135.212", 20001, "fangguanlianbrowser", "2rbvtg9GQ1JraA1");
+        String url = "http://free.ipwhois.io/json/";
+        Trans trans = new Trans("23.105.2.118", "uE5LkzM81UhUaA1");
+        String rs = trans.get(url);
+        System.out.println(rs);
     }
 
     /**
@@ -58,7 +60,7 @@ public class Trans {
      */
     public String get(String url) {
         StringBuffer sb = new StringBuffer();
-        HttpHost proxy = new HttpHost(host,port);
+        HttpHost proxy = new HttpHost(host, port);
         // 设置认证
         CredentialsProvider provider = new BasicCredentialsProvider();
 
@@ -102,8 +104,7 @@ public class Trans {
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
             return "";
-        }
-        finally {
+        } finally {
             httpGet.releaseConnection();
         }
     }