Merge branch 'staging' into 'master'

Staging

See merge request !82

src/main/java/com/edgec/browserbackend/account/service/impl/PaymentServiceImpl.java

@@ -365,9 +365,10 @@ public class PaymentServiceImpl implements PaymentService {
     public String alipayPutPayOrder(String username, int amount, String by) {
 
         Account byName = accountService.findByName(username);
-        if (byName == null)
+        if (byName == null )
             throw new ClientRequestException(AccountErrorCode.NAMENOTEXIST, "account does not exist: " + username);
-
+//        if (byName.getPermission() < 8)
+//            throw new ClientRequestException(AccountErrorCode.NOPERMISSION, "account does not have permission: " + username);
         boolean isVpsClient = true;
 
         UserPayment internalOrder = new UserPayment();
@@ -420,6 +421,8 @@ public class PaymentServiceImpl implements PaymentService {
         Account byName = accountService.findByName(username);
         if (byName == null)
             throw new ClientRequestException(AccountErrorCode.NAMENOTEXIST, "account does not exist: " + username);
+//        if (byName.getPermission() < 8)
+//            throw new ClientRequestException(AccountErrorCode.NOPERMISSION, "account does not have permission: " + username);
 
         boolean isVpsClient = true;
 

src/main/java/com/edgec/browserbackend/browser/repository/IpResourceRepository.java

@@ -37,7 +37,7 @@ public interface IpResourceRepository extends MongoRepository<IpResource, String
 
     List<IpResource> findByValidTimeBetweenAndIsDeleted(long beginTime, long endTime, boolean isDeleted);
 
-    IpResource findByShopIdAndIsDeleted(String shopId, boolean isDeleted);
+    IpResource findFirstByShopIdAndIsDeleted(String shopId, boolean isDeleted);
 
     int countByStatusAndIdInAndIsDeleted(int status, List<String> ipIds, boolean isDeleted);
 

src/main/java/com/edgec/browserbackend/browser/repository/IpResourceRepositoryCustom.java

@@ -11,4 +11,6 @@ public interface IpResourceRepositoryCustom {
     boolean unLockTask(String id);
 
     List<IpResource> sampleTasks(int status, long timestamp);
+
+    List<IpResource> sampleTasks(List<Integer> status);
 }

src/main/java/com/edgec/browserbackend/browser/repository/IpResourceRepositoryCustomImpl.java

@@ -65,7 +65,22 @@ public class IpResourceRepositoryCustomImpl implements IpResourceRepositoryCusto
     public List<IpResource> sampleTasks(int status, long timestamp) {
         Criteria matchCriteria  = new Criteria();
         matchCriteria.orOperator(where("status").is(status).and("isLocked").is(false).and("isDeleted").is(false),
-                where("isLocked").is(true).and("lockTimestamp").lte(timestamp)).and("isDeleted").is(false);
+                where("status").is(status).and("isLocked").is(true).and("lockTimestamp").lte(timestamp)).and("isDeleted").is(false);
+
+        MatchOperation match = Aggregation.match(matchCriteria);
+
+        SampleOperation sample = Aggregation.sample(20);
+        AggregationResults<IpResource> results = mongoTemplate.aggregate(Aggregation.newAggregation(match, sample), IpResource.class, IpResource.class);
+        List<IpResource> mappedResults = results.getMappedResults();
+
+        return mappedResults;
+    }
+
+    @Override
+    public List<IpResource> sampleTasks(List<Integer> status) {
+        Criteria matchCriteria  = new Criteria();
+        matchCriteria.orOperator(where("status").in(status).and("isLocked").is(false).and("isDeleted").is(false).and("lockTimestamp").lte(Instant.now().toEpochMilli()),
+                where("status").in(status).and("isLocked").is(true).and("lockTimestamp").lte((Instant.now().toEpochMilli()) - 60*1000*30)).and("isDeleted").is(false);
 
         MatchOperation match = Aggregation.match(matchCriteria);
 

src/main/java/com/edgec/browserbackend/browser/service/Impl/IpAndShopServiceImpl.java

@@ -65,7 +65,7 @@ public class IpAndShopServiceImpl implements IpAndShopService {
         if (ipResource == null)
             throw new ClientRequestException(BrowserErrorCode.IPNOTEXIST);
 
-        IpResource bind = ipResourceRepository.findByShopIdAndIsDeleted(shopId, false);
+        IpResource bind = ipResourceRepository.findFirstByShopIdAndIsDeleted(shopId, false);
         if (bind != null) {
             shopRequestDto.setShopId(shopId);
             shopRequestDto.setIpId(ipResource.getId());

src/main/java/com/edgec/browserbackend/browser/service/Impl/ShopServiceImpl.java

@@ -100,9 +100,11 @@ public class ShopServiceImpl implements ShopService {
             }
             userShopRepository.save(userShop);
             if (StringUtils.isNotBlank(account.getParent())) {
-                userShop.setUsername(account.getParent());
-                userShop.setGroupId("-1");
-                userShopRepository.save(userShop);
+                UserShop userShop1 = new UserShop();
+                userShop1.setUsername(account.getParent());
+                userShop1.setShopId(id);
+                userShop1.setGroupId("-1");
+                userShopRepository.save(userShop1);
             }
             //可以优化
             account.setShopCount(account.getShopCount() + 1);
@@ -161,9 +163,11 @@ public class ShopServiceImpl implements ShopService {
                 }
                 userShopRepository.save(userShop);
                 if (StringUtils.isNotBlank(account.getParent())) {
-                    userShop.setUsername(account.getParent());
-                    userShop.setGroupId("-1");
-                    userShopRepository.save(userShop);
+                    UserShop userShop1 = new UserShop();
+                    userShop1.setUsername(account.getParent());
+                    userShop1.setShopId(id);
+                    userShop1.setGroupId("-1");
+                    userShopRepository.save(userShop1);
                 }
                 //可以优化
                 account.setShopCount(account.getShopCount() + 1);
@@ -219,7 +223,7 @@ public class ShopServiceImpl implements ShopService {
         Shop shop = shopRepository.findById(shopId).orElse(null);
         if (shop == null)
             throw new ClientRequestException(BrowserErrorCode.SHOPNOTEXIST);
-        IpResource ipResource = ipResourceRepository.findByShopIdAndIsDeleted(shop.getShopId(), false);
+        IpResource ipResource = ipResourceRepository.findFirstByShopIdAndIsDeleted(shop.getShopId(), false);
         if (ipResource != null) {
             ipResource.setShopName(null);
             ipResource.setShopId(null);
@@ -326,7 +330,7 @@ public class ShopServiceImpl implements ShopService {
         if (shop == null) {
             throw new ClientRequestException(BrowserErrorCode.SHOPNOTEXIST);
         }
-        IpResource ipResource = ipResourceRepository.findByShopIdAndIsDeleted(shopId, false);
+        IpResource ipResource = ipResourceRepository.findFirstByShopIdAndIsDeleted(shopId, false);
         if (ipResource == null)
             throw new ClientRequestException(BrowserErrorCode.IPNOTEXIST);
         String group = userShopRepository.findByUsernameAndShopId(username, shop.getShopId()).getGroupId();
@@ -372,7 +376,7 @@ public class ShopServiceImpl implements ShopService {
                 List<String> allIds = userShopRepository.findByUsername(username).stream()
                         .map(x -> x.getShopId()).collect(Collectors.toList());
                 for (String id:allIds) {
-                    IpResource ipResource = ipResourceRepository.findByShopIdAndIsDeleted(id, false);
+                    IpResource ipResource = ipResourceRepository.findFirstByShopIdAndIsDeleted(id, false);
                     if (ipResource == null)
                         shopIds.add(id);
                 }
@@ -391,7 +395,7 @@ public class ShopServiceImpl implements ShopService {
                 List<String> allIds = userShopRepository.findByUsernameAndGroupId(username, groupId).stream()
                         .map(x -> x.getShopId()).collect(Collectors.toList());
                 for (String id:allIds) {
-                    IpResource ipResource = ipResourceRepository.findByShopIdAndIsDeleted(id, false);
+                    IpResource ipResource = ipResourceRepository.findFirstByShopIdAndIsDeleted(id, false);
                     if (ipResource == null)
                         shopIds.add(id);
                 }
@@ -415,7 +419,7 @@ public class ShopServiceImpl implements ShopService {
         logger.error("shops.size" + shops.getNumberOfElements());
         logger.error("shops.content.size " + shops.getContent().size());
         shops.getContent().stream().forEach(x -> {
-            IpResource ipResource = ipResourceRepository.findByShopIdAndIsDeleted(x.getShopId(), false);
+            IpResource ipResource = ipResourceRepository.findFirstByShopIdAndIsDeleted(x.getShopId(), false);
             if (ipResource == null)
                 ipResource = new IpResource();
             String group1 = userShopRepository.findByUsernameAndShopId(username, x.getShopId()).getGroupId();
@@ -449,7 +453,7 @@ public class ShopServiceImpl implements ShopService {
         List<String> unbind = new ArrayList<>();
         if (allShopIds != null && allShopIds.size() > 0) {
             for (String id:allShopIds) {
-                IpResource ipResource = ipResourceRepository.findByShopIdAndIsDeleted(id, false);
+                IpResource ipResource = ipResourceRepository.findFirstByShopIdAndIsDeleted(id, false);
                 if (ipResource == null) {
                     unbind.add(id);
                 }

src/main/java/com/edgec/browserbackend/browser/task/BrowserTask.java

@@ -2,6 +2,8 @@ package com.edgec.browserbackend.browser.task;
 
 import com.alibaba.fastjson.JSONObject;
 import com.edgec.browserbackend.account.domain.IpChargeRequestDto;
+import com.edgec.browserbackend.account.domain.QueryIpUrlList;
+import com.edgec.browserbackend.account.repository.QueryIpUrlListRepository;
 import com.edgec.browserbackend.account.service.AccountService;
 import com.edgec.browserbackend.browser.domain.IpResource;
 import com.edgec.browserbackend.browser.dto.IpBuyResultDto;
@@ -11,6 +13,7 @@ import com.edgec.browserbackend.browser.repository.IpResourceRepository;
 import com.edgec.browserbackend.browser.service.IpAndShopService;
 import com.edgec.browserbackend.common.commons.utils.NotifyUtils;
 import com.edgec.browserbackend.common.utils.ThreadPoolUtils;
+import com.edgec.browserbackend.common.utils.Trans;
 import okhttp3.*;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -26,6 +29,7 @@ import org.springframework.web.client.RestTemplate;
 
 import java.io.IOException;
 import java.time.Instant;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -54,6 +58,9 @@ public class BrowserTask {
     @Autowired
     private IpAndShopService ipAndShopService;
 
+    @Autowired
+    private QueryIpUrlListRepository queryIpUrlListRepository;
+
     @Value("${spring.profiles.active}")
     private String profiles;
 
@@ -212,6 +219,39 @@ public class BrowserTask {
         }
     }
 
+//    @Scheduled(cron = "0 0/1 * * * ?")
+    public void healthCheck() {
+        String URL = (profiles.equals("dev") || profiles.equals("staging")) ? TESTURL : CLOUDAMURL;
+        List<IpResource> ipResources = ipResourceRepository.sampleTasks(Arrays.asList(0, 2));
+        for (IpResource ipResource : ipResources) {
+            long start = System.currentTimeMillis();
+            CompletableFuture.runAsync(() -> {
+                if (ipResourceRepository.lockTask(ipResource)) {
+                    try {
+                        QueryIpUrlList queryIpUrlList = queryIpUrlListRepository.findAll().get(0);
+                        Trans trans;
+                        if (ipResource.isSpecialLine()) {
+                            trans = new Trans();
+                        }
+
+                    } catch (Exception e) {
+                        log.error(e.getMessage(), e);
+                    } finally {
+                        long end = System.currentTimeMillis();
+                        log.debug("queryIpTask {} execution time is: " + (end - start) / 1000 + "s", ipResource.getId());
+                        try {
+                            ipResourceRepository.unLockTask(ipResource.getId());
+                        } catch (Throwable th) {
+                            log.error("unlock failed", th);
+                            //try again
+                            ipResourceRepository.unLockTask(ipResource.getId());
+                        }
+                    }
+                }
+            }, ThreadPoolUtils.queryIpTasksPool);
+        }
+    }
+
 
     public static class HttpClientutils {
         static OkHttpClient.Builder builder = new OkHttpClient.Builder();

src/main/java/com/edgec/browserbackend/common/utils/Trans.java

+package com.edgec.browserbackend.common.utils;
+import com.edgec.browserbackend.common.commons.utils.NotifyUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.conn.params.ConnRouteParams;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+public class Trans {
+    private final Logger logger = LoggerFactory.getLogger(Trans.class);
+    static int timeout = 10 * 1000;// 以秒为单位
+    private static String specialHost = "39.108.183.128";//117.177.243.7
+    private static int specialPort = 20008;
+    private String host;
+    private int port;
+    private String username;
+    private String password;
+
+    public Trans(String host, int port, String username, String password) {
+        this.host = host;
+        this.port = port;
+        this.password = password;
+        this.username = username;
+    }
+
+    public Trans() {
+
+    }
+
+    /**
+     * 主入口方法
+     * @param args
+     */
+    public static void main(String[] args) {
+        String url = "http://pv.sohu.com/cityjson";
+        Trans trans = new Trans("120.76.135.212", 20001, "fangguanlianbrowser", "2rbvtg9GQ1JraA1");
+        String result = trans.get(url, false);
+    }
+
+    public HttpClient getHttpClient() {
+        DefaultHttpClient httpClient = new DefaultHttpClient();
+        httpClient.getCredentialsProvider().setCredentials(
+                new AuthScope(host, port),
+                new UsernamePasswordCredentials(username, password));
+        HttpHost proxy = new HttpHost(host,port);
+        httpClient.getParams().setParameter(ConnRouteParams.DEFAULT_PROXY, proxy);
+        return httpClient;
+    }
+
+    /**
+     * 向指定URL发送GET方法的请求
+     */
+    public String get(String url, boolean special) {
+        StringBuffer sb = new StringBuffer();
+        //创建HttpClient实例
+        HttpClient client = this.getHttpClient();
+        //创建httpGet
+        HttpGet httpGet = new HttpGet("http://pv.sohu.com/cityjson");
+        //执行
+        try {
+            HttpResponse response = client.execute(httpGet);
+
+            HttpEntity entry = response.getEntity();
+
+            if(entry != null)
+            {
+                InputStreamReader is = new InputStreamReader(entry.getContent());
+                BufferedReader br = new BufferedReader(is);
+                String str = null;
+                while((str = br.readLine()) != null)
+                {
+                    sb.append(str.trim());
+                }
+                br.close();
+            }
+
+        } catch (ClientProtocolException e) {
+            NotifyUtils.sendMessage("防关联浏览器 ip " + host + " 代理异常", e, NotifyUtils.MsgType.WEBHOOK);
+            logger.error(e.getMessage(), e);
+        } catch (IOException e) {
+            NotifyUtils.sendMessage("防关联浏览器 ip " + host + " 代理异常", e, NotifyUtils.MsgType.WEBHOOK);
+            logger.error(e.getMessage(), e);
+        }
+        System.out.println(sb.toString());
+        return sb.toString();
+    }
+}
\ No newline at end of file