Skip to content

B
browser-backend
Commit 9eca9001 authored by SN150021's avatar SN150021
Browse files
Options

定时任务优化

parent 631af91e
Show whitespace changes
Inline Side-by-side
Showing with 278 additions and 31 deletions
src/main/java/com/edgec/browserbackend/browser/domain/CloudPlatformOrder.java
View file @ 9eca9001
......@@ -23,8 +23,14 @@ public class CloudPlatformOrder {
@Id
private String id;
private String ipResourceId;
private String owner;
private String platformOrderId;
private String ipResourceId;//ip资源id
private String regionId;
private String platformOrderId;//云平台订单id
private String platformType;//云平台类型
}
src/main/java/com/edgec/browserbackend/browser/repository/CloudPlatformOrderRepository.java 0 → 100644
View file @ 9eca9001
package com.edgec.browserbackend.browser.repository;
import com.edgec.browserbackend.browser.domain.CloudPlatformOrder;
import com.edgec.browserbackend.browser.domain.IpResource;
import java.util.List;
import org.springframework.data.mongodb.repository.MongoRepository;
/**
* @Description //云平台订单表
* @Date 2024/7/30 16:38
* @Author SN150021
**/
public interface CloudPlatformOrderRepository extends MongoRepository<CloudPlatformOrder, String> {
List<CloudPlatformOrder> findByOwnerAndIpResourceId(String owner, String ipResourceId);
}
src/main/java/com/edgec/browserbackend/browser/task/BrowserTask.java
View file @ 9eca9001
......@@ -2,12 +2,17 @@ package com.edgec.browserbackend.browser.task;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.aliyun.sdk.service.ecs20140526.models.DeleteInstanceResponse;
import com.aliyun.sdk.service.ecs20140526.models.DescribeInstancesResponse;
import com.aliyun.sdk.service.ecs20140526.models.DescribeInstancesResponseBody;
import com.aliyun.sdk.service.ecs20140526.models.RunInstancesResponse;
import com.edgec.browserbackend.account.domain.Account;
import com.edgec.browserbackend.account.domain.IpChargeRequestDto;
import com.edgec.browserbackend.account.domain.QueryIpUrlList;
import com.edgec.browserbackend.account.repository.AccountRepository;
import com.edgec.browserbackend.account.repository.QueryIpUrlListRepository;
import com.edgec.browserbackend.account.service.AccountService;
import com.edgec.browserbackend.browser.domain.CloudPlatformOrder;
import com.edgec.browserbackend.browser.domain.IpCountRecord;
import com.edgec.browserbackend.browser.domain.IpResource;
import com.edgec.browserbackend.browser.domain.IpType;
......@@ -15,19 +20,25 @@ import com.edgec.browserbackend.browser.dto.DeleteIpResultDto;
import com.edgec.browserbackend.browser.dto.IpBuyResultDto;
import com.edgec.browserbackend.browser.dto.IpInfoResultDto;
import com.edgec.browserbackend.browser.dto.ShopRequestDto;
import com.edgec.browserbackend.browser.repository.CloudPlatformOrderRepository;
import com.edgec.browserbackend.browser.repository.IpCountRecordRepository;
import com.edgec.browserbackend.browser.repository.IpResourceRepository;
import com.edgec.browserbackend.browser.service.IpAndShopService;
import com.edgec.browserbackend.common.client.AliEcsClient;
import com.edgec.browserbackend.common.commons.utils.NotifyUtils;
import com.edgec.browserbackend.common.enums.RegionMappingEnum;
import com.edgec.browserbackend.common.utils.ThreadPoolUtils;
import com.edgec.browserbackend.common.utils.Trans;
import javax.management.ObjectName;
import okhttp3.*;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.formula.DataValidationEvaluator.ValidationEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.cloud.CloudPlatform;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Scheduled;
......@@ -73,6 +84,9 @@ public class BrowserTask {
@Value("${spring.profiles.active}")
private String profiles;
@Autowired
private CloudPlatformOrderRepository cloudPlatformOrderRepository;
public Map<String, String> buildGetHeader() {
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json");
......@@ -104,11 +118,168 @@ public class BrowserTask {
return ipChargeRequestDto;
}
/**
* 购买ip的定时任务，每分钟一次(新逻辑
*/
//@Scheduled(cron = "0 0/1 * * * ?")
public void buyIpNewTasks() {
String logs = "【BrowserTask -- buyIpNewTasks】 ";
log.info("{}, ****************ip resource purchase start****************", logs);
long time = Instant.now().minusSeconds(300).toEpochMilli();
List<IpResource> ipResources = ipResourceRepository.sampleTasks(6, time);
log.info("{}, ip resource purchase tasks : {}", logs, ipResources.size());
List<CompletableFuture> futureList = new ArrayList<>();
for (IpResource ipResource : ipResources) {
long start = System.currentTimeMillis();
CompletableFuture future = CompletableFuture.runAsync(() -> {
if (ipResourceRepository.lockTask(ipResource)) {
log.info("{}, ip resource purchase : {}", logs, ipResource.getId());
try {
RegionMappingEnum byRegion = RegionMappingEnum.getByRegion(ipResource.getRegionCn());
if(Objects.isNull(byRegion)){
log.error("buyIpNewTasks-购买ip error：查询不到地区映射-region:{}",ipResource.getRegion());
return;
}
//判断买的哪个地域的ip
CloudPlatformOrder cloudPlatformOrder = new CloudPlatformOrder();
cloudPlatformOrder.setOwner(ipResource.getOwner());
cloudPlatformOrder.setIpResourceId(ipResource.getId());
boolean result = false;
if(ipResource.getVendorCn().equals("阿里云")){
//获取阿里云服务器region映射
String aliResion = null;
if(ipResource.getRegion().equals("us")){
RegionMappingEnum regionMappingEnum = RegionMappingEnum.getUsRandom();
aliResion = regionMappingEnum.getAliRegionApi();
}else if(ipResource.getRegion().equals("mainland")){
RegionMappingEnum regionMappingEnum = RegionMappingEnum.getCnRandom();
aliResion = regionMappingEnum.getAliRegionApi();
}else {
aliResion = byRegion.getAliRegionApi();
}
if(StringUtils.isEmpty(aliResion)){
log.error("buyIpNewTasks-购买ip error：查询不到阿里云映射api-region:{}",ipResource.getRegion());
return;
}
RunInstancesResponse response = AliEcsClient.runInstances(
ipResource.getOwner(), ipResource.getUsername(),ipResource.getPassword(), aliResion);
if (null ==response || !response.getStatusCode().equals(200)) {
log.error("{}, fail to buy ip : {}", logs, JSON.toJSON(response.getBody()));
}else if(Objects.nonNull(response.getBody())){
String instanceId = response.getBody().getInstanceIdSets().getInstanceIdSet().get(0);
cloudPlatformOrder.setPlatformOrderId(instanceId);
cloudPlatformOrder.setPlatformType("aliyun");
cloudPlatformOrder.setRegionId(aliResion);
ipResource.setStatus(3);
ipResourceRepository.save(ipResource);
cloudPlatformOrderRepository.save(cloudPlatformOrder);
result = true;
}
}
/**
* 购买ip的定时任务，每分钟一次
* 购买失败处理
*
* 购买后，长时间未能成功的ip资源视为购买失败。
* 购买malaysia实例失败后不进行重新购买
* 购买时长超过20分钟的ip资源释放，并将购买花销退回账户余额。
*/
boolean overtime = ipResource.getPurchasedTime() < Instant.now().minusSeconds(7200).toEpochMilli();
/*boolean problemRegion = "malaysia".equals(ipResource.getRegion());
boolean deal = overtime || problemRegion;*/
if (!result && overtime) {
log.info("{}, ip resource purchase failure, {}", logs, ipResource);
IpChargeRequestDto ipChargeRequestDto = buildIpChargeRequestDto(ipResource, 3, 0);
accountService.chargeByMoney(ipResource.getOwner(), -ipResource.getPrice(), ipChargeRequestDto);
if (ipResource.getShopIds() != null && ipResource.getShopIds().size() > 0) {
ShopRequestDto shopRequestDto = new ShopRequestDto();
shopRequestDto.setIpId(ipResource.getId());
shopRequestDto.setShopIds(ipResource.getShopIds());
ipAndShopService.unBindShops(ipResource.getUsername(), shopRequestDto);
}
ipResourceRepository.deleteById(ipResource.getId());
}
} finally {
long end = System.currentTimeMillis();
log.info("{}, buy ip task {} execution time is: {}s", logs, ipResource.getId(), (end - start) / 1000);
try {
ipResourceRepository.unLockTask(ipResource.getId());
} catch (Throwable th) {
log.error("{}, unlock failed : {}", logs, th.getMessage(), th);
//try again
ipResourceRepository.unLockTask(ipResource.getId());
}
}
}
},
ThreadPoolUtils.buyIpTasksPool
);
futureList.add(future);
}
CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
}
@Scheduled(cron = "0 0/1 * * * ?")
public void queryIpNewTasks() {
long time = Instant.now().minusSeconds(300).toEpochMilli();
List<IpResource> ipResources = ipResourceRepository.sampleTasks(3, time);
for (IpResource ipResource : ipResources) {
long start = System.currentTimeMillis();
CompletableFuture.runAsync(
() -> {
if (ipResourceRepository.lockTask(ipResource)) {
try {
//查询cloudPlatform
List<CloudPlatformOrder> cloudPlatformOrders = cloudPlatformOrderRepository.findByOwnerAndIpResourceId(
ipResource.getOwner(), ipResource.getId());
if(CollectionUtils.isEmpty(cloudPlatformOrders)){
log.error("queryIpTask- 查询云平台订单失败：入参：{}",JSON.toJSON(ipResource));
return;
}
CloudPlatformOrder cloudPlatformOrder = cloudPlatformOrders.get(0);
if("aliyun".equals(cloudPlatformOrder.getPlatformType())){
DescribeInstancesResponse response = AliEcsClient.getDescribeInstances(
ipResource.getOwner(), cloudPlatformOrder.getPlatformOrderId(),
cloudPlatformOrder.getRegionId());
if (!response.getStatusCode().equals(200)) {
log.error(", fail to query ip : {}", JSON.toJSON(response.getBody()));
}
if(Objects.nonNull(response.getBody())){
DescribeInstancesResponseBody body = response.getBody();
//TODO 查询ip信息并且封装
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
NotifyUtils.sendMessage("浏览器后端 queryIpTasks() 又炸了，赶紧看啊", e, NotifyUtils.MsgType.WEBHOOK);
} finally {
long end = System.currentTimeMillis();
log.debug("queryIpTask {} execution time is: " + (end - start) / 1000 + "s", ipResource.getId());
try {
ipResourceRepository.unLockTask(ipResource.getId());
} catch (Throwable th) {
log.error("unlock failed", th);
//try again
ipResourceRepository.unLockTask(ipResource.getId());
}
}
}
},
ThreadPoolUtils.queryIpTasksPool
);
}
}
/**
* 购买ip的定时任务，每分钟一次
*/
//@Scheduled(cron = "0 0/1 * * * ?")
public void buyIpTasks() {
String logs = "【BrowserTask -- buyIpTasks】 ";
log.info("{}, ****************ip resource purchase start****************", logs);
......@@ -222,7 +393,7 @@ public class BrowserTask {
CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
}
@Scheduled(cron = "0 0/1 * * * ?")
//@Scheduled(cron = "0 0/1 * * * ?")
public void queryIpTasks() {
String URL = (profiles.equals("dev") || profiles.equals("staging")) ? TESTURL : CLOUDAMURL;
long time = Instant.now().minusSeconds(300).toEpochMilli();
......@@ -367,8 +538,30 @@ public class BrowserTask {
*/
private void deleteOverdueIp(IpResource ipResource){
String logs = "【deleteOverdueIp】 ";
List<CloudPlatformOrder> list = cloudPlatformOrderRepository.findByOwnerAndIpResourceId(
ipResource.getOwner(), ipResource.getId());
if(CollectionUtils.isNotEmpty(list)){
CloudPlatformOrder cloudPlatformOrder = list.get(0);
Boolean isDelet = false;
if("aliyun".equals(cloudPlatformOrder.getPlatformType())){
DeleteInstanceResponse response = AliEcsClient.deleteInstance(
ipResource.getOwner(), cloudPlatformOrder.getPlatformOrderId());
if(null != response && response.getStatusCode().equals(200)) {
isDelet = true;
}
}
if(isDelet){
ipResource.setShopIds(null);
ipResource.setDeleted(true);
ipResourceRepository.save(ipResource);
}else {
log.error("Scheduled {}, {} delete failed", logs, ipResource.getAddr());
}
}
/* String logs = "【deleteOverdueIp】 ";
String path = "/intelligroup/ipresources?accountId=browser&ip=";
try {
if (doDelete(path, ipResource.getAddr())) {
ipResource.setShopIds(null);
ipResource.setDeleted(true);
......@@ -378,7 +571,7 @@ public class BrowserTask {
}
} catch (Exception e) {
log.error("Scheduled {}, Exception : {}", logs, e.getMessage(), e);
}
}*/
}
private final static String SUCCESS = "success";
......
src/main/java/com/edgec/browserbackend/common/client/AliEcsClient.java
View file @ 9eca9001
......@@ -45,6 +45,18 @@ public class AliEcsClient {
return "lt-wz9g4apcsdali7t0lec8";
}
//脚本userData
private static String getUserData() {
return "#!/bin/sh\n"
+ "echo \"users %s:CL:%s\n"
+ "allow %s *\n"
+ "auth strong\n"
+ "proxy -p8080\n"
+ "proxy -p8443 -a\n"
+ "socks -p1080\n"
+ "flush\" | tee /etc/3proxy.cfg\n"
+ "3proxy -C 3proxy.cfg";
}
/*
* @Description //创建链接
* @Date 2024/7/25 10:32
......@@ -67,18 +79,11 @@ public class AliEcsClient {
/**
* 创建并运行实例
*/
public static RunInstancesResponse runInstances(String username, String regionId, String password) {
public static RunInstancesResponse runInstances(String owner,String username, String password, String regionId) {
AsyncClient client = createClient();
RunInstancesResponse runInstancesResponse = null;
String userData = "#!/bin/sh\n"
+ "echo \"users 3ASka71a:CL:7asMpA16\n"
+ "allow 3ASka71a *\n"
+ "auth strong\n"
+ "proxy -p8080\n"
+ "proxy -p8443 -a\n"
+ "socks -p1080\n"
+ "flush\" | tee /etc/3proxy.cfg\n"
+ "3proxy -C 3proxy.cfg";
String userData = String.format(getUserData(), username, password, username);
try {
RunInstancesRequest runInstancesRequest = RunInstancesRequest.builder()
.regionId(regionId)
......@@ -86,11 +91,11 @@ public class AliEcsClient {
.userData(Aes.base64Encode(userData.getBytes()))
.keyPairName("proxy-ip")
.build();
log.info("调用aliEcs创建并运行实例，入参：userName:{},regionId:{}", username, regionId);
log.info("调用aliEcs创建并运行实例，入参：owner:{},regionId:{}", owner, regionId);
CompletableFuture<RunInstancesResponse> response = client.runInstances(
runInstancesRequest);
runInstancesResponse = response.get();
log.info("调用aliEcs创建并运行实例，响应参数:userName:{},response:{}", username,
log.info("调用aliEcs创建并运行实例，响应参数:owner:{},response:{}", owner,
JSON.toJSON(runInstancesResponse));
} catch (Exception e) {
log.error("fail to aliEcs runInstances {}", e.getMessage());
......@@ -117,7 +122,7 @@ public class AliEcsClient {
/**
* 查询单个实例
*/
public static DescribeInstancesResponse getDescribeInstances(String username, String instanceId,
public static DescribeInstancesResponse getDescribeInstances(String owner, String instanceId,
String regionId) {
AsyncClient client = createClient();
DescribeInstancesResponse resp = null;
......@@ -128,12 +133,12 @@ public class AliEcsClient {
.regionId(regionId)
.instanceIds(instanceIds)
.build();
log.info("调用aliEcs查询单个实例，入参：userName:{},regionId:{},instanceId:{}",
username, regionId, instanceId);
log.info("调用aliEcs查询单个实例，入参：owner:{},regionId:{},instanceId:{}",
owner, regionId, instanceId);
CompletableFuture<DescribeInstancesResponse> response = client.describeInstances(
describeInstancesRequest);
resp = response.get();
log.info("调用aliEcs查询单个实例，响应参数:userName:{},response:{}", username,
log.info("调用aliEcs查询单个实例，响应参数:owner:{},response:{}", owner,
JSON.toJSON(resp));
} catch (Exception e) {
......@@ -147,7 +152,7 @@ public class AliEcsClient {
/**
* 删除ECS
*/
public static DeleteInstanceResponse deleteInstance(String username, String instanceId) {
public static DeleteInstanceResponse deleteInstance(String owner, String instanceId) {
AsyncClient client = createClient();
DeleteInstanceResponse resp = null;
try {
......@@ -155,12 +160,12 @@ public class AliEcsClient {
.instanceId(instanceId)
.force(true)
.build();
log.info("调用aliEcs删除ECS实例，入参：userName:{},instanceId:{}",
username, instanceId);
log.info("调用aliEcs删除ECS实例，入参：owner:{},instanceId:{}",
owner, instanceId);
CompletableFuture<DeleteInstanceResponse> response = client.deleteInstance(
deleteInstanceRequest);
resp = response.get();
log.info("调用aliEcs删除ECS实例，响应参数:userName:{},response:{}", username,
log.info("调用aliEcs删除ECS实例，响应参数:owner:{},response:{}", owner,
JSON.toJSON(resp));
} catch (Exception e) {
log.error("fail to aliEcs deleteInstance {}", e.getMessage());
......
src/main/java/com/edgec/browserbackend/common/enums/RegionMappingEnum.java
View file @ 9eca9001
package com.edgec.browserbackend.common.enums;
import java.util.Arrays;
import java.util.Objects;
import java.util.Random;
import lombok.AllArgsConstructor;
import lombok.Getter;
......@@ -37,7 +40,6 @@ public enum RegionMappingEnum {
INDIA("india","孟买",""),//待确认
MEAST("meast","迪拜","me-east-1"),
;
......@@ -47,4 +49,30 @@ public enum RegionMappingEnum {
private final String aliRegionApi;//阿里云地区api
public static RegionMappingEnum getByRegion(String type) {
return Arrays.stream(RegionMappingEnum.values())
.filter(item -> Objects.equals(item.getRegionCn(), type))
.findFirst().orElse(null);
}
public static RegionMappingEnum getUsRandom(){
Random random = new Random();
int randomNumber = random.nextInt(2);
if(randomNumber == 0){
return RegionMappingEnum.US_CALIFORNIA;
}else {
return US_VIRGINIA.US_VIRGINIA;
}
}
public static RegionMappingEnum getCnRandom(){
Random random = new Random();
int randomNumber = random.nextInt(2);
if(randomNumber == 0){
return RegionMappingEnum.US_CALIFORNIA;
}else {
return US_VIRGINIA.US_VIRGINIA;
}
}
}
src/test/java/com/edgec/browserbackend/BrowserBackendApplicationTests.java
View file @ 9eca9001
......@@ -26,7 +26,7 @@ class BrowserBackendApplicationTests {
@Test
void buyEcsTest() throws Exception {
AliEcsClient.runInstances("11","cn-shenzhen","2222");
//AliEcsClient.runInstances("11","cn-shenzhen","2222");
//AliEcsClient.getDescribeInstances("测试","i-wz9hf0i13xj6h904k14p","cn-shenzhen");
//DeleteInstanceResponse response = AliEcsClient.deleteInstance("测试", "i-wz9ebrei1dvowzej6nfn");
}
......@@ -43,10 +43,8 @@ class BrowserBackendApplicationTests {
+ "flush\" | tee /etc/3proxy.cfg\n"
+ "3proxy -C 3proxy.cfg";
System.out.println(Aes.base64Encode(userData.getBytes()));
byte[] bytes = Aes.base64Decode(
"IyEvYmluL3NoCmVjaG8gInVzZXJzIDNBU2thNzFhOkNMOjdhc01wQTE2CmFsbG93IDNBU2thNzFhICAqCmF1dGggc3Ryb25nCnByb3h5IC1wODA4MApwcm94eSAtcDg0NDMgLWEKc29ja3MgLXAxMDgwCmZsdXNoIiB8IHRlZSAvZXRjLzNwcm94eS5jZmcKM3Byb3h5IC1DIDNwcm94eS5jZmc=");
System.out.println(new String(bytes,"UTF-8"));
String userData1 = String.format(userData, "ces11", "123123", "ces11");
System.out.println(userData1);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment