# GrabOptimandReviewsData.java

    package cloud.wconnect.rateshop.businesstask.reviewanalysis;

    import cloud.wconnect.commonlib.component.BeanMapper;
    import cloud.wconnect.commonlib.cons.BaseConstant;
    import cloud.wconnect.commonlib.pojo.dto.PropertyDTO;
    import cloud.wconnect.commonlib.redis.PropertyI18nRedisService;
    import cloud.wconnect.commonlib.service.MailInternalService;
    import cloud.wconnect.commonlib.util.DateTimeUtils;
    import cloud.wconnect.commonlib.util.HttpUtils;
    import cloud.wconnect.commonlib.util.JsonUtils;
    import cloud.wconnect.rateshop.cons.rateshop.DatasourceEnum;
    import cloud.wconnect.rateshop.cons.rateshop.PlatformEnum;
    import cloud.wconnect.rateshop.entity.reviewanalysis.OptimandSetting;
    import cloud.wconnect.rateshop.entity.reviewanalysis.Review;
    import cloud.wconnect.rateshop.pojo.dto.reviewanalysis.OptimandReviews;
    import cloud.wconnect.rateshop.pojo.dto.reviewanalysis.OptimandReviewsData;
    import cloud.wconnect.rateshop.service.reviewanalysis.IOptimandSettingService;
    import cloud.wconnect.rateshop.service.reviewanalysis.IReviewService;
    import com.fasterxml.jackson.databind.ObjectMapper;
    import com.google.common.collect.Maps;
    import lombok.extern.slf4j.Slf4j;
    import org.apache.commons.collections.CollectionUtils;
    import org.redisson.api.RSet;
    import org.redisson.api.RedissonClient;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.data.redis.core.RedisTemplate;
    import org.springframework.stereotype.Component;

    import java.text.MessageFormat;
    import java.time.LocalDate;
    import java.time.LocalDateTime;
    import java.time.ZonedDateTime;
    import java.time.format.DateTimeFormatter;
    import java.util.HashMap;
    import java.util.List;
    import java.util.Map;
    import java.util.concurrent.TimeUnit;
    import java.util.function.Function;
    import java.util.stream.Collectors;

    /**
     * 抓取Optimand的评论数据
     *
     * @author lichen
     * @date 2024/12/28
     */
    @Slf4j
    @Component
    public class GrabOptimandReviewsData {

        @Autowired
        private RedisTemplate<String, Object> redisTemplate;
        @Autowired
        private IOptimandSettingService optimandSettingService;
        @Autowired
        private PropertyI18nRedisService propertyI18nRedisService;
        @Autowired
        private IReviewService reviewService;
        @Autowired
        private MailInternalService mailInternalService;
        @Autowired
        private RedissonClient redissonClient;
        @Autowired
        private BeanMapper beanMapper;
        @Autowired
        private ObjectMapper objectMapper;
        /**
         * 定时任务并发key
         */
        private static final String PREFIX_KEY = "wconnect:task:review-analysis:grab-optimand-review:";
        /**
         * 定时任务执行结果key
         */
        private static final String PREFIX_KEY_RESULT = "wconnect:task:review-analysis:grab-optimand-review-result:";

        /**
         * 每天凌晨1,4,7点抓取Optimand数据
         *
         * @return void
         * @author lichen
         * @date 2024/12/28
         */
    //    @Scheduled(cron = "${schedule.cron.grab-optimand-review}", zone = "${default.timezone}")
        public void grabOptimandReviewsData() {
            //判断并发
            String currentDateWithHour = LocalDateTime.now().format(DateTimeFormatter.ofPattern(BaseConstant.DATE_FORMAT_YYMMDDHH));
            if (!preventConcurrency(PREFIX_KEY + currentDateWithHour)) {
                return;
            }
            log.info("GrabOptimandReviewsData.start");
            try {
                //查询Optimand配置
                List<OptimandSetting> optimandSettings = optimandSettingService.list();
                if (CollectionUtils.isEmpty(optimandSettings)) return;
                List<Integer> propertyIds = optimandSettings.stream().map(OptimandSetting::getPropertyId).toList();
                Map<Integer, Integer> propertyId2IdProp = optimandSettings.stream().collect(Collectors.toMap(OptimandSetting::getPropertyId, OptimandSetting::getIdProp));

                //过滤出有效的资产
                List<PropertyDTO> effectivePropertyDTOS = propertyI18nRedisService.list(propertyIds).stream().filter(PropertyDTO::getStatus).toList();
                if (CollectionUtils.isEmpty(effectivePropertyDTOS)) return;
                List<Integer> effectivePropertyIds = effectivePropertyDTOS.stream().map(PropertyDTO::getId).toList();
                Map<Integer, PropertyDTO> propertyId2PropertyDTO = effectivePropertyDTOS.stream().collect(Collectors.toMap(PropertyDTO::getId, Function.identity()));

                //当前时间
                String currentDate = LocalDate.now().format(DateTimeFormatter.ofPattern(BaseConstant.DATE_FORMAT_YYYYMMDD));
                //从缓存中获取已经抓取过的资产ID
                RSet<Integer> alreadyGrabPropertyIds = redissonClient.getSet(PREFIX_KEY_RESULT + currentDate);
                //设置过期时间为10个小时
                alreadyGrabPropertyIds.expire(10, TimeUnit.HOURS);

                //等待抓取的资产ID集合
                List<Integer> waitGrabPropertyIds = effectivePropertyIds;
                if (CollectionUtils.isNotEmpty(alreadyGrabPropertyIds)) {
                    waitGrabPropertyIds = effectivePropertyIds.stream().filter(propertyId -> !alreadyGrabPropertyIds.contains(propertyId)).toList();
                }
                if (CollectionUtils.isEmpty(waitGrabPropertyIds)) return;

                for (Integer propertyId : waitGrabPropertyIds) {
                    PropertyDTO propertyDTO = propertyId2PropertyDTO.get(propertyId);
                    Integer idProp = propertyId2IdProp.get(propertyId);
                    try {
                        //获取7天前的日期
                        ZonedDateTime zonedDateTime = DateTimeUtils.dateNow(propertyDTO.getTimezone()).minusDays(7);
                        String queryDate = DateTimeUtils.parseZonedDateTimeToStr(zonedDateTime, BaseConstant.DATE_FORMAT_YYYY_MM_DD);
                        //抓取评论数据
                        List<OptimandReviews> optimandReviewsData = grabReviewsData(queryDate, idProp);
                        if (CollectionUtils.isNotEmpty(optimandReviewsData)) {
                            long nowTimestamp = DateTimeUtils.nowTimestamp();
                            //持久化处理
                            List<Review> reviews = beanMapper.mapList(optimandReviewsData, Review.class, (data, entity) -> {
                                entity.setTenantId(propertyDTO.getTenantId());
                                entity.setPropertyId(propertyId);
                                entity.setDatasourceId(DatasourceEnum.OPTIMAND.getDatasourceId());
                                entity.setPlatformId(PlatformEnum.BOOKING_COM.getCode());
                                entity.setReviewDate(data.getReview_date().replace("-", ""));
                                entity.setTravellerType(data.getTraveller_type());
                                entity.setCreateTime(nowTimestamp);
                            });
                            //批量保存
                            reviewService.saveBatch(propertyDTO, queryDate.replace("-", ""), reviews);
                        }
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                        //发送预警邮件
                        String subject = MessageFormat.format("[Warning] - Property:{0} Grab optimand reviews data error", propertyDTO.getName());
                        mailInternalService.sendAlertEmail(subject, e.getMessage());
                        //任意资产抓取失败，继续抓取下一个资产的数据
                        continue;
                    }
                    //抓取成功后添加到已经抓取的集合中
                    alreadyGrabPropertyIds.add(propertyId);
                }
                log.info("GrabOptimandReviewsData.end");
            } catch (Exception e) {
                log.error("GrabOptimandReviewsData.error, exception:" + e.getMessage());
                log.error(e.getMessage(), e);
                //发送预警邮件
                mailInternalService.sendAlertEmail("[Warning] - Grab optimand reviews data task execution error", e.getMessage());
            }
        }

        /**
         * 抓取评论数据
         *
         * @param queryDate 查询日期
         * @param idProp    optimand平台的唯一标识
         * @return void
         * @author lichen
         * @date 2024/12/30
         */
        private List<OptimandReviews> grabReviewsData(String queryDate, Integer idProp) {
            //请求参数
            HashMap<String, Object> hashMap = Maps.newHashMapWithExpectedSize(6);
            hashMap.put("api_key", "friendwell.com");
            hashMap.put("action", "get_booking_reviews");
            hashMap.put("id_prop", idProp);
            hashMap.put("start_day", queryDate);
            hashMap.put("end_day", queryDate);
            hashMap.put("realtime_data", 0);
            String requestJson = JsonUtils.convertObjToString(objectMapper, hashMap);

            OptimandReviewsData optimandReviewsData = HttpUtils.postJson("https://optimand.com/api/bar/rs_api.php", null,
                    requestJson, OptimandReviewsData.class, objectMapper);
            return optimandReviewsData.getData();
        }

        /**
         * 检测定时任务是否并发执行
         *
         * @param key 缓存key
         * @return java.lang.Boolean
         * @author lichen
         * @date 2023/9/29
         */
        private Boolean preventConcurrency(String key) {
            return redisTemplate.opsForValue().setIfAbsent(key, 0, 1, TimeUnit.DAYS);
        }
    }