From cd858be89a6a9becd26b8bef862d35d010f8c77c Mon Sep 17 00:00:00 2001 From: yixian Date: Wed, 15 Jul 2020 01:21:15 +1000 Subject: [PATCH] ato report task --- .../analysis/core/ATOReportService.java | 10 + .../core/impls/ATOReportServiceImpl.java | 182 +++++++++++++++++- .../manage/dev/web/TestController.java | 25 +-- 3 files changed, 205 insertions(+), 12 deletions(-) diff --git a/src/main/java/au/com/royalpay/payment/manage/analysis/core/ATOReportService.java b/src/main/java/au/com/royalpay/payment/manage/analysis/core/ATOReportService.java index b1b018dcd..bfa668ac7 100644 --- a/src/main/java/au/com/royalpay/payment/manage/analysis/core/ATOReportService.java +++ b/src/main/java/au/com/royalpay/payment/manage/analysis/core/ATOReportService.java @@ -1,5 +1,9 @@ package au.com.royalpay.payment.manage.analysis.core; +import com.alibaba.fastjson.JSONObject; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; import java.util.Date; /** @@ -8,4 +12,10 @@ import java.util.Date; public interface ATOReportService { String exportBTTPSFile(Date from, Date to); + + void startExportTask(Date from, Date to); + + JSONObject taskStatus(); + + void downloadFile(HttpServletResponse resp) throws IOException; } diff --git a/src/main/java/au/com/royalpay/payment/manage/analysis/core/impls/ATOReportServiceImpl.java b/src/main/java/au/com/royalpay/payment/manage/analysis/core/impls/ATOReportServiceImpl.java index ecc98a8cd..2ab4bf43f 100644 --- a/src/main/java/au/com/royalpay/payment/manage/analysis/core/impls/ATOReportServiceImpl.java +++ b/src/main/java/au/com/royalpay/payment/manage/analysis/core/impls/ATOReportServiceImpl.java @@ -4,26 +4,40 @@ import au.com.royalpay.payment.manage.analysis.beans.ato.*; import au.com.royalpay.payment.manage.analysis.core.ATOReportService; import au.com.royalpay.payment.manage.mappers.payment.TransactionMapper; import au.com.royalpay.payment.manage.mappers.system.ClientMapper; +import au.com.royalpay.payment.tools.connections.attachment.core.AttachmentClient; +import au.com.royalpay.payment.tools.env.PlatformEnvironment; +import au.com.royalpay.payment.tools.exceptions.BadRequestException; import au.com.royalpay.payment.tools.exceptions.ServerErrorException; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.apache.commons.codec.Charsets; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateFormatUtils; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import javax.annotation.Resource; +import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * Create by yixian at 2018-08-31 1:23 @@ -38,6 +52,16 @@ public class ATOReportServiceImpl implements ATOReportService { @Value("classpath:data/category/billbuddyindustry.json") private org.springframework.core.io.Resource industryResource; private Map industryMap; + private final StringRedisTemplate redisTemplate; + private final String prefix; + private final AttachmentClient client; + private Progress progress; + + public ATOReportServiceImpl(StringRedisTemplate redisTemplate, @Value("${app.redis.prefix}") String prefix, AttachmentClient client) { + this.redisTemplate = redisTemplate; + this.prefix = prefix; + this.client = client; + } @PostConstruct public void loadIndustryConfiguration() throws IOException { @@ -102,14 +126,90 @@ public class ATOReportServiceImpl implements ATOReportService { return data.outputBTTPS(); } + private String progressKey() { + return prefix + ":tasks:exporting-bttps"; + } + + private String reportFileKey() { + return prefix + ":caching:exporting-bttps-file"; + } + + @Override + public void startExportTask(Date from, Date to) { + if (progress != null && !progress.isFinished()) { + throw new BadRequestException("In Progress"); + } + if (redisTemplate.boundValueOps(progressKey()).get() != null) { + throw new BadRequestException("In Progress"); + } + new Thread(() -> { + progress = new Progress(); + try { + String content = exportBTTPSFile(from, to); + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + progress.setStatus("Uploading,filesize:" + contentBytes.length); + String filename = "royalpay_ato_report_" + new DateTime(from).toString("yyyyMMdd") + "_to_" + new DateTime(to).toString("yyyyMMdd") + ".bttps"; + JSONObject file = client.uploadFile(new ByteArrayInputStream(contentBytes), filename, true); + String fileid = file.getString("fileid"); + logger.info("uploaded ATO report to {}", fileid); + progress.setFileid(fileid); + redisTemplate.boundValueOps(reportFileKey()).set(file.toJSONString(), Duration.ofDays(1)); + } catch (Exception e) { + progress.setStatus(e.getMessage()); + synchronizeProgress(); + } finally { + progress.setFinished(); + synchronizeProgress(); + } + }).start(); + } + + @Override + public JSONObject taskStatus() { + String status = redisTemplate.boundValueOps(progressKey()).get(); + if (status != null) { + JSONObject std = JSON.parseObject(status); + std.remove("fileid"); + return std; + } else { + String file = redisTemplate.boundValueOps(reportFileKey()).get(); + JSONObject std = new JSONObject(); + if (file != null) { + std.put("success", true); + std.put("finished", true); + } + return std; + } + } + + @Override + public void downloadFile(HttpServletResponse resp) throws IOException { + String file = redisTemplate.boundValueOps(reportFileKey()).get(); + if (file != null) { + JSONObject fileInfo = JSON.parseObject(file); + String filename = StringUtils.substringAfterLast(fileInfo.getString("url"), "/"); + resp.setHeader(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.builder("attachment").filename(filename).build().toString()); + resp.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE); + try (OutputStream out = resp.getOutputStream()) { + client.getFileContent(fileInfo.getString("fileid"), out); + out.flush(); + } + } else { + throw new BadRequestException("No file available"); + } + } + private void loadMonthTransactions(ReportingPartyData reportingParty, DateTime startOfMon, DateTime endOfMon) { List clients = clientMapper.listClientsWithTransactionsSettled(startOfMon.toDate(), endOfMon.toDate()); + if (progress != null) { + progress.setMonth(startOfMon, clients); + } clients.parallelStream().forEach(clientId -> loadClientMonthTransactions(reportingParty, clientId, startOfMon, endOfMon)); } private void loadClientMonthTransactions(ReportingPartyData reportingParty, Integer clientId, DateTime startOfMon, DateTime endOfMon) { BusinessData biz = reportingParty.findBusiness(clientId); - logger.debug("Exporting date range for client[{}]:{} ~ {}",clientId,startOfMon.toString("yyyy-MM-dd"), endOfMon.toString("yyyy-MM-dd")); + logger.debug("Exporting date range for client[{}]:{} ~ {}", clientId, startOfMon.toString("yyyy-MM-dd"), endOfMon.toString("yyyy-MM-dd")); if (biz == null) { JSONObject cli = clientMapper.findClientIgnoreInvalid(clientId); AddressInfo addr = new AddressInfo(cli.getString("address"), cli.getString("suburb"), cli.getString("state"), cli.getString("postcode"), cli.getString("country")); @@ -128,5 +228,85 @@ public class ATOReportServiceImpl implements ATOReportService { for (JSONObject analysis : analysisList) { biz.addTransaction(new TransactionSummaryData(analysis, startOfMon, endOfMon)); } + if (progress != null) { + progress.markClientFinished(clientId); + } + } + + private void synchronizeProgress() { + redisTemplate.boundValueOps(progressKey()).set(JSON.toJSONString(progress), Duration.ofMinutes(5)); + } + + public static class Progress { + private String currentMonth; + private Map clientsStatus; + private final String server; + private String fileid; + private boolean finished; + + private String status; + + public Progress() { + status = "Init"; + server = PlatformEnvironment.getEnv().appName(); + } + + public void setMonth(DateTime start, List clients) { + currentMonth = start.toString("yyyy-MM"); + clientsStatus = clients.stream().collect(ConcurrentHashMap::new, (map, cid) -> map.put(cid, false), Map::putAll); + } + + public String getFileid() { + return fileid; + } + + public Progress setFileid(String fileid) { + this.fileid = fileid; + return this; + } + + public boolean isFinished() { + return finished; + } + + public boolean isSuccess() { + return fileid != null; + } + + public void setFinished() { + this.finished = true; + } + + public String getStatus() { + return status; + } + + public int getProgress() { + if (clientsStatus == null || clientsStatus.isEmpty()) { + return 0; + } + int total = clientsStatus.size(); + long finishedCount = clientsStatus.values().stream().filter(success -> success).count(); + return (int) finishedCount * 100 / total; + } + + public String getCurrentMonth() { + return currentMonth; + } + + public String getServer() { + return server; + } + + public Progress setStatus(String status) { + this.status = status; + return this; + } + + public void markClientFinished(Integer clientId) { + if (clientsStatus != null) { + clientsStatus.put(clientId, true); + } + } } } diff --git a/src/main/java/au/com/royalpay/payment/manage/dev/web/TestController.java b/src/main/java/au/com/royalpay/payment/manage/dev/web/TestController.java index 738a378f5..4e871d2e2 100644 --- a/src/main/java/au/com/royalpay/payment/manage/dev/web/TestController.java +++ b/src/main/java/au/com/royalpay/payment/manage/dev/web/TestController.java @@ -55,7 +55,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.ui.Model; import org.springframework.util.Assert; -import org.springframework.util.MimeTypeUtils; import org.springframework.validation.Errors; import org.springframework.web.bind.annotation.*; import org.springframework.web.servlet.ModelAndView; @@ -67,10 +66,8 @@ import javax.validation.Valid; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.io.OutputStream; import java.math.BigDecimal; import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; @@ -504,17 +501,23 @@ public class TestController { } - @ManagerMapping(value = "/reports/ato_reports/royalpay_ato_report_{year}.bttps", role = ManagerRole.DEVELOPER, method = RequestMethod.GET) + @ManagerMapping(value = "/reports/ato_reports/royalpay_ato_report_{year}.bttps", role = ManagerRole.DEVELOPER, method = RequestMethod.POST) @ReadOnlyConnection - public void downloadATOReport(@PathVariable String year, HttpServletResponse resp) throws IOException { + public JSONObject exportATOReport(@PathVariable String year) { DateTime from = DateTime.parse(year + "-07-01"); DateTime to = from.plusYears(1).plusDays(-1); - String atoReport = atoReportService.exportBTTPSFile(from.toDate(), to.toDate()); - resp.setContentType(MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE); - try (OutputStream out = resp.getOutputStream()) { - out.write(atoReport.getBytes(StandardCharsets.UTF_8)); - out.flush(); - } + atoReportService.startExportTask(from.toDate(), to.toDate()); + return atoReportService.taskStatus(); + } + + @ManagerMapping(value = "/reports/ato_reports/royalpay_ato_report", role = ManagerRole.DEVELOPER, method = RequestMethod.GET) + public JSONObject exportATOStatus() { + return atoReportService.taskStatus(); + } + + @ManagerMapping(value = "/reports/ato_reports/royalpay_ato_report.bttps", role = ManagerRole.DEVELOPER, method = RequestMethod.GET) + public void downloadReport(HttpServletResponse response) throws IOException { + atoReportService.downloadFile(response); } @ManagerMapping(value = "/custom_declare_check/{reportId}/detail", method = RequestMethod.GET, role = ManagerRole.DEVELOPER)