msb-public-java
/
RuoYi-Cloud
mirror of https://gitee.com/y_project/RuoYi-Cloud.git
1
0
Code Issues Projects Releases Wiki Activity

1、36壁纸爬虫配置信息编写

Browse Source 
2、抽出配置功能，前端可以配置修改
pull/254/head
xjs 4 years ago
parent 15690d4d02
commit 8638b4de30
16 changed files with 461 additions and 52 deletions
Show Stats Download Patch File Download Diff File

12
ruoyi-api/ruoyi-api-system/src/main/java/com/ruoyi/system/api/RemoteConfigService.java
Unescape View File

@ -6,6 +6,8 @@ import com.ruoyi.system.api.factory.RemoteConfigFallbackFactory;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestParam;
/**
* 系统配置服务
@ -23,4 +25,14 @@ public interface RemoteConfigService {
@GetMapping(value = "/config/configKeyForRPC/{configKey}")
R<String> getConfigKeyForRPC(@PathVariable("configKey") String configKey);
/**
* 根据key修改参数配置
*
* @return r
* @since 2022-02-21
*/
@PutMapping("/config/editForRPC")
public R editForRPC(@RequestParam("key") String key, @RequestParam("value") String value);
}

9
ruoyi-api/ruoyi-api-system/src/main/java/com/ruoyi/system/api/factory/RemoteConfigFallbackFactory.java
Unescape View File

@ -15,11 +15,16 @@ import org.springframework.stereotype.Component;
public class RemoteConfigFallbackFactory implements FallbackFactory<RemoteConfigService> {
@Override
public RemoteConfigService create(Throwable cause) {
log.error("系统配置服务调用失败:{}", cause.getMessage());
return new RemoteConfigService() {
@Override
public R<String> getConfigKeyForRPC(String configKey) {
log.error("系统配置服务调用失败:{}", cause.getMessage());
return R.fail("系统配置服务调用失败");
return R.fail("系统配置服务查询调用失败");
}
@Override
public R editForRPC(String key, String value) {
return R.fail("系统配置服务修改调用失败");
}
};
}

52
ruoyi-modules/ruoyi-system/src/main/java/com/ruoyi/system/controller/SysConfigController.java
Unescape View File

@ -21,13 +21,12 @@ import java.util.List;
/**
* 参数配置 信息操作处理
*
*
* @author ruoyi
*/
@RestController
@RequestMapping("/config")
public class SysConfigController extends BaseController
{
public class SysConfigController extends BaseController {
@Autowired
private ISysConfigService configService;
@ -36,8 +35,7 @@ public class SysConfigController extends BaseController
*/
@RequiresPermissions("system:config:list")
@GetMapping("/list")
public TableDataInfo list(SysConfig config)
{
public TableDataInfo list(SysConfig config) {
startPage();
List<SysConfig> list = configService.selectConfigList(config);
return getDataTable(list);
@ -46,8 +44,7 @@ public class SysConfigController extends BaseController
@Log(title = "参数管理", businessType = BusinessType.EXPORT)
@RequiresPermissions("system:config:export")
@PostMapping("/export")
public void export(HttpServletResponse response, SysConfig config)
{
public void export(HttpServletResponse response, SysConfig config) {
List<SysConfig> list = configService.selectConfigList(config);
ExcelUtil<SysConfig> util = new ExcelUtil<SysConfig>(SysConfig.class);
util.exportExcel(response, list, "参数数据");
@ -57,8 +54,7 @@ public class SysConfigController extends BaseController
* 根据参数编号获取详细信息
*/
@GetMapping(value = "/{configId}")
public AjaxResult getInfo(@PathVariable Long configId)
{
public AjaxResult getInfo(@PathVariable Long configId) {
return AjaxResult.success(configService.selectConfigById(configId));
}
@ -66,15 +62,15 @@ public class SysConfigController extends BaseController
* 根据参数键名查询参数值
*/
@GetMapping(value = "/configKey/{configKey}")
public AjaxResult getConfigKey(@PathVariable String configKey)
{
public AjaxResult getConfigKey(@PathVariable String configKey) {
return AjaxResult.success(configService.selectConfigByKey(configKey));
}
/**
* 根据参数键名查询参数值forRpc
* @since 2022-02-20
*
* @Author xjs
* @since 2022-02-20
*/
@GetMapping(value = "/configKeyForRPC/{configKey}")
public R<String> getConfigKeyForRPC(@PathVariable String configKey) {
@ -87,10 +83,8 @@ public class SysConfigController extends BaseController
@RequiresPermissions("system:config:add")
@Log(title = "参数管理", businessType = BusinessType.INSERT)
@PostMapping
public AjaxResult add(@Validated @RequestBody SysConfig config)
{
if (UserConstants.NOT_UNIQUE.equals(configService.checkConfigKeyUnique(config)))
{
public AjaxResult add(@Validated @RequestBody SysConfig config) {
if (UserConstants.NOT_UNIQUE.equals(configService.checkConfigKeyUnique(config))) {
return AjaxResult.error("新增参数'" + config.getConfigName() + "'失败，参数键名已存在");
}
config.setCreateBy(SecurityUtils.getUsername());
@ -103,24 +97,35 @@ public class SysConfigController extends BaseController
@RequiresPermissions("system:config:edit")
@Log(title = "参数管理", businessType = BusinessType.UPDATE)
@PutMapping
public AjaxResult edit(@Validated @RequestBody SysConfig config)
{
if (UserConstants.NOT_UNIQUE.equals(configService.checkConfigKeyUnique(config)))
{
public AjaxResult edit(@Validated @RequestBody SysConfig config) {
if (UserConstants.NOT_UNIQUE.equals(configService.checkConfigKeyUnique(config))) {
return AjaxResult.error("修改参数'" + config.getConfigName() + "'失败，参数键名已存在");
}
config.setUpdateBy(SecurityUtils.getUsername());
return toAjax(configService.updateConfig(config));
}
/**
* 根据key修改参数配置
* @return r
* @since 2022-02-21
*/
@PutMapping("editForRPC")
public R editForRPC(@RequestParam("key") String key, @RequestParam("value") String value) {
SysConfig sysConfig = new SysConfig();
sysConfig.setConfigKey(key);
sysConfig.setConfigValue(value);
int i = configService.updateConfigByKey(sysConfig);
return i == 1 ? R.ok() : R.fail();
}
/**
* 删除参数配置
*/
@RequiresPermissions("system:config:remove")
@Log(title = "参数管理", businessType = BusinessType.DELETE)
@DeleteMapping("/{configIds}")
public AjaxResult remove(@PathVariable Long[] configIds)
{
public AjaxResult remove(@PathVariable Long[] configIds) {
configService.deleteConfigByIds(configIds);
return success();
}
@ -131,8 +136,7 @@ public class SysConfigController extends BaseController
@RequiresPermissions("system:config:remove")
@Log(title = "参数管理", businessType = BusinessType.CLEAN)
@DeleteMapping("/refreshCache")
public AjaxResult refreshCache()
{
public AjaxResult refreshCache() {
configService.resetConfigCache();
return AjaxResult.success();
}

11
ruoyi-modules/ruoyi-system/src/main/java/com/ruoyi/system/mapper/SysConfigMapper.java
Unescape View File

@ -1,8 +1,9 @@
package com.ruoyi.system.mapper;
import java.util.List;
import com.ruoyi.system.domain.SysConfig;
import java.util.List;
/**
* 参数配置 数据层
*
@ -65,4 +66,12 @@ public interface SysConfigMapper
* @return 结果
*/
public int deleteConfigByIds(Long[] configIds);
/**
* 根据key修改参数配置
* @param config 配置实体
* @return int
* @since 2022-02-21
*/
int updateConfigByKey(SysConfig config);
}

11
ruoyi-modules/ruoyi-system/src/main/java/com/ruoyi/system/service/ISysConfigService.java
Unescape View File

@ -1,8 +1,9 @@
package com.ruoyi.system.service;
import java.util.List;
import com.ruoyi.system.domain.SysConfig;
import java.util.List;
/**
* 参数配置 服务层
*
@ -80,4 +81,12 @@ public interface ISysConfigService
* @return 结果
*/
public String checkConfigKeyUnique(SysConfig config);
/**
* 根据key修改参数配置
* @param config 配置实体
* @return int
* @since 2022-02-21
*/
int updateConfigByKey(SysConfig config);
}

20
ruoyi-modules/ruoyi-system/src/main/java/com/ruoyi/system/service/impl/SysConfigServiceImpl.java
Unescape View File

@ -1,10 +1,5 @@
package com.ruoyi.system.service.impl;
import java.util.Collection;
import java.util.List;
import javax.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.ruoyi.common.core.constant.Constants;
import com.ruoyi.common.core.constant.UserConstants;
import com.ruoyi.common.core.exception.ServiceException;
@ -14,6 +9,12 @@ import com.ruoyi.common.redis.service.RedisService;
import com.ruoyi.system.domain.SysConfig;
import com.ruoyi.system.mapper.SysConfigMapper;
import com.ruoyi.system.service.ISysConfigService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.Collection;
import java.util.List;
/**
* 参数配置 服务层实现
@ -195,6 +196,15 @@ public class SysConfigServiceImpl implements ISysConfigService
return UserConstants.UNIQUE;
}
@Override
public int updateConfigByKey(SysConfig config) {
int row = configMapper.updateConfigByKey(config);
if (row > 0) {
redisService.setCacheObject(getCacheKey(config.getConfigKey()), config.getConfigValue());
}
return row;
}
/**
* 设置cache key
*

8
ruoyi-modules/ruoyi-system/src/main/resources/mapper/system/SysConfigMapper.xml
Unescape View File

@ -97,7 +97,13 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
</set>
where config_id = #{configId}
</update>
<update id="updateConfigByKey" parameterType="SysConfig">
update sys_config
set config_value =#{configValue}
where config_key =#{configKey}
</update>
<delete id="deleteConfigById" parameterType="Long">
delete from sys_config where config_id = #{configId}
</delete>

28
ruoyi-ui/src/api/business/webmagic/_36wallpaper/wallpaper36.js
Unescape View File

@ -0,0 +1,28 @@
import request from '@/utils/request'
// 获取参数配置
export function getSettings() {
return request({
url: '/webmagic/_36wallpaper/getSettings',
method: 'get',
})
}
// 修改参数配置
export function updateSettings(json) {
return request({
url: '/webmagic/_36wallpaper/updateSettings',
method: 'put',
params: json
})
}
// 重置参数配置
export function resetSettings() {
return request({
url: '/webmagic/_36wallpaper/reset',
method: 'put',
})
}

122
ruoyi-ui/src/views/business/webmagic/_36wallpaper/settings/index.vue
Unescape View File

@ -0,0 +1,122 @@
<template>
<div>
<el-row>
<el-col :span="24">
<div class="top_col">
<span style="cursor: pointer;font-size: 14px">参数配置</span>
</div>
</el-col>
</el-row>
<el-row :gutter="15">
<el-form ref="elForm" :model="formData" :rules="rules" size="medium" label-width="100px">
<el-col :span="3">
<el-form-item label="下载图片" prop="downloadImg" required>
<el-switch v-model="formData.downloadImg" active-color="#3292CD" inactive-color="#D74747">
</el-switch>
</el-form-item>
</el-col>
<el-col :span="3">
<el-form-item label="初始化爬取" prop="init" required>
<el-switch v-model="formData.init" active-color="#3292CD" inactive-color="#D74747"></el-switch>
</el-form-item>
</el-col>
<el-col :span="7">
<el-form-item label="路径" prop="path">
<el-input v-model="formData.path" placeholder="请输入下载图片路径" :maxlength="200" clearable
prefix-icon='el-icon-s-tools' :style="{width: '100%'}"></el-input>
</el-form-item>
</el-col>
<el-col :span="9">
<el-form-item>
<el-button size="mini" type="primary" @click="submitForm">提交</el-button>
<el-button size="mini" @click="resetForm">重置</el-button>
<el-button size="mini" type="info" icon="el-icon-refresh" @click="resetSettings">恢复默认</el-button>
</el-form-item>
</el-col>
</el-form>
</el-row>
</div>
</template>
<script>
import {getSettings,updateSettings,resetSettings} from '@/api/business/webmagic/_36wallpaper/wallpaper36'
export default {
name: "Wallpaper_36Settings",
data() {
return {
//参数信息
formData: {
downloadImg: false,
init: false,
path: undefined,
},
rules: {
path: [{
required: true,
message: '请输入下载图片路径路径',
trigger: 'blur'
}],
},
}
},
created() {
this.getSettings()
},
methods: {
//获取参数信息
getSettings() {
getSettings().then(res => {
this.formData = res.data
})
},
//恢复默认（重置）
resetSettings() {
resetSettings().then(res =>{
this.$modal.msgSuccess("重置成功");
this.getSettings()
})
},
submitForm() {
this.$refs['elForm'].validate(valid => {
if (!valid) return
let json ={
json:null
}
json.json = JSON.stringify(this.formData);
updateSettings(json).then(res =>{
this.$modal.msgSuccess("修改成功");
})
})
},
resetForm() {
this.$refs['elForm'].resetFields()
},
}
}
</script>
<style scoped>
.top_col {
height: 40px;
margin: 10px 10px;
background-color: #13C2C2;
line-height: 40px;
padding-left: 20px;
border-radius: 17px;
font-size: 13px;
}
</style>

13
ruoyi-ui/src/views/business/webmagic/_36wallpaper/show/index.vue
Unescape View File

@ -0,0 +1,13 @@
<template>
</template>
<script>
export default {
name: "_36wallpaperShow"
}
</script>
<style scoped>
</style>

5
xjs-business/xjs-business-common/src/main/java/com/xjs/consts/RegexConst.java
Unescape View File

@ -32,4 +32,9 @@ public class RegexConst {
* 数字校验正则
*/
public static final String NUMBER_REGEX= "[0-9]*";
/**
* 文件路径正则
*/
public static final String FILE_PATH_REGEX= "^[a-zA-Z]:(((\\\\(?! )[^/:*?<>\\\"\"|\\\\]+)+\\\\?)|(\\\\)?)\\s*$";
}

35
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/consts/_36wallpaperConst.java
Unescape View File

@ -0,0 +1,35 @@
package com.xjs._36wallpaper.consts;
/**
* 36壁纸网配置参数常量类
* @author xiejs
* @since 2022-02-21
*/
public class _36wallpaperConst {
/**
* 是否全网爬虫
*/
public static boolean INIT = false;
/**
* 是否下载图片带磁盘
*/
public static boolean DOWNLOAD_IMG = false;
/**
* 图片保存到磁盘的路径
*/
public static String PATH = "D:\\Dev\\WebCrawler\\36wallpaper";
/**
* redis的key
*/
public static final String REDIS_KEY = "sys_config:xjs.webmagic._36wallpaper";
/**
* 系统配置表中的key
*/
public static final String CONFIG_KEY = "xjs.webmagic._36wallpaper";
}

42
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/controller/_36wallpaperController.java
Unescape View File

@ -1,27 +1,61 @@
package com.xjs._36wallpaper.controller;
import com.alibaba.fastjson.JSONObject;
import com.ruoyi.common.core.domain.R;
import com.ruoyi.common.core.web.domain.AjaxResult;
import com.xjs._36wallpaper.service._36wallpaperService;
import com.xjs._36wallpaper.task._36wallpaperTask;
import com.xjs.web.MyBaseController;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import java.util.Objects;
/**
* 36壁纸网爬虫controller
*
* @author xiejs
* @since 2022-02-20
*/
@RestController
@RequestMapping("_36wallpaper")
@Api(tags = "爬虫模块-36壁纸网")
public class _36wallpaperController {
public class _36wallpaperController extends MyBaseController {
@Autowired
private _36wallpaperTask wallpaperTask;
@Autowired
private _36wallpaperService wallpaperService;
@GetMapping("getSettings")
@ApiOperation("获取参数配置")
public AjaxResult getSettings() {
JSONObject jsonObject = wallpaperService.getSettings();
if (Objects.nonNull(jsonObject)) {
return AjaxResult.success(jsonObject);
}
return AjaxResult.error();
}
@PutMapping("updateSettings")
@ApiOperation("修改参数配置")
public AjaxResult updateSettings(@RequestParam("json") String json) {
boolean b=wallpaperService.updateSettings(json);
return toAjax(b);
}
@PutMapping("reset")
@ApiOperation("重置参数配置")
public AjaxResult resetSettings() {
boolean b=wallpaperService.resetSettings();
return toAjax(b);
}
//----------------------远程rpc调用---------------------------
@GetMapping("taskForPRC")

22
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/service/_36wallpaperService.java
Unescape View File

@ -1,5 +1,6 @@
package com.xjs._36wallpaper.service;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.extension.service.IService;
import com.xjs._36wallpaper.pojo._36wallpaper;
@ -15,4 +16,25 @@ public interface _36wallpaperService extends IService<_36wallpaper> {
* @return int
*/
int deleteRepeatData();
/**
* 从配置管理中获取指定的参数配置
* @return json
*/
JSONObject getSettings();
/**
* 修改配置参数
* @param json json
* @return boolean
*/
boolean updateSettings(String json);
/**
* 重置配置参数
* @return boolean
*/
boolean resetSettings();
}

98
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/service/impl/_36wallpaperServiceImpl.java
Unescape View File

@ -1,12 +1,23 @@
package com.xjs._36wallpaper.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.ruoyi.common.core.constant.HttpStatus;
import com.ruoyi.common.core.domain.R;
import com.ruoyi.common.redis.service.RedisService;
import com.ruoyi.system.api.RemoteConfigService;
import com.xjs._36wallpaper.mapper._36wallpaperMapper;
import com.xjs._36wallpaper.pojo._36wallpaper;
import com.xjs._36wallpaper.service._36wallpaperService;
import com.xjs.exception.BusinessException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.regex.Pattern;
import static com.xjs._36wallpaper.consts._36wallpaperConst.*;
import static com.xjs.consts.RegexConst.FILE_PATH_REGEX;
/**
* 36壁纸网service实现
@ -20,8 +31,95 @@ public class _36wallpaperServiceImpl extends ServiceImpl<_36wallpaperMapper, _36
@Resource
private _36wallpaperMapper wallpaperMapper;
@Resource
private RemoteConfigService remoteConfigService;
@Autowired
private RedisService redisService;
@Override
public int deleteRepeatData() {
return wallpaperMapper.deleteRepeatData();
}
@Override
public JSONObject getSettings() {
JSONObject json;
//先从redis中查，没有就远程调用系统配置服务查数据库
if (redisService.hasKey(REDIS_KEY)) {
String cacheObject = redisService.getCacheObject(REDIS_KEY);
try {
json = JSONObject.parseObject(cacheObject);
return json;
} catch (Exception e) {
log.error("json格式转换异常："+e.getMessage());
}
}
R<String> r = remoteConfigService.getConfigKeyForRPC(CONFIG_KEY);
if (r.getCode() == HttpStatus.SUCCESS) {
try {
json = JSONObject.parseObject(r.getData());
return json;
} catch (Exception e) {
log.error("json格式转换异常："+e.getMessage());
}
}
return null;
}
@Override
public boolean updateSettings(String json) {
//校验json格式是否正确
try {
JSONObject jsonObject = JSONObject.parseObject(json);
String downloadImg = "downloadImg";
String path = "path";
String init = "init";
if (jsonObject.containsKey(init) && jsonObject.containsKey(downloadImg) && jsonObject.containsKey(path)) {
//校验json内的参数key是否正确
jsonObject.getBoolean(init);
jsonObject.getBoolean(downloadImg);
String pathValue = jsonObject.getString(path);
//校验磁盘地址是否正确
boolean matches = Pattern.matches(FILE_PATH_REGEX, pathValue);
if (!matches) {
throw new BusinessException("文件路径格式不正确");
}
R r = remoteConfigService.editForRPC(CONFIG_KEY, json);
if (r.getCode() == HttpStatus.SUCCESS) {
return true;
}
}else {
throw new BusinessException("json属性不正确,必须包含 init、path、downloadImg 三个属性");
}
} catch (Exception e) {
log.error("json格式转换异常"+e.getMessage());
throw new BusinessException("json格式转换异常" + e.getMessage());
}
return false;
}
@Override
public boolean resetSettings() {
//构建json
JSONObject jsonObject = new JSONObject();
jsonObject.put("path",PATH);
jsonObject.put("init",INIT);
jsonObject.put("downloadImg",DOWNLOAD_IMG);
R r = remoteConfigService.editForRPC(CONFIG_KEY, jsonObject.toJSONString());
if (r.getCode() == HttpStatus.SUCCESS) {
return true;
}
return false;
}
}

25
xjs-business/xjs-business-webmagic/src/main/java/com/xjs/_36wallpaper/webmagic/_36wallpaperProcessor.java
Unescape View File

@ -3,6 +3,7 @@ package com.xjs._36wallpaper.webmagic;
import com.alibaba.fastjson.JSONObject;
import com.ruoyi.common.redis.service.RedisService;
import com.ruoyi.system.api.RemoteConfigService;
import com.xjs._36wallpaper.consts._36wallpaperConst;
import com.xjs._36wallpaper.pojo._36wallpaper;
import com.xjs._36wallpaper.service._36wallpaperService;
import lombok.extern.log4j.Log4j2;
@ -25,6 +26,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static com.xjs._36wallpaper.consts._36wallpaperConst.CONFIG_KEY;
import static com.xjs._36wallpaper.consts._36wallpaperConst.REDIS_KEY;
import static com.xjs.consts.RedisConst.REPTILE_COUNT;
import static com.xjs.consts.ReptileConst._36_WALLPAPER_URL;
@ -42,27 +45,17 @@ public class _36wallpaperProcessor implements PageProcessor {
/**
* 是否全网爬虫
*/
private boolean init = false;
private boolean init ;
/**
* 是否下载图片带磁盘
*/
private boolean downloadImg = false;
private boolean downloadImg;
/**
* 图片保存到磁盘的路径
*/
private String path = "D:\\Dev\\WebCrawler\\36wallpaper";
/**
* redis的key
*/
public static final String REDIS_KEY = "sys_config:xjs.webmagic._36wallpaper";
/**
* 系统配置表中的key
*/
public static final String CONFIG_KEY = "xjs.webmagic._36wallpaper";
private String path ;
/**
* 请求头key
@ -104,6 +97,11 @@ public class _36wallpaperProcessor implements PageProcessor {
* 初始化参数
*/
private void initParameter() {
//先赋予默认值
this.init= _36wallpaperConst.INIT;
this.path= _36wallpaperConst.PATH;
this.downloadImg= _36wallpaperConst.DOWNLOAD_IMG;
//判断redis中是否存在
Boolean hasKey = redisService.hasKey(REDIS_KEY);
JSONObject json;
@ -135,7 +133,6 @@ public class _36wallpaperProcessor implements PageProcessor {
} catch (Exception e) {
log.error("JSON转换异常:"+e.getMessage());
}
}
}

Write Preview
Loading…
Cancel
Save