正则采集器之二——后台搭建

本人是java程序员，所以使用java界流行的架构来搭建后台。后台使用java作为后端语言，mysql作为数据库，redis作为缓存中间件。采用SpringBoot作为java端依赖管理、bean生命周期管理的容器，mybatis作为数据库持久化框架，shiro作为鉴权框架。

1、新建maven项目

命名为reptile，接着加入依赖:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"><parent><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-parent</artifactId><version>2.4.2</version></parent><modelVersion>4.0.0</modelVersion><groupId>com.learn.reptile</groupId><artifactId>reptile</artifactId><version>0.0.1-SNAPSHOT</version><properties><java.version>1.8</java.version><mybatis.plus.version>3.4.2</mybatis.plus.version><druid.version>1.1.18</druid.version><mysql.version>8.0.13</mysql.version><!--<log4j.version>1.2.17</log4j.version>--><hutool.version>5.5.8</hutool.version><guava.version>28.1-jre</guava.version><shiro.version>1.13.0</shiro.version><httpclient.version>4.5.3</httpclient.version></properties><dependencies><!-- Druid引入 --><dependency><groupId>com.alibaba</groupId><artifactId>druid-spring-boot-starter</artifactId><version>${druid.version}</version></dependency><!-- 数据源驱动 --><dependency><groupId>mysql</groupId><artifactId>mysql-connector-java</artifactId><version>${mysql.version}</version></dependency><!-- mybatis-plus依赖 --><dependency><groupId>com.baomidou</groupId><artifactId>mybatis-plus-boot-starter</artifactId><version>${mybatis.plus.version}</version></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-web</artifactId><exclusions><exclusion><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-logging</artifactId></exclusion></exclusions></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-log4j2</artifactId><version>2.4.2</version></dependency><!-- spring-boot-starter-data-redis --><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-data-redis</artifactId></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-aop</artifactId></dependency><dependency><groupId>redis.clients</groupId><artifactId>jedis</artifactId></dependency><dependency><groupId>org.projectlombok</groupId><artifactId>lombok</artifactId><!--            <optional>true</optional>--></dependency><!-- guava支持 --><dependency><groupId>com.google.guava</groupId><artifactId>guava</artifactId><version>${guava.version}</version></dependency><dependency><groupId>com.alibaba.fastjson2</groupId><artifactId>fastjson2</artifactId><version>2.0.13</version></dependency><dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-devtools</artifactId><scope>runtime</scope><optional>true</optional></dependency><!--        shiro 核心类库--><dependency><groupId>org.apache.shiro</groupId><artifactId>shiro-core</artifactId><version>${shiro.version}</version></dependency><!-- shiro 整合sping --><dependency><groupId>org.apache.shiro</groupId><artifactId>shiro-spring</artifactId><version>${shiro.version}</version></dependency><!-- 日志框架依赖 --><!--<dependency><groupId>log4j</groupId><artifactId>log4j</artifactId><version>${log4j.version}</version></dependency>--><dependency><groupId>org.apache.httpcomponents</groupId><artifactId>httpclient</artifactId><version>${httpclient.version}</version></dependency><dependency><groupId>cn.hutool</groupId><artifactId>hutool-all</artifactId><version>${hutool.version}</version></dependency><!--        okhttp依赖--><dependency><groupId>com.squareup.okhttp3</groupId><artifactId>okhttp</artifactId><version>4.9.1</version></dependency></dependencies><build><resources><resource><directory>src/main/resources</directory><filtering>true</filtering></resource><resource><directory>src/main/java</directory><includes><include>**/*.xml</include></includes></resource></resources><plugins><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-compiler-plugin</artifactId><configuration><source>8</source><target>8</target></configuration></plugin><plugin><groupId>org.springframework.boot</groupId><artifactId>spring-boot-maven-plugin</artifactId><configuration><includeSystemScope>true</includeSystemScope></configuration><executions><execution><goals><goal>repackage</goal></goals></execution></executions></plugin></plugins></build>
</project>

接着新建包

• com.learn.reptile.config 配置文件
• com.learn.reptile.entity 实体类
• com.learn.reptile.mapper 数据库操作接口
• com.learn.reptile.utils 工具类
• com.learn.reptile.web web层类

新建启动类 ReptileApplication

package com.learn.reptile;import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;@SpringBootApplication(scanBasePackages = "com.learn.reptile")
public class ReptileApplication {public static void main(String[] args) {SpringApplication.run(ReptileApplication.class, args);}}

2、配置文件

在src/main/resources下新建配置文件application.yml，内容如下:

server:port: 8080spring:application:name: reptilemain:allow-bean-definition-overriding: truejackson:date-format: yyyy-MM-dd HH:mmtime-zone: GMT+8servlet:multipart:max-file-size: 100MBmax-request-size: 100MBredis:database: 1host: sincerely-hostpassword: 85861367LiGzydatasource:username: testpassword: testurl: jdbc:mysql://localhost:3306/reptile?useUnicode=true&useSSL=false&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&allowMultiQueries=true&allowPublicKeyRetrieval=truedriver-class-name: com.mysql.cj.jdbc.Driverinitial-size: 2min-idle: 1max-active: 20max-wait: 60000timeBetweenEvictionRunsMillis: 60000minEvictableIdleTimeMillis: 300000validationQuery: "select 'x'"testWhileIdle: truetestOnBorrow: falsetestOnReturn: falsepoolPreparedStatements: truemaxPoolPreparedStatementPerConnectionSize: 20filters: statmybatis-plus:global-config:id-type: 0db-column-underline: falserefresh-mapper: truetype-aliases-package: com.learn.reptile.entity.pomapper-locations: classpath:/mapper/*.xmlconfiguration:log-impl: org.apache.ibatis.logging.stdout.StdOutImpllogging:config: classpath:log4j2.xmlshiro:session-timeout: 1000000

log4j2.xml定义好日志级别、记录位置和格式

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" monitorInterval="30"><Properties><Property name="LOG_PATTERN">%d{yyyy-MM-dd HH:mm:ss.SSS} %5p %logger{36} --- [%15.15t] %l : %m%n</Property></Properties><Appenders><Console name="Console" target="SYSTEM_OUT" follow="true"><PatternLayout pattern="${LOG_PATTERN}"/></Console><RollingRandomAccessFile name="DebugFile"fileName="logs/debug.log"filePattern="logs/debug-%d{yyyy-MM-dd}.log"><PatternLayout pattern="${LOG_PATTERN}"/><Policies><TimeBasedTriggeringPolicy interval="1" modulate="true" /><SizeBasedTriggeringPolicy size="10MB" /></Policies><!-- Max 10 files will be created everyday --><DefaultRolloverStrategy max="10"><Delete basePath="logs" maxDepth="10"><!-- Delete all files older than 30 days --><IfLastModified age="30d" /></Delete></DefaultRolloverStrategy></RollingRandomAccessFile><Async name="AsyncDebugFile"><AppenderRef ref="DebugFile" /></Async><RollingRandomAccessFile name="InfoFile"fileName="logs/info.log"filePattern="logs/info-%d{yyyy-MM-dd}.log"><PatternLayout pattern="${LOG_PATTERN}"/><Policies><TimeBasedTriggeringPolicy interval="1" modulate="true" /><SizeBasedTriggeringPolicy size="10MB" /></Policies><!-- Max 10 files will be created everyday --><DefaultRolloverStrategy max="10"><Delete basePath="logs" maxDepth="10"><!-- Delete all files older than 30 days --><IfLastModified age="30d" /></Delete></DefaultRolloverStrategy></RollingRandomAccessFile><Async name="AsyncInfoFile"><AppenderRef ref="InfoFile" /></Async><RollingRandomAccessFile name="WarnFile"fileName="logs/warn.log"filePattern="logs/warn-%d{yyyy-MM-dd}.log"><PatternLayout pattern="${LOG_PATTERN}"/><Policies><TimeBasedTriggeringPolicy interval="1" modulate="true" /><SizeBasedTriggeringPolicy size="10MB" /></Policies><!-- Max 10 files will be created everyday --><DefaultRolloverStrategy max="10"><Delete basePath="logs" maxDepth="10"><!-- Delete all files older than 30 days --><IfLastModified age="30d" /></Delete></DefaultRolloverStrategy></RollingRandomAccessFile><Async name="AsyncWarnFile"><AppenderRef ref="WarnFile" /></Async><RollingRandomAccessFile name="ErrorFile"fileName="logs/error.log"filePattern="logs/error-%d{yyyy-MM-dd}.log"><PatternLayout pattern="${LOG_PATTERN}"/><Policies><TimeBasedTriggeringPolicy interval="1" modulate="true" /><SizeBasedTriggeringPolicy size="10MB" /></Policies><!-- Max 10 files will be created everyday --><DefaultRolloverStrategy max="10"><Delete basePath="logs" maxDepth="10"><!-- Delete all files older than 30 days --><IfLastModified age="30d" /></Delete></DefaultRolloverStrategy></RollingRandomAccessFile><Async name="AsyncErrorFile"><AppenderRef ref="ErrorFile" /></Async></Appenders><Loggers><Logger name="com.learn.reptile" level="debug" additivity="false"><AppenderRef ref="Console" /><AppenderRef ref="AsyncDebugFile" /></Logger><Root level="info"><AppenderRef ref="Console" /><AppenderRef ref="AsyncDebugFile" /><AppenderRef ref="AsyncInfoFile" /><AppenderRef ref="AsyncWarnFile" /><AppenderRef ref="AsyncErrorFile" /></Root></Loggers>
</Configuration>

github代码：GitHub - guzhangyu/reptile-api: 商品正则采集器