Common JAVA postures and methods collected by programmers who have worked for three years

Serialization and deserialization of objects

<dependency>
    <groupId>org.javatuples</groupId>
    <artifactId>javatuples</artifactId>
    <version>1.2</version>
</dependency>
package com.cqcdi.securitybrain.vulnerability_scanner.utils;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.TreeNode;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.*;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.TextNode;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

public final class Json {

    private static final Logger LOGGER = LoggerFactory.getLogger(Json.class);
    private static final String EMPTY_JSON_OBJECT = "{}";
    public static final ObjectMapper MAPPER = new ObjectMapper();

    /**
     * Date format parameter format yyyy MM DD HH: mm: SS
     */
    public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");

    static {
        SimpleModule simpleModule = new SimpleModule();
        simpleModule.addDeserializer(LocalDateTime.class, new LocalDatetimeDeserializer());
        simpleModule.addSerializer(LocalDateTime.class, new LocalDatetimeSerializer());
        simpleModule.addSerializer(Timestamp.class, new TimestampSerializer());
        MAPPER.registerModule(simpleModule);
        MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
        MAPPER.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE);
    }

    /**
     * Object to Json
     *
     * @param object Object
     * @return String
     */
    public static String toJson(Object object) {
        StringWriter sw = new StringWriter();
        if (Objects.isNull(object)) {
            return EMPTY_JSON_OBJECT;
        }
        try {
            MAPPER.writeValue(MAPPER.getFactory().createGenerator(sw), object);
        } catch (Exception ex) {
            LOGGER.error("Object to json occur error, detail:", ex);
        }
        return sw.toString();
    }

    /**
     * Json Convert to object
     *
     * @param jsonString json
     * @param tClass     type
     * @param <T>        Generic parameters
     * @return T
     */
    public static <T> T toObject(String jsonString, Class<T> tClass) {
        if (Objects.isNull(jsonString) || jsonString.trim().isEmpty()) {
            return null;
        }
        try {
            return MAPPER.readValue(jsonString, tClass);
        } catch (Exception ex) {
            LOGGER.error("Json to object occur error, detail:", ex);
        }
        return null;
    }

    /**
     * Json Convert to object
     *
     * @param fileName File name
     * @param tClass   type
     * @param <T>      Generic parameters
     * @return T
     */
    public static <T> T fileToObject(String fileName, Class<T> tClass) {
        if (Objects.isNull(fileName) || fileName.trim().isEmpty()) {
            return null;
        }
        try {
            return MAPPER.readValue(new File(fileName), tClass);
        } catch (Exception ex) {
            LOGGER.error("Json to object occur error, detail:", ex);
        }
        return null;
    }

    /**
     * Json Convert to object
     *
     * @param <T>        generic paradigm
     * @param jsonString json
     * @param reference  Type reference
     * @return T
     */
    public static <T> T toObject(String jsonString, TypeReference<T> reference) {
        if (Objects.isNull(jsonString) || jsonString.trim().isEmpty()) {
            return null;
        }
        try {
            return MAPPER.readValue(jsonString, reference);
        } catch (Exception ex) {
            LOGGER.error("Json to object occur error, detail:", ex);
        }
        return null;
    }

    /**
     * Convert input stream to object
     *
     * @param <T>       generic paradigm
     * @param stream    Input stream
     * @param reference Type reference
     * @return T
     */
    public static <T> T toObject(InputStream stream, TypeReference<T> reference) {
        if (Objects.isNull(stream)) {
            return null;
        }
        try {
            return MAPPER.readValue(stream, reference);
        } catch (Exception ex) {
            LOGGER.error("Json to object occur error, detail:", ex);
        }
        return null;
    }

    /**
     * Convert input stream to object
     *
     * @param <T>    generic paradigm
     * @param stream Input stream
     * @param tClass Type reference
     * @return T
     */
    public static <T> T toObject(InputStream stream, Class<T> tClass) {
        if (Objects.isNull(stream)) {
            return null;
        }
        try {
            return MAPPER.readValue(stream, tClass);
        } catch (Exception ex) {
            LOGGER.error("Json to object occur error, detail:", ex);
        }
        return null;
    }

    /**
     * JSON String to Map
     *
     * @param json JSON character string
     * @return map
     */
    public static Map<String, Object> toMap(String json) {
        if (Objects.nonNull(json) && !json.isEmpty()) {
            String tmp = json.trim();
            if (tmp.charAt(0) == '{' && tmp.charAt(tmp.length() - 1) == '}') {
                return toObject(json, new TypeReference<Map<String, Object>>() {
                });
            }
        }
        return null;
    }

    private final static Map<String, Map<String, Pair<Field, Method>>> dynamicTypeFieldPairsCache = new HashMap<>();

    /**
     * LocalDatetime Type serialization
     */
    public static class LocalDatetimeSerializer extends JsonSerializer<LocalDateTime> {
        /**
         * Serialize LocalDateTime
         *
         * @param localDateTime      LocalDateTime
         * @param jsonGenerator      JsonGenerator
         * @param serializerProvider SerializerProvider
         * @throws IOException IOException
         */
        @Override
        public void serialize(LocalDateTime localDateTime, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException {
            jsonGenerator.writeString(DATE_TIME_FORMATTER.format(localDateTime));
        }

        /**
         * Get LocalDateTime class
         *
         * @return LocalDateTime.class
         */
        @Override
        public Class<LocalDateTime> handledType() {
            return LocalDateTime.class;
        }
    }

    /**
     * LocalDatetime Type deserialization
     */
    public static class LocalDatetimeDeserializer extends JsonDeserializer<LocalDateTime> {


        /**
         * LocalDateTime Deserialization
         *
         * @param jsonParser             JsonParser
         * @param deserializationContext DeserializationContext
         * @return LocalDateTime
         * @throws IOException IOException
         */
        @Override
        public LocalDateTime deserialize(JsonParser jsonParser, DeserializationContext deserializationContext)
                throws IOException {
            TreeNode tree = jsonParser.getCodec().readTree(jsonParser);
            if (tree instanceof TextNode) {
                TextNode node = (TextNode) tree;
                String value = node.textValue();
                return LocalDateTime.parse(value, DATE_TIME_FORMATTER);
            }
            return null;
        }
    }


    /**
     * LocalDatetime Type serialization
     */
    public static class TimestampSerializer extends JsonSerializer<Timestamp> {
        /**
         * Serialize Timestamp
         *
         * @param timestamp          Timestamp
         * @param jsonGenerator      JsonGenerator
         * @param serializerProvider SerializerProvider
         * @throws IOException IOException
         */
        @Override
        public void serialize(Timestamp timestamp, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException {
            LocalDateTime localDateTime = null;
            if (Objects.nonNull(timestamp)) {
                localDateTime = timestamp.toLocalDateTime();
            }
            jsonGenerator.writeString(DATE_TIME_FORMATTER.format(localDateTime));
        }

        /**
         * Get Timestamp class
         *
         * @return Timestamp.class
         */
        @Override
        public Class<Timestamp> handledType() {
            return Timestamp.class;
        }
    }
}

With this, you can do json to object, json to HashMap, HashMap to object, which is more practical in serialization and deserialization.

Conversion and processing of timestamp, time and time string

public class DatetimeUtil {

    public static LocalDateTime getDateTimeOfTimestamp(long timestamp) {
        Instant instant = Instant.ofEpochMilli(timestamp);
        ZoneId zone = ZoneId.systemDefault();
        return LocalDateTime.ofInstant(instant, zone);
    }

    public static String getViewStrOfDatetime(LocalDateTime localDateTime) {
        return localDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
    }

    public static String getStrofTimestamp(long timestamp) {
        return getViewStrOfDatetime(getDateTimeOfTimestamp(timestamp));
    }

    public static Long getTimestampOfDatetime(LocalDateTime localDateTime) {
        return localDateTime.toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
    }

    public static Long getTimeStampOfViewStr(String value) {
        return getTimestampOfDatetime(LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
    }

    public static LocalDateTime getLocalDatetimeByStr(String value){
        return DatetimeUtil.getDateTimeOfTimestamp(DatetimeUtil.getTimeStampOfViewStr(value));
    }
}

Similarly, there are time stamps, date and date string conversion and processing

public class DateUtil {

    public static LocalDate getDateTimeOfTimestamp(long timestamp) {
        return Instant.ofEpochMilli(timestamp).atZone(ZoneId.systemDefault()).toLocalDate();
    }

    public static String getViewStrOfDatetime(LocalDate localDateTime) {
        return localDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
    }

    public static String getStrofTimestamp(long timestamp) {
        return getViewStrOfDatetime(getDateTimeOfTimestamp(timestamp));
    }

    public static Long getTimestampOfDatetime(LocalDate localDateTime) {
        return localDateTime.atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli();
    }

    public static Long getTimeStampOfViewStr(String value) {
        try {
            return getTimeStampOfViewStrWithFormat(value, "yyyy-MM-dd");
        }catch (Exception e){
//            e.printStackTrace();
        }
        try {
            return getTimeStampOfViewStrWithFormat(value, "yyyy/MM/dd");
        }catch (Exception e){
//            e.printStackTrace();
        }
        try {
            return getTimeStampOfViewStrWithFormat(value, "yyyy year MM month dd day");
        }catch (Exception e){

        }
        try {
            if (value.length() == 4){
                StringBuilder x = new StringBuilder(value).append("-12-30");
                return getTimeStampOfViewStrWithFormat(x.toString(), "yyyy-MM-dd");
            }
        }catch (Exception e){

        }
        return getTimeStampOfViewStrWithFormat(value, "yyyy year");
    }

    public static Long getTimeStampOfViewStrWithFormat(String value, String format) {
        return getTimestampOfDatetime(LocalDate.parse(value, DateTimeFormatter.ofPattern(format)));
    }


    public static LocalDate getLocalDatetimeByStr(String value) {
        return DateUtil.getDateTimeOfTimestamp(DateUtil.getTimeStampOfViewStr(value));
    }

    public static Integer getWeekWithDate(LocalDate localDate){
        return localDate.get(WeekFields.of(DayOfWeek.MONDAY, 1).weekOfYear());
    }
}

At the same time, it is necessary to convert objects between time and date.

public static LocalDate time2Date(LocalDateTime localDateTime){
    return getLocalDatetimeByStr(DatetimeUtil.getViewStrOfDatetime(localDateTime).split(" ")[0]);
}

public static LocalDateTime date2Time(LocalDate localDate){
    return DatetimeUtil.getLocalDatetimeByStr(getViewStrOfDatetime(localDate)+" 00:00:00");
}

With these methods, you can easily transform in the field of time. Simple and fast.

Conversion and processing between string and array

In the actual coding process, we sometimes use strings to represent the value of the array, or we may convert it back.

public class ArrayStrUtil {

    public static List<String> str2Array(String str) {
        List<String> ans = new ArrayList<>();
        if (Objects.nonNull(str)) {
            String[] tmp = str.split(",");
            for (String x : tmp) {
                if (Objects.nonNull(x) && StringUtils.hasText(x)) {
                    ans.add(x);
                }
            }
        }
        return ans;
    }

    public static List<Long> str2LArray(String str) {
        List<Long> ans = new ArrayList<>();
        List<String> tmp = str2Array(str);
        for (String x : tmp) {
            ans.add(Long.parseLong(x));
        }
        return ans;
    }

    public static List<Integer> str2IArray(String str) {
        List<Integer> ans = new ArrayList<>();
        List<String> tmp = str2Array(str);
        for (String x : tmp) {
            ans.add(Integer.parseInt(x));
        }
        return ans;
    }

    public static Set<String> str2Set(String str) {
        Set<String> ans = new HashSet<>();
        if (Objects.nonNull(str)) {
            String[] tmp = str.split(",");
            for (String x : tmp) {
                if (Objects.nonNull(x)) {
                    ans.add(x);
                }
            }
        }
        return ans;
    }

    public static String slist2Str(List<String> a, String b) {
        StringBuilder ans = new StringBuilder();
        int cnt = 0;
        for (String x : a) {
            if (cnt == 0) {
                ans.append(x);
            } else {
                ans.append(b).append(x);
            }
            cnt++;
        }
        return ans.toString();
    }

    public static String llist2Str(List<Long> a, String b) {
        StringBuilder ans = new StringBuilder();
        int cnt = 0;
        for (Long x : a) {
            if (cnt == 0) {
                ans.append(x);
            } else {
                ans.append(b).append(x);
            }
            cnt++;
        }
        return ans.toString();
    }

    public static String ilist2Str(List<Integer> a, String b) {
        StringBuilder ans = new StringBuilder();
        int cnt = 0;
        for (Integer x : a) {
            if (cnt == 0) {
                ans.append(x);
            } else {
                ans.append(b).append(x);
            }
            cnt++;
        }
        return ans.toString();
    }

}

This tool can be supplemented according to their own needs, reduce code writing, and increase efficiency.

Handling domain names and ip addresses

public final class DomainUtil {


    public static String getTopDomain(String url) {
        String RE_TOP =
                "[\\w-]+\\.(com.cn|net.cn|gov.cn|org\\.nz|org.cn|com|net|org|gov|cc|biz|info|cn|co|edu.cn|ac.cn|mil.cn)\\b()*";
        Pattern pattern = Pattern.compile(RE_TOP, Pattern.CASE_INSENSITIVE);
        String result = url;
        try {
            Matcher matcher = pattern.matcher(url);
            if (matcher.find()) {
                result = matcher.group();
            }
        } catch (Exception e) {
            System.out.println("[getTopDomain ERROR]====>");
            e.printStackTrace();
        }
        return result;
    }

    public static Boolean isIpFormat(String ip) {
        String rexp =
                "([1-9]|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])(\\.(\\d|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])){3}";

        Pattern pat = Pattern.compile(rexp);

        Matcher mat = pat.matcher(ip);

        return mat.find();
    }

    public static long ip2Int(String ipString) {
        // Take ip segments
        String[] ipSlices = ipString.split("\\.");
        long rs = 0;
        for (int i = 0; i < ipSlices.length; i++) {
            // Each segment of ip is parsed into int and shifted to the left by 8 bits according to the position
            long intSlice = Long.parseLong(ipSlices[i]) << 8 * i;
            // Or operation
            rs = rs | intSlice;
        }
        return rs;
    }

    public static String getDomain(String url){
        return url.split("//")[1].split("/")[0];
    }
}

These methods are often used in crawler scenes. You need to extract the main domain name and sub domain name, judge whether the currently extracted url is ip type, and so on.

md5 encoding

There is a ready-made library, but you can also write it yourself, which is equivalent to defining the algorithm parameters of md5, which is safer

public class MD5Utils {
    public static String compMd5(String plainText) {
        String re_md5 = new String();
        try {
            MessageDigest md = MessageDigest.getInstance("MD5");
            md.update(plainText.getBytes());
            byte b[] = md.digest();
            int i;
            StringBuffer buf = new StringBuffer("");
            for (int offset = 0; offset < b.length; offset++) {
                i = b[offset];
                if (i < 0)
                    i += 256;
                if (i < 16)
                    buf.append("0");
                buf.append(Integer.toHexString(i));
            }
            re_md5 = buf.toString();
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }
        return re_md5;
    }

    public static String compMd5WithByte(byte[] x) {
        String re_md5 = new String();
        try {
            MessageDigest md = MessageDigest.getInstance("MD5");
            md.update(x);
            byte b[] = md.digest();
            int i;
            StringBuffer buf = new StringBuffer("");
            for (int offset = 0; offset < b.length; offset++) {
                i = b[offset];
                if (i < 0)
                    i += 256;
                if (i < 16)
                    buf.append("0");
                buf.append(Integer.toHexString(i));
            }
            re_md5 = buf.toString();
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }
        return re_md5;
    }
}

In a non spring container, get the component in the spring container

@Component
public class SpringContextUtil1 implements ApplicationContextAware {

    private static ApplicationContext applicationContext;

    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
        SpringContextUtil1.applicationContext = applicationContext;
    }

    public static Object getBean(String beanName) {
        return applicationContext.getBean(beanName);
    }

    public static <T> T getBean(Class<T> beanClass) {
        return applicationContext.getBean(beanClass);
    }

    public static <T> T getBean(String beanName, Class<T> beanClass) {
        return applicationContext.getBean(beanName, beanClass);
    }
}

This is often used. For example, your code does not use @ Component or @ Server annotations. At this time, your use of @ Autoware does not take effect, so you can use this tool class to obtain the contents in the spring container.

Calculate log a as bottom n

public class MathUtils {
    /**
     * Calculate log a as bottom n
     *
     * @param a
     * @param n
     * @return
     */
    public static Double getLogAN(Integer a, Integer n) {
        return Math.log10(n * 1.0) / Math.log10(a * 1.0);
    }
}

This method is mainly used in scenarios where you need to calculate exponential scores.

File processing

Create file recursively

public static void createFileRecursion(String fileName, Integer height) throws IOException {
        Path path = Paths.get(fileName);
        if (Files.exists(path)) {
            // TODO: 2021/11/13 if the file exists
            return;
        }
        if (Files.exists(path.getParent())) {
            // TODO: 2021/11/13 if the parent file exists, create the file directly
            if (height == 0) {
                Files.createFile(path);
            } else {
                Files.createDirectory(path);
            }
        } else {
            createFileRecursion(path.getParent().toString(), height + 1);
            // Todo: this step on November 13, 2021 can ensure that the parent of the path must exist. Now you need to build yourself
            createFileRecursion(fileName, height);
        }
    }

The following is complete

public class GenerateFile {


    public static void createFile(String folder, String fileName, String value) {
        fileName = folder + "/" + fileName;
        try {
            createFileRecursion(fileName, 0);
            BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(fileName, false));
            bufferedOutputStream.write(value.getBytes());
            bufferedOutputStream.flush();
            bufferedOutputStream.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void createFileRecursion(String fileName, Integer height) throws IOException {
        Path path = Paths.get(fileName);
        if (Files.exists(path)) {
            // TODO: 2021/11/13 if the file exists
            return;
        }
        if (Files.exists(path.getParent())) {
            // TODO: 2021/11/13 if the parent file exists, create the file directly
            if (height == 0) {
                Files.createFile(path);
            } else {
                Files.createDirectory(path);
            }
        } else {
            createFileRecursion(path.getParent().toString(), height + 1);
            // Todo: this step on November 13, 2021 can ensure that the parent of the path must exist. Now you need to build yourself
            createFileRecursion(fileName, height);
        }
    }


    public static void createFileWithRelativePath(String folder, String fileName, String value) {
        File directory = new File(".");
        try {
            fileName = directory.getCanonicalPath() + "\\" + folder + "/" + fileName;
            createFileRecursion(fileName, 0);
        } catch (IOException e) {
            e.printStackTrace();
        }
        try {
            BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(fileName, false));
            bufferedOutputStream.write(value.getBytes());
            bufferedOutputStream.flush();
            bufferedOutputStream.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void appendFileWithRelativePath(String folder, String fileName, String value) {
        File directory = new File(".");
        try {
            fileName = directory.getCanonicalPath() + "\\" + folder + "/" + fileName;
            createFileRecursion(fileName, 0);
        } catch (IOException e) {
            e.printStackTrace();
        }
        try {
            BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(fileName, true));
            bufferedOutputStream.write(value.getBytes());
            bufferedOutputStream.flush();
            bufferedOutputStream.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

Send request

For the simplest, you can use resttemplate. Here is a detail that I use more. Only when I master it in my own hands can I have good results.

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpHost;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.*;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.Map;
import java.util.Objects;

public class SendReq {

    private static HttpClient createHttpClient() {
        Protocol easyhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
        Protocol.registerProtocol("https", easyhttps);
        MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
        connectionManager.getParams().setDefaultMaxConnectionsPerHost(1000);
        connectionManager.getParams().setMaxTotalConnections(1000);
        HttpClient httpclient = new HttpClient(connectionManager);
        return httpclient;
    }

    public static ResBody sendReq(String url, String method, Map<String, Object> params, Map<String, String> requestHeaders) {
        GetMethod httpget = null;
        HeadMethod httphead = null;
        PostMethod httpPost = null;
        DeleteMethod httpDelete = null;
        PutMethod httpPut = null;
        HttpClient httpClient = null;
        String rawHeader = "";
        String responce = "";
        String rawResponce = "";
        String responseHeader = "";
        String responseBody = "";
        int code = 0;
        try {
            String line;
            if (method.equalsIgnoreCase("HEAD")) {
                httphead = new HeadMethod(url);
                for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
                    httphead.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
                }
                httphead.setFollowRedirects(true);
                httpClient = createHttpClient();
                code = httpClient.executeMethod(httphead);
                httphead.releaseConnection();
            } else if (method.equalsIgnoreCase("GET")) {
                httpget = new GetMethod(url);
                httpget.setFollowRedirects(true);
                for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
                    httpget.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
                }
                httpClient = createHttpClient();
                code = httpClient.executeMethod(httpget);
                BufferedReader input = new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));
                rawHeader = httpget.getStatusLine() + "\r\n";
                Header[] headers = httpget.getResponseHeaders();
                StringBuffer buf = new StringBuffer();
                for (int a = 0; a < headers.length; ++a) {
                    buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
                }
                rawHeader = rawHeader + buf.toString();
                responseHeader = rawHeader;
                buf = new StringBuffer();

                while ((line = input.readLine()) != null) {
                    buf.append("\r\n" + line);
                }

                responce = buf.toString();
                responseBody = responce;
                input.close();
                rawResponce = rawHeader + responce;
                // Todo: 2021 / 3 / 17 response is the last value
                Header contentType = httpget.getResponseHeader("Content-Type");
                if (contentType != null && contentType.getValue().startsWith("text")) {
                    // Todo: the page will be parsed on March 17, 2021. By default, it will not be parsed first
                }
                Thread.sleep(10L);
                httpget.releaseConnection();
            } else if (method.equalsIgnoreCase("POST") || method.equalsIgnoreCase("PUT") || method.equalsIgnoreCase("DELETE")) {
                BufferedReader input = null;
                Header[] headers = null;
                if (method.equalsIgnoreCase("POST")) {
                    httpPost = new PostMethod(url);
                    for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
                        httpPost.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
                    }
                    String toJson = Json.toJson(params);
                    RequestEntity se = new StringRequestEntity (toJson ,"application/json" ,"UTF-8");

                    httpPost.setRequestEntity(se);
                    httpClient = createHttpClient();
                    code = httpClient.executeMethod(httpPost);
                    input = new BufferedReader(new InputStreamReader(httpPost.getResponseBodyAsStream()));
                    rawHeader = httpPost.getStatusLine() + "\r\n";
                    headers = httpPost.getResponseHeaders();
                } else if (method.equalsIgnoreCase("PUT")) {
                    httpPut = new PutMethod(url);
                    for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
                        httpPut.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
                    }
//                    httpPut.setFollowRedirects(true);
                    String toJson = Json.toJson(params);
                    RequestEntity se = new StringRequestEntity (toJson ,"application/json" ,"UTF-8");
                    httpPut.setRequestEntity(se);
                    httpClient = createHttpClient();
                    code = httpClient.executeMethod(httpPut);
                    input = new BufferedReader(new InputStreamReader(httpPut.getResponseBodyAsStream()));
                    rawHeader = httpPut.getStatusLine() + "\r\n";
                    headers = httpPut.getResponseHeaders();
                } else if (method.equalsIgnoreCase("DELETE")) {
                    httpDelete = new DeleteMethod(url);
                    for (Map.Entry<String, String> requestHeader : requestHeaders.entrySet()) {
                        httpDelete.setRequestHeader(requestHeader.getKey(), requestHeader.getValue());
                    }
//                    httpDelete.setFollowRedirects(true);
                    httpClient = createHttpClient();
                    code = httpClient.executeMethod(httpDelete);
                    input = new BufferedReader(new InputStreamReader(httpDelete.getResponseBodyAsStream()));
                    rawHeader = httpDelete.getStatusLine() + "\r\n";
                    headers = httpDelete.getResponseHeaders();
                }
                StringBuffer buf = new StringBuffer();
                for (int a = 0; a < headers.length; ++a) {
                    buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
                }
                rawHeader = rawHeader + buf.toString();
                responseHeader = rawHeader;
                buf = new StringBuffer();

                while ((line = input.readLine()) != null) {
                    buf.append("\r\n" + line);
                }
                responce = buf.toString();
                responseBody = responce;
                input.close();
                rawResponce = rawHeader + responce;
                Thread.sleep(10L);
                if (Objects.nonNull(httpDelete)) httpDelete.releaseConnection();
                if (Objects.nonNull(httpPost)) httpPost.releaseConnection();
                if (Objects.nonNull(httpPut)) httpPut.releaseConnection();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return new ResBody(rawHeader, responce, rawResponce, responseHeader, responseBody, code);
    }

    public static class ResBody {
        private String rawHeader;
        private String responce;
        private String rawResponce;
        private String responseHeader;
        private String responseBody;
        private Integer code;

        public ResBody(String rawHeader, String responce, String rawResponce, String responseHeader, String responseBody, Integer code) {
            this.rawHeader = rawHeader;
            this.responce = responce;
            this.rawResponce = rawResponce;
            this.responseHeader = responseHeader;
            this.responseBody = responseBody;
            this.code = code;
        }

        public Integer getCode() {
            return code;
        }

        public void setCode(Integer code) {
            this.code = code;
        }

        public String getRawHeader() {
            return rawHeader;
        }

        public String getRawResponce() {
            return rawResponce;
        }

        public String getResponce() {
            return responce;
        }

        public String getResponseBody() {
            return responseBody;
        }

        public String getResponseHeader() {
            return responseHeader;
        }

        public void setRawHeader(String rawHeader) {
            this.rawHeader = rawHeader;
        }

        public void setRawResponce(String rawResponce) {
            this.rawResponce = rawResponce;
        }

        public void setResponce(String responce) {
            this.responce = responce;
        }

        public void setResponseBody(String responseBody) {
            this.responseBody = responseBody;
        }

        public void setResponseHeader(String responseHeader) {
            this.responseHeader = responseHeader;
        }

    }
}

among

import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;

public class EasyX509TrustManager implements X509TrustManager {
    private X509TrustManager standardTrustManager = null;

    public EasyX509TrustManager(KeyStore keystore) throws NoSuchAlgorithmException, KeyStoreException {
        TrustManagerFactory factory = TrustManagerFactory.getInstance("SunX509");
        factory.init(keystore);
        TrustManager[] trustmanagers = factory.getTrustManagers();
        if (trustmanagers.length == 0) {
            throw new NoSuchAlgorithmException("SunX509 trust manager not supported");
        } else {
            this.standardTrustManager = (X509TrustManager) trustmanagers[0];
        }
    }

    public X509Certificate[] getAcceptedIssuers() {
        return null;
    }

    public void checkClientTrusted(X509Certificate[] certs, String authType) {
    }

    public void checkServerTrusted(X509Certificate[] certs, String authType) {
    }

    public boolean isServerTrusted(X509Certificate[] certs) {
        return true;
    }

    public boolean isClientTrusted(X509Certificate[] certs) {
        return true;
    }
}

Establish https transmission channel

import org.apache.commons.httpclient.ConnectTimeoutException;
import org.apache.commons.httpclient.HttpClientError;
import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;

import javax.net.SocketFactory;
import javax.net.ssl.KeyManager;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import java.io.IOException;
import java.net.*;
import java.security.KeyStore;
import java.security.SecureRandom;

public class EasySSLProtocolSocketFactory implements SecureProtocolSocketFactory {
    private SSLContext sslcontext = null;

    private static SSLContext createEasySSLContext() {
        try {
            SSLContext context = SSLContext.getInstance("SSL");
            context.init((KeyManager[]) null, new TrustManager[]{new EasyX509TrustManager((KeyStore) null)}, (SecureRandom) null);
            return context;
        } catch (Exception var1) {
            throw new HttpClientError(var1.toString());
        }
    }

    private SSLContext getSSLContext() {
        if (this.sslcontext == null) {
            this.sslcontext = createEasySSLContext();
        }

        return this.sslcontext;
    }

    public Socket createSocket(String host, int port, InetAddress clientHost, int clientPort) throws IOException, UnknownHostException {
        return this.getSSLContext().getSocketFactory().createSocket(host, port, clientHost, clientPort);
    }

    public Socket createSocket(String host, int port, InetAddress localAddress, int localPort, HttpConnectionParams params) throws IOException, UnknownHostException, ConnectTimeoutException {
        if (params == null) {
            throw new IllegalArgumentException("Parameters may not be null");
        } else {
            int timeout = params.getConnectionTimeout();
            SocketFactory socketfactory = this.getSSLContext().getSocketFactory();
            if (timeout == 0) {
                return socketfactory.createSocket(host, port, localAddress, localPort);
            } else {
                Socket socket = socketfactory.createSocket();
                SocketAddress localaddr = new InetSocketAddress(localAddress, localPort);
                SocketAddress remoteaddr = new InetSocketAddress(host, port);
                socket.bind(localaddr);
                socket.connect(remoteaddr, timeout);
                return socket;
            }
        }
    }

    public Socket createSocket(String host, int port) throws IOException, UnknownHostException {
        return this.getSSLContext().getSocketFactory().createSocket(host, port);
    }

    public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException, UnknownHostException {
        return this.getSSLContext().getSocketFactory().createSocket(socket, host, port, autoClose);
    }

    public boolean equals(Object obj) {
        return obj != null && obj.getClass().equals(EasySSLProtocolSocketFactory.class);
    }

    public int hashCode() {
        return EasySSLProtocolSocketFactory.class.hashCode();
    }
}

playwrite - automated test or web2.0 crawler

@Service
public class PlayWriteMachineService implements InitializingBean {


    public List<String> getUrlAndParseHtml(String url, String selector, String functionStr) {
        Playwright playwright = Playwright.create();
        Browser browser = createBrowser("chromium", playwright);
        BrowserContext context = createContext(browser);
        Page page = context.newPage();
        System.out.println(page.title());
        try {

            List<String> values = getOnePage(url, page, selector, functionStr);
            return values;
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            page.close();
            context.close();
            browser.close();
            playwright.close();
        }
        return new ArrayList<>();
    }

    @Override
    public void afterPropertiesSet() throws Exception {
    }

    private BrowserContext createContext(Browser browser) {
        BrowserContext context = browser.newContext(new Browser.NewContextOptions()
                .setIgnoreHTTPSErrors(true)
                .setJavaScriptEnabled(true)
                .setViewportSize(1920, 1080)
                .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.130 Safari/537.36"));
        return context;
    }

    private Browser createBrowser(String name, Playwright playwright) {
        try {
            switch (name) {
                case "firefox":
                    return playwright.firefox().launch();
                case "chromium":
                    return playwright.chromium().launch(new BrowserType.LaunchOptions().setHeadless(true).setTimeout(120 * 1000));
                case "webkit":
                    return playwright.webkit().launch();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

    public List<String> getOnePage(String nodeUrl, Page page, String selector, String functionStr) {
        // Todo: create a page on April 19, 2021, and then go to the link
        // TODO: 2021/4/19 inject something into the page to achieve a certain goal

        Set<String> pageAllUrls = new ConcurrentSkipListSet<>();
        Map<String, String> postDatas = new ConcurrentHashMap<>();
        page.setDefaultNavigationTimeout(110 * 1000);
        page.onRequest(request -> {
            if (Objects.nonNull(request)) {
                String method = request.method();
                String url = request.url();
                if (method.toLowerCase().equals("post") && Objects.nonNull(request.postData())) {
                    postDatas.put(url, request.postData());
                }
                pageAllUrls.add(request.url());
                if (request.isNavigationRequest()) {
                    try {
                        if (!url.equals(nodeUrl)) {
                            request.response();
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }
        });
        page.onResponse(response -> {
            if (response.status() >= 300 && response.status() < 400) {
                pageAllUrls.add(response.headers().getOrDefault("location", ""));
            }
        });

        page.onFrameNavigated(frame -> {
            pageAllUrls.add(frame.url());
        });

        page.onDialog(Dialog::dismiss);
        page.navigate(nodeUrl, new Page.NavigateOptions());
        page.waitForTimeout(5000);
        // TODO: 2021/4/19 eval
        List<String> values = evalAndGetValue(page, selector, functionStr);
        return values;
    }

    private static List<String> evalAndGetValue(Page page, String selector, String functionStr) {
        System.out.println(selector);
        System.out.println(functionStr);
        List<String> ans = (List<String>) page.evalOnSelectorAll(selector, functionStr);
        List<String> res = new ArrayList<>();
        for (String x : ans) {
            if (StringUtils.hasText(x)) {
                res.add(x);
            }
        }
        return res;
    }
}

This example is just the code of my own application scenario. This is a simple example. It works well, but there are pits.
Official website address: https://github.com/microsoft/playwright-java
API address: https://playwright.dev/docs/api/class-playwright
maven

<dependency>
  <groupId>com.microsoft.playwright</groupId>
  <artifactId>playwright</artifactId>
  <version>1.10.0</version>
</dependency>

Extracting key information using regular expressions

for (Pattern pattern : patterns) {
    Matcher matcher = pattern.matcher(html);
    while (matcher.find()) {
        tmp.add(matcher.group());
        System.out.println("begin = " + matcher.start() + ", end = " + matcher.end());
    }
}
// Among them, Pattern can be
Pattern.compile("<!--[\\s\\S]*?-->");
obtain

find one, and then call group to get one. Of course, there is a better way. You can put parentheses in the regular expression and enter the variable name. When the regular expression is found, it will be implemented through matcher.group("variable name").

cache

When you can't use redis, you need cache to speed up the scene

@Service
public class LruCacheWithTTL {

    /**
     * Cache key rule: query criteria are keys
     * Unique cache flag (used to distinguish different business scenarios) + condition (condition 1: value (if any) + condition 2: value (if any) + condition 3: value (if any))
     * Cache value rule: the Json of the returned value is the value
     */
    private LRUMap<String, String> cache = new LRUMap<>(20, 10000);
    private ConcurrentHashMap<String, Long> ttls = new ConcurrentHashMap<>();

    public String getCacheJson(Integer bizCode, Map<String, Object> condition) {
        String key = bizCode + "+" + getSortedConditionStr(condition);
        if (ttls.containsKey(key)) {
            if (DatetimeUtil.getDateTimeOfTimestamp(ttls.get(key)).plusSeconds(5).isBefore(LocalDateTime.now())) {
                // Todo: on September 29, 2021, if the current time does not timeout, the result will be returned directly
                ttls.remove(key);
            }else {
                // Todo: delete cache on September 29, 2021
                return cache.get(key);
            }
        }
        return null;
    }

    private String getSortedConditionStr(Map<String, Object> condition) {
        Set<String> keys = condition.keySet();
        List list = new ArrayList(keys);
        Object[] ary = list.toArray();
        Arrays.sort(ary);
        list = Arrays.asList(ary);
        String str = "";
        for (int i = 0; i < list.size(); i++) {
            str += list.get(i) + "=" + condition.get(list.get(i) + "") + "&";
        }
        return str;
    }

    public Boolean putCacheJson(Integer bizCode, Map<String, Object> condition, String value) {
        try {
            String key = bizCode + "+" + getSortedConditionStr(condition);
            ttls.put(key, DatetimeUtil.getTimestampOfDatetime(LocalDateTime.now()));
            cache.put(key, value);
            return true;
        }catch (Exception e){
            e.printStackTrace();
        }
        return false;
    }
}

Call command line

Process process = Runtime.getRuntime().exec("ping www.qq.com");

transfer files

import lombok.Getter;
import lombok.Setter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile;

import java.io.*;
import java.util.Map;
import java.util.Objects;

@Service
public class UploadFileService {

    private static Log logger = LogFactory.getLog(UploadFileService.class);

    @Autowired
    private RestTemplate restTemplate;

    @Autowired
    private TConfigRepository tConfigRepository;

    public void demo(MultipartFile file) {
        if (tConfigRepository.findByConfigNameAndDeleteMarkFalse("oos_url").isPresent()) {
            String oosUrl = tConfigRepository.findByConfigNameAndDeleteMarkFalse("oos_url").get().getConfigValue();
            if (StringUtils.hasText(oosUrl)) {
                MultiValueMap<String, Object> param = new LinkedMultiValueMap<>();
                param.add("file", new FileSystemResource(convert(file)));
                HttpHeaders headers = new HttpHeaders();
                headers.setContentType(MediaType.MULTIPART_FORM_DATA);
                HttpEntity<MultiValueMap<String, Object>> httpEntity = new HttpEntity<MultiValueMap<String, Object>>(param, headers);
                ResponseEntity<FileRes> resResponseEntity = restTemplate.postForEntity(oosUrl + "/v1/upload", httpEntity, FileRes.class);
                if (resResponseEntity.getStatusCode().is2xxSuccessful()) {
                    FileRes res = resResponseEntity.getBody();
                    if (Objects.nonNull(res)) {
                        logger.info(res.getMessage());
                        logger.info(res.getCode());
                        if (res.getCode().equals("00000")) {
                            logger.info(String.format("%s %s", res.getInfo().get("fileName"), res.getInfo().get("uuid")));
                        }
                    }
                    logger.info("null");
                }

            }
        }
    }

    public File convert(MultipartFile file) {
        File convFile = new File("temp_file", file.getOriginalFilename());
        if (!convFile.getParentFile().exists()) {
            logger.info("mkdir:" + convFile.getParentFile().mkdirs());
        }
        try {
            convFile.createNewFile();
            FileOutputStream fos = new FileOutputStream(convFile);
            fos.write(file.getBytes());
            fos.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return convFile;
    }
}

Using multivariate groups in java

<dependency>
    <groupId>org.javatuples</groupId>
    <artifactId>javatuples</artifactId>
    <version>1.2</version>
</dependency>

Unit (1 element)
Pair<A,B> (2 elements)
Triplet<A,B,C> (3 elements)
Quartet<A,B,C,D> (4 elements)
Quintet<A,B,C,D,E> (5 elements)
Sextet<A,B,C,D,E,F> (6 elements)
Septet<A,B,C,D,E,F,G> (7 elements)
Octet<A,B,C,D,E,F,G,H> (8 elements)
Ennead<A,B,C,D,E,F,G,H,I> (9 elements)
Decade<A,B,C,D,E,F,G,H,I,J> (10 elements)

springboot bootless boot mode

spring.main.web-application-type=none
// Then start the system task, that is, a thread does not die. However, if you want to stop stably, you must listen to the port

cron

Inspection Time
https://www.bejson.com/othertools/cronvalidate/
Generate cron
https://www.bejson.com/othertools/cron/

Tags: Java Back-end

Posted on Fri, 03 Dec 2021 06:54:22 -0500 by jbdphp