rust爬虫相关|日常技术

1.发送GET请求，设置请求头，用scraper解析html。

use reqwest::header::{HeaderMap};
use reqwest::blocking::Client;
use scraper::{Html, Selector};

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let mut headers = HeaderMap::new();
    headers.insert("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.50".parse().unwrap());
    let resp = client.get("https://www.douban.com/").headers(headers).send()?.text();
    match resp {
        Ok(val) => {
            let document = Html::parse_document(&val);
            let selector = Selector::parse("span#icp>a").unwrap();
            let element = document.select(&selector).last().unwrap();
            println!("{:?}",element.inner_html());
        },
        Err(err) => println!("{}",err)
    }
    Ok(())
}

2.POST请求，提交url参数和body，并解析json。

use std::collections::HashMap;

use reqwest::header::{HeaderMap, HeaderValue};
use reqwest::blocking::Client;
use serde_json::Value;


fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let mut headers = HeaderMap::new();
    headers.insert("Content-Type",HeaderValue::from_static("application/x-www-form-urlencoded"));
    headers.insert("Referer",HeaderValue::from_static("https://accounts.douban.com/passport/login_popup?login_source=anony"));
    headers.insert("User-Agent", HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.50"));
    let mut params = HashMap::new();
    params.insert("remember", "true");
    params.insert("name", "17872855521");
    params.insert("password", "123456789");
    params.insert("ticket", "");
    params.insert("randstr", "");
    params.insert("tc_app_id","2044348370");
    // let resp = client.post("https://accounts.douban.com/j/mobile/login/basic").body("{}").query(&params).headers(headers).send()?.json::<Value>()?;
    // let parsed_json:Value = serde_json::from_value(&resp)?;
    // let parsed_json:Value = serde_json::from_value(&resp).unwrap();
    let resp = client.post("https://accounts.douban.com/j/mobile/login/basic").body("{}").query(&params).headers(headers).send()?.text()?;
    let parsed_json:Value = serde_json::from_str(&resp)?;
    println!("{:#?}",parsed_json);
    Ok(())
}

？和unwrap()处理result类型或者Option类型时最主要的区别是unwarp会出现panic，而?只会抛出Err，panic比普通Err多了调用栈，会出现报错行数更方便分析错误。

3.rust和JavaScript进行交互

use js_sandbox::Script;

fn main() -> Result<(), Box<dyn std::error::Error>> {
	//let mut script = Script::from_file("temp.js").unwrap(); //读取文件方式
    let mut script = Script::from_string("function toMd5Hex(value){return '202cb962ac59075b964b07152d234b70';}").unwrap(); //读取字符串形式
	let result:String= script.call("toMd5Hex", &"123").unwrap();

	println!("{}",result);
	Ok(())
}

4.rust调用opencv识别滑块验证码缺口位置的两种方式：

1.缺口图片和带缺口的背景图片，下面一个是slider.rs文件，一个是main.rs程序入口文件：

use opencv::prelude::*;
use std::collections::HashSet;
use opencv::core::{Mat,Vec3b, Rect, min_max_loc, Point, Scalar, Vector};
use opencv::imgproc::{canny, cvt_color, COLOR_RGB2GRAY, COLOR_GRAY2RGB, match_template, rectangle, TM_CCORR_NORMED};
use opencv::imgcodecs::{imwrite, imread, IMREAD_COLOR};

pub struct SlideCrack{
    gap:String,
    bg:String,
    out:String
}

impl SlideCrack {
    pub fn new(gap:String, bg:String,  out:String) ->SlideCrack{
        SlideCrack{
            gap,
            bg,
            out
        }
    }

    fn clear_white(&self, img: &String) -> Mat{
        let img = imread(img,IMREAD_COLOR).unwrap();
        let rows = img.rows();
        let cols = img.cols();
        let mut min_x = 255;
        let mut min_y = 255;
        let mut max_x = 0;
        let mut max_y = 0;
        for x in 1..rows{
            for y in 1..cols{
                let mut t = HashSet::new();
                let img_slice = img.at_nd::<Vec3b>(&[x,y]).unwrap().into_iter();
                for val in img_slice {
                   t.insert(val);
                }
                if t.len() >= 2{
                    if x <= min_x{
                        min_x = x;
                    }else if x >= max_x {
                        max_x = x;
                    };
                    if y <= min_y{
                        min_y = y;
                    }else if y >= max_y {
                        max_y = y;
                    }
                }
            }
        }
        let img_ = Mat::roi(&img, Rect::new(min_y, min_x, max_y-min_y, max_x-min_x)).unwrap();
        return img_;
    }

    fn template_match(&self, tpl:Mat, mut target:Mat) -> i32{
        let th = tpl.cols();
        let tw = tpl.rows();
        let mask = Mat::default();
        let mut result = Mat::default();
        let _ = match_template(&target, &tpl, &mut result, TM_CCORR_NORMED, &mask);
        let mut min_val: f64 = 0.0;
        let mut max_val: f64 = 0.0;
        let mut min_loc: Point = Point::new(0,0);
        let mut max_loc: Point = Point::new(0,0);
        let mask = Mat::default();
        let _ = min_max_loc(&result, Some(&mut min_val), Some(&mut max_val), Some(&mut min_loc), Some(&mut max_loc), &mask);
        let face_rec = Rect {
            x: max_loc.to_vec2()[0] as i32,
            y: max_loc.to_vec2()[1] as i32,
            width: tw as i32,
            height: th as i32,
        };
        let _ = rectangle(&mut target, face_rec, Scalar::new(0.0, 0.0, 255.0, 0.0), 2, 8, 0,);
        let _ = imwrite(&self.out, &target, &Vector::<i32>::default()).unwrap();
        return max_loc.to_vec2()[0];
    }

    fn image_edge_detection(&self,img:Mat) -> Mat{
        let mut edges =  Mat::default();
        let _ = canny(&img, &mut edges, 100.,200.,3,false);
        return edges;
    }

    pub fn discern(&self) -> i32{
        let  img1 = self.clear_white(&self.gap);
        let mut gray =  Mat::default();
        let _ = cvt_color(&img1, &mut gray, COLOR_RGB2GRAY, 0).unwrap();
        let slide = self.image_edge_detection(gray);
        let mut back = imread(&self.bg,0).unwrap();
        back = self.image_edge_detection(back);
        let mut slide_pic =  Mat::default();
        let _ = cvt_color(&slide, &mut slide_pic, COLOR_GRAY2RGB, 0).unwrap();
        let mut back_pic =  Mat::default();
        let _ = cvt_color(&back, &mut back_pic, COLOR_GRAY2RGB, 0).unwrap();
        let distance = self.template_match(slide_pic, back_pic);
        return distance;
    }
}

mod slide;
use slide::SlideCrack;

fn main() {
    let img1 = r"C:\Users\yuese\Desktop\rustTest\study\src\1_1.png";
    let img2 = r"C:\Users\yuese\Desktop\rustTest\study\src\1_2.png";
    let img3 = r"C:\Users\yuese\Desktop\rustTest\study\src\1_3.png";
    let sc:SlideCrack = SlideCrack::new(img1.to_string(), img2.to_string(), img3.to_string());
    let val = sc.discern();
    println!("{:?}",val);
}

2.完整的背景图片和带缺口的背景图片，下面一个是slider.rs文件，一个是main.rs程序入口文件：

use image::{DynamicImage, GenericImageView};

pub struct SlideCrack {
    gap: String,
    bg: String,
}

impl SlideCrack {
    pub fn new(gap: String, bg: String) -> SlideCrack {
        SlideCrack {
            gap,
            bg,
        }
    }

    fn pixel_is_equal(&self, image1: image::DynamicImage, image2:DynamicImage, x: u32, y: u32) -> bool {
        let pixel1 = image1.get_pixel(x, y);
        let pixel2 = image2.get_pixel(x, y);
        let threshold = 60;
        let dif1 = pixel1[0] as i32 - pixel2[0] as i32;
        let dif2 = pixel1[1] as i32 - pixel2[1] as i32;
        let dif3 = pixel1[2] as i32 - pixel2[2] as i32;
        return if (dif1.abs() < threshold) && (dif2.abs() < threshold) && (dif3.abs() < threshold) {
            true
        } else {
            false
        }
    }

    fn get_gap(&self, image1: image::DynamicImage, image2: image::DynamicImage) -> i32 {
        let mut left = 60;
        let width = image1.dimensions().0 as i32;
        let height = image1.dimensions().1 as i32;
        for i in left..width {
            for j in 0..height {
                if !self.pixel_is_equal(image1.clone(), image2.clone(), i.try_into().unwrap(), j.try_into().unwrap()){
                    left = i;
                    return left;
                }
            }
        }
        return left;
    }

    pub fn run(&self) -> i32{
        let image1 = image::open(self.bg.clone()).unwrap();
        let image2 = image::open(self.gap.clone()).unwrap();
        let gap = self.get_gap(image1, image2);
        return gap;
    }
}

mod slide;
use slide::SlideCrack;

fn main() {
    let img1 = r"C:\Users\yuese\Desktop\rustTest\study\src\1_1.png";
    let img2 = r"C:\Users\yuese\Desktop\rustTest\study\src\1_2.png";
    let sc = SlideCrack::new(img1.to_string(), img2.to_string());
    let val = sc.run();
    println!("{:?}",val);
}

5.rust常用类型转换

from	to	函数
&str	String	String::from(s) 或 s.to_string() 或 s.to_owned()
&str	&[u8]	s.as_bytes()
&str	Vec<u8>	s.as_bytes().to_vec()
String	&[u8]	s.as_bytes()
String	&str	s.as_str() 或 &s
String	Vec<u8>	s.into_bytes()
&[u8]	&str	std::str::from_utf8(s).unwrap()
&[u8]	String	String::from_utf8(s).unwrap()
&[u8]	Vec<u8>	s.to_vec()
Vec<u8>	&str	std::str::from_utf8(&s).unwrap()
Vec<u8>	String	String::from_utf8(s).unwrap()
Vec<u8>	&[u8]	&s 或 s.as_slice()

6.常用编码解码和摘要以及加解密

1.base64编码解码

[dependencies]
base64 = "0.21.2"

use base64::{Engine as _, engine::general_purpose};


fn main() {
    let orig = "data";
    let encoded: String = general_purpose::STANDARD.encode(orig.as_bytes());
    println!("{:?}", encoded);
    let decoded = general_purp
ose::STANDARD.decode(encoded).unwrap();
    println!("{:?}",String::from_utf8(decoded).unwrap())
}

2.md5摘要

[dependencies]
md-5 = "0.10.5"
hex = "0.4.3"

use md5::{Md5, Digest};

fn main() {
    let mut hasher = Md5::new();
    hasher.update("hello world".as_bytes());
    let result = hasher.finalize();
    let hex_string = hex::encode(result);
    println!("{}", hex_string); 
}