hello-go/spider/spiders/qianlima/visitor_test.go

package qianlima

import (
	"fmt"
	"git.aionnect.com/hello-go/spider/common"
	"git.aionnect.com/hello-go/spider/spiders"
	"git.aionnect.com/hello-go/spider/spiders/qianlima/items"
	"net/http"
	"testing"
)

func TestVisitorSpider(t *testing.T) {
	// 从起始页开始执行爬虫
	params := &items.PagingParams{PagingNo: 1, DaysLimit: 7}
	startingUrl := fmt.Sprintf(TargetUrl, params.PagingNo)
	target := &common.Target{
		Key:     VisitorPaging,
		Method:  http.MethodPost,
		URL:     startingUrl,
		Referer: RefererUrl,
		Item:    params,
	}
	spiders.Run(target,
		NewVisitorPagingSpider(),
		NewVisitorDetailSpider(),
	)
	fmt.Println("Done!")
}

func TestVisitorDetailSpider(t *testing.T) {
	// 单独测试详情页爬虫，主要为了测试不同网页编码是否解析正常
	// 未传Item参数，不会写数据库
	targets := []*common.Target{
		{Key: VisitorDetail, Method: http.MethodGet, URL: "http://www.qianlima.com/zb/detail/20200710_186393093.html"}, // UTF8
		{Key: VisitorDetail, Method: http.MethodGet, URL: "http://www.qianlima.com/zb/detail/20200705_185702517.html"}, // GBK
		{Key: VisitorDetail, Method: http.MethodGet, URL: "http://www.qianlima.com/zb/detail/20200703_185597771.html"}, // GBK
	}
	for _, target := range targets {
		spiders.Run(target,
			NewVisitorDetailSpider(),
		)
	}
}