hello-go/spider/common/proxy_info.go

package common

import (
	"encoding/xml"
	"git.aionnect.com/aionnect/go-common/utils"
	"git.aionnect.com/aionnect/go-common/utils/date"
	"git.aionnect.com/aionnect/go-common/utils/jsonutil"
	"net"
	"strconv"
	"strings"
	"time"
)

// 类型
const (
	HTTP  = "HTTP"
	HTTPS = "HTTPS"
	SOCKS = "SOCKS"
)

// 匿名度
const (
	High   string = "高匿名"
	Normal string = "透明"
)

type ProxyInfo struct {
	IP         string        `json:"ip" xorm:"ip"`
	Port       int           `json:"port" xorm:"port"`
	Anonymity  string        `json:"anonymity" xorm:"anonymity"`
	Type       string        `json:"type" xorm:"type"`
	Location   string        `json:"location" xorm:"location"`
	Speed      time.Duration `json:"speed" xorm:"speed"`
	UpdateTime date.Datetime `json:"updateTime" xorm:"update_time"`

	ID        utils.Long    `xorm:"'id' not null pk BIGINT(20)" json:"id"`
	UpdatedAt date.Datetime `xorm:"'update_at' updated not null TIMESTAMP" json:"updatedAt,omitempty"`
	CreatedAt date.Datetime `xorm:"'create_at' not null DATETIME" json:"createdAt,omitempty"`
}

func ConvertToIP(text string) string {
	ip := net.ParseIP(strings.TrimSpace(text))
	if ip.IsLoopback() || ip.IsUnspecified() {
		return ""
	}
	return ip.String()
}

func ConvertToSpeed(text string) time.Duration {
	text = strings.TrimSpace(text)
	if text == "" {
		return 0
	}

	text = strings.ReplaceAll(text, "秒", "")
	n, err := strconv.ParseFloat(text, 10)
	if nil != err {
		return 0
	}

	return time.Duration(n * float64(time.Second))
}

func (ProxyInfo) TableName() string {
	return "proxy_info"
}

func (that *ProxyInfo) UnmarshalJSON(value []byte) error { // 反序列化Json的时候，如果没有ID，自动给一个
	type Alias ProxyInfo
	alias := &struct {
		*Alias
	}{Alias: (*Alias)(that)}
	if err := jsonutil.Unmarshal(value, &alias); err != nil {
		return err
	}
	if nil != alias && alias.ID == 0 {
		alias.ID = utils.NextId()
	}
	return nil
}

func (that *ProxyInfo) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { // 反序列化XML的时候，如果没有ID，自动给一个
	type Alias ProxyInfo
	alias := &struct {
		*Alias
	}{Alias: (*Alias)(that)}
	if err := d.DecodeElement(&alias, &start); err != nil {
		return err
	}
	if nil != alias && alias.ID == 0 {
		alias.ID = utils.NextId()
	}
	return nil
}

// 分页参数
type ProxyPagingParams struct {
	PagingNo int // 页码
	Limit    int // 一次最多爬多少条（因为按分页爬，可能会稍微超出一点点）
	Count    int // 本次已爬有效数据条数计数
}