hello-go/spider/dao/proxy_dao.go

package dao

import (
	"fmt"
	"git.aionnect.com/aionnect/go-common/utils"
	"git.aionnect.com/aionnect/go-common/utils/date"
	"git.aionnect.com/hello-go/spider/common"
	"math/rand"
	"strings"
	"sync"
	"time"
	"xorm.io/xorm"
)

var proxyOnce sync.Once
var proxyDao *ProxyDao

// 代理信息数据访问对象
type ProxyDao struct {
	db    *xorm.Engine          // 数据库访问对象
	cache *common.ConcurrentMap // 本地缓存
}

// 返回代理信息数据库访问对象
func NewProxyDao() *ProxyDao {
	proxyOnce.Do(func() {
		proxyDao = &ProxyDao{
			db:    DB("spider"),
			cache: common.NewConcurrentMap(),
		}
	})
	return proxyDao
}

// 保存代理信息到数据库
func (d *ProxyDao) Save(proxies []*common.ProxyInfo) {
	if nil == proxies || len(proxies) == 0 {
		return
	}

	for i := 0; i < len(proxies); i++ {
		if proxies[i].ID == 0 {
			proxies[i].ID = utils.NextId()
		}
		proxies[i].CreatedAt = date.Now()
	}

	err := utils.Insert(d.db, &proxies)
	if nil != err {
		fmt.Printf("save proxies to db failed %s", err.Error())
	}
}

const ReadClause = `select distinct addr from (
  select concat(ip, ':', port) as 'addr'
  from proxy_info
  where (type = 'HTTP' or type = 'HTTPS')
    and anonymity = '高匿名'
  order by update_time desc, speed
  limit 100) T`

// 查询代理信息
func (d *ProxyDao) Get() string {
	// 当缓存中数据量过少时，从数据库中读取最新的100条
	if d.cache.Len() < 10 {
		var addrs []string
		err := d.db.SQL(ReadClause).Find(&addrs)
		if nil != err {
			fmt.Printf("read proxies to db failed %s", err.Error())
		} else if nil != addrs && len(addrs) > 0 {
			m := make(map[string]interface{})
			for i := 0; i < len(addrs); i++ {
				m[addrs[i]] = true
			}
			d.cache.Append(m)
		}
	}
	// 从缓存中返回随机一个
	keys := d.cache.Keys()
	rand.Seed(time.Now().UnixNano())
	idx := rand.Intn(len(keys))
	return keys[idx]
}

// 删除代理信息
func (d *ProxyDao) Remove(key string) {
	key = strings.TrimSpace(key)
	if key == "" {
		return
	}
	// 缓存中移除
	d.cache.Remove(key)
	// 数据库中移除
	idx := strings.LastIndex(key, ":")
	ip := key[:idx]
	port := strings.TrimLeft(key[idx:], ":")
	_, err := d.db.Where("ip=? and port=?", ip, port).Delete(&common.ProxyInfo{})
	if nil != err {
		fmt.Printf("remove proxy to db failed %s", err.Error())
	}
}