package dao import ( "fmt" "git.aionnect.com/aionnect/go-common/utils" "git.aionnect.com/aionnect/go-common/utils/date" "git.aionnect.com/hello-go/spider/common" "math/rand" "strings" "sync" "time" "xorm.io/xorm" ) var proxyOnce sync.Once var proxyDao *ProxyDao // 代理信息数据访问对象 type ProxyDao struct { db *xorm.Engine // 数据库访问对象 cache *common.ConcurrentMap // 本地缓存 } // 返回代理信息数据库访问对象 func NewProxyDao() *ProxyDao { proxyOnce.Do(func() { proxyDao = &ProxyDao{ db: DB("spider"), cache: common.NewConcurrentMap(), } }) return proxyDao } // 保存代理信息到数据库 func (d *ProxyDao) Save(proxies []*common.ProxyInfo) { if nil == proxies || len(proxies) == 0 { return } for i := 0; i < len(proxies); i++ { if proxies[i].ID == 0 { proxies[i].ID = utils.NextId() } proxies[i].CreatedAt = date.Now() } err := utils.Insert(d.db, &proxies) if nil != err { fmt.Printf("save proxies to db failed %s", err.Error()) } } const ReadClause = `select distinct addr from ( select concat(ip, ':', port) as 'addr' from proxy_info where (type = 'HTTP' or type = 'HTTPS') and anonymity = '高匿名' order by update_time desc, speed limit 100) T` // 查询代理信息 func (d *ProxyDao) Get() string { // 当缓存中数据量过少时,从数据库中读取最新的100条 if d.cache.Len() < 10 { var addrs []string err := d.db.SQL(ReadClause).Find(&addrs) if nil != err { fmt.Printf("read proxies to db failed %s", err.Error()) } else if nil != addrs && len(addrs) > 0 { m := make(map[string]interface{}) for i := 0; i < len(addrs); i++ { m[addrs[i]] = true } d.cache.Append(m) } } // 从缓存中返回随机一个 keys := d.cache.Keys() rand.Seed(time.Now().UnixNano()) idx := rand.Intn(len(keys)) return keys[idx] } // 删除代理信息 func (d *ProxyDao) Remove(key string) { key = strings.TrimSpace(key) if key == "" { return } // 缓存中移除 d.cache.Remove(key) // 数据库中移除 idx := strings.LastIndex(key, ":") ip := key[:idx] port := strings.TrimLeft(key[idx:], ":") _, err := d.db.Where("ip=? and port=?", ip, port).Delete(&common.ProxyInfo{}) if nil != err { fmt.Printf("remove proxy to db failed %s", err.Error()) } }