init

.gitignore

@@ -0,0 +1,126 @@
+# ---> Go
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+
+# ---> Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# ---> Node
+# Logs
+logs
+*.log
+npm-debug.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git
+node_modules
+
+bin
+.idea
+.DS_Store
+*.mp3
+*.mp4
+output
+vendor
+go.sum
+
+deploy/bin

ABSubStr.go

@@ -0,0 +1,40 @@
+package main
+
+func main() {
+	println(getAns("ABAAABBBA"))
+	println(getAns("ABAAAABBBA"))
+	println(getAns("ABAAAAABBBA"))
+	println(getAns("AAAAAA"))
+	println(getAns("BABAABBABABABAAABAABABABBBAAABBAAABAABBBABBAAABABBBBBBBBBBBBAABBBBAAABAAAAABBBBBABBABBAABABAAABBABAABBAAAABBAABBBABABABABABBBAAABBBABAABAAAABBBBBAABBBABBBBAAABBABBAAABBABBABBBBBBBBBBBBBAABBAAAABAABABAABBBABBAAAAAABAABAABABBABAABBAAABBBBAABABBBABABAABAAABABBBBAABBAAABBBBBBABBAABAABABBBBBBBABBAAAABBABABBBBBBABABAABBABAABABBBBABBABBAABBAABBABABABBBAABBAABAABAABBBBBBBBBABAAABBAAAABBBAABABABBABAAAABABBABBAABAAABBBBBBAABABAAABAAABBABAABAABAAAABABAABABABAABABAAABBBBBAAAAAABAABBBABBBABBBBBBAAABABAAAABABAAAAAABABAABABBBABAAAAABAABBBAABBBAAAABAAAAABBBABABABBBAABAAAAAABABBBABABBBABBABABBBBABABBABABBBABABBAAAABAABBBAABABABABABAAABAABBAAAABAAAAABAABBBBABABBBBAABAABBBBABBAAAAAAAAAABBBAAABAAABBAAAAAAABBAAAAAABAAABBAABABABABBAAAAAABABAAABABABBBABAABBBABAABBAAAABBBBAAAABBAAAAABABBBABBAAABABABBAABABAABBBBAABAAABAABBAAAAAABAAABBBAAAABBABABAABABAAABBAABBABABBBAABBABBAABABABAAABABBAABBABABABBABBBABAAAAAABBAABBBAAABAAABAAABBBBBABBABBBABBBAABBABAAAAABBABABBABAAAAAABBBABABBABAABABAABBAAAAABABBAAABABABABBBAABB"))
+}
+
+func getAns(S string) int {
+	length := len(S)
+	sum := 0
+	sumArr := make([]int, length)
+	posMap := make(map[int]int)
+	for i := 0; i < length; i++ {
+		if S[i] == 'A' {
+			sum++
+		} else {
+			sum--
+		}
+		sumArr[i] = sum
+		posMap[sum] = i
+	}
+	if sum == 0 {
+		return length
+	} else if sum == length || sum == -length {
+		return 0
+	}
+	maxLen := 0
+	for i := 0; i < length; i++ {
+		if sumArr[i] != 0 && maxLen < posMap[sumArr[i]]-i {
+			maxLen = posMap[sumArr[i]] - i
+		}
+	}
+	if posMap[0] != posMap[len(posMap)-1] && posMap[0]+1 > maxLen {
+		maxLen = posMap[0] + 1
+	}
+	return maxLen
+}

Order.go

@@ -0,0 +1,72 @@
+package main
+
+func main() {
+	//order := []int {2,3,1}
+	//pattern := [][]int {{2,2,0},{0,1,1},{1,1,0}}
+	order := []int{2, 3, 1}
+	pattern := [][]int{{2, 2, 0}, {0, 1, 2}, {0, 1, 0}, {1, 1, 0}}
+	println(getMinRemaining(order, pattern))
+}
+
+// BFS 广度优先搜索
+type Item struct {
+	sum     int
+	pattern []int
+}
+
+func getMinRemaining(order []int, pattern [][]int) int {
+	total := 0
+	for i := 0; i < len(order); i++ {
+		total += order[i]
+	}
+	sumArr := make([]int, len(pattern))
+
+	res := total
+	var queue []Item
+	for i := 0; i < len(pattern); i++ {
+		matched := true
+		for j := 0; j < len(pattern[i]); j++ {
+			v := pattern[i][j]
+			sumArr[i] += v
+			if v > order[j] {
+				matched = false
+			}
+		}
+		if sumArr[i] != 0 && matched {
+			queue = append(queue, Item{sumArr[i], pattern[i]})
+			if res > total-sumArr[i] {
+				res = total - sumArr[i]
+			}
+		}
+	}
+
+	for len(queue) > 0 {
+		var newQueue []Item
+		for _, item := range queue {
+			for i, p := range pattern {
+				routerSum := sumArr[i] + item.sum
+				diff := total - routerSum
+				if diff >= 0 {
+					var newPattern []int
+					for j := 0; j < len(order); j++ {
+						pNo := p[j] + item.pattern[j]
+						if pNo <= order[j] {
+							newPattern = append(newPattern, pNo)
+						}
+					}
+					if len(newPattern) == len(order) {
+						if diff == 0 {
+							return 0
+						}
+						newQueue = append(newQueue, Item{routerSum, newPattern})
+						if diff < res {
+							res = diff
+						}
+					}
+				}
+			}
+		}
+		queue = newQueue
+	}
+	return res
+}

README.md

@@ -0,0 +1,5 @@
+## 交叉编译
+
+> GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -tags=jsoniter -ldflags "-s" -v -o ./deploy/bin/hello-go ./temp.go
+> docker build --no-cache --disable-content-trust=true -t ccr.ccs.tencentyun.com/yowootech/hello-go ./deploy
+> docker push ccr.ccs.tencentyun.com/yowootech/hello-go

WXNotifyTest.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+)
+
+type TemplateMsg struct {
+	ToUser      string                         `json:"touser"`
+	TemplateId  string                         `json:"template_id"`
+	Url         string                         `json:"url"`
+	MiniProgram TemplateMsgMiniprogramInfo     `json:"miniprogram"`
+	Data        map[string]TemplateMsgDataItem `json:"data"`
+}
+
+type TemplateMsgMiniprogramInfo struct {
+	AppId    string `json:"appid"`
+	PagePath string `json:"pagepath"`
+}
+
+type TemplateMsgDataItem struct {
+	Value string `json:"value"`
+	Color string `json:"color"`
+}
+
+func sendTemplateMsg(bytesData []byte) string {
+	reader := bytes.NewReader(bytesData)
+	url := "https://api.weixin.qq.com/cgi-bin/message/template/send?access_token=13_NI_fTDDbC34LHbs1qtNPxbCdL1dnzzL9vLadYLcc_fPufMWtaNpKadstYk_PvNrD8aeBHrJeeMjvEv_oI29WjTFyPlja2VEpma1paIpAoDm9TTwtNm45zsRdFhVP_QfbJUU0xypR1zM1Dd9XUSHjABAGET"
+	resp, err := http.Post(url, "application/json;charset=utf-8", reader)
+	if err != nil {
+		fmt.Println(err.Error())
+		return "request error"
+	}
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	defer resp.Body.Close()
+	if err != nil {
+		fmt.Println(err.Error())
+		return "response error"
+	}
+	return string(respBytes)
+}
+
+func main() {
+	data := make(map[string]TemplateMsgDataItem)
+	data["first"] = TemplateMsgDataItem{Value: "您发布的房源已经上线7天了，请确认房源状态是否已发生变化。"}
+	data["keyword3"] = TemplateMsgDataItem{Value: "金地梅陇镇F栋3001"}
+	data["remark"] = TemplateMsgDataItem{Value: "如房源仍然有效，请忽略本消息。"}
+	msg := TemplateMsg{ToUser: "oAj8Y07DX2guxp-vmtATurBLt9PQ", TemplateId: "GBuf5jfz6oj1YNUaPBpieQrR8IK6cnxiL0BGNZ-bueA", Data: data} // 有屋叽里咕噜爷房源状态更新提醒
+	bytesData, err := json.Marshal(msg)
+	if err != nil {
+		log.Fatal(err.Error())
+		return
+	}
+
+	now := time.Now()
+
+	n := 1
+	counter := make(chan bool, n)
+	for i := 0; i < n; i++ {
+		go func(bytes []byte) {
+			respStr := sendTemplateMsg(bytes)
+			//fmt.Println(respStr)
+			if !strings.Contains(respStr, `"errcode":0,"errmsg":"ok"`) {
+				fmt.Println(respStr)
+			}
+			counter <- false
+		}(bytesData)
+	}
+
+	for i := 0; i < n; i++ {
+		<-counter
+	}
+	fmt.Println("The end")
+	fmt.Println("cost time:", float64(time.Since(now).Nanoseconds())/1000000.0, "ms")
+}

alicloud/live_sign.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"git.wanpinghui.com/WPH/go_common/wph"
+	"git.wanpinghui.com/WPH/go_common/wph/date"
+	"net/url"
+	"strings"
+)
+
+func GetLiveSingedURL(scheme, host, app, stream, key string, expire date.Datetime, args url.Values) string {
+	scheme = strings.TrimSpace(scheme)
+	host = strings.TrimSpace(host)
+	app = strings.TrimSpace(app)
+	stream = strings.TrimSpace(stream)
+	key = strings.TrimSpace(key)
+	if scheme == "" {
+		scheme = "rtmp"
+	}
+	if host == "" || app == "" || stream == "" || key == "" || expire.IsZero() {
+		return ""
+	}
+	nonce := wph.NextId() // 随机码
+
+	path := fmt.Sprintf("/%s/%s", app, stream)               // /AppName/StreamName及后缀
+	paramStr := fmt.Sprintf("%d-%d-0", expire.Unix(), nonce) // 地址过期时间UNIX时间戳-随机码-0
+	origin := fmt.Sprintf("%s-%s-%s", path, paramStr, key)   // 待签名字符串
+	md5hash := md5sum(origin)
+	result := fmt.Sprintf("%s://%s%s?auth_key=%s-%s", scheme, host, path, paramStr, md5hash) // 结果URL
+	if nil != args && len(args) > 0 {
+		result = fmt.Sprintf("%s&%s", result, args.Encode())
+	}
+	return result
+}
+
+func md5sum(str string) string {
+	h := md5.New()
+	h.Write([]byte(str))
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+var DefaultAudioArgs url.Values
+
+func init() {
+	DefaultAudioArgs = url.Values{}
+	DefaultAudioArgs.Add("onlyaudio", "1")
+}
+
+// 测试
+func main() {
+	expire := date.Now().AddDays(1)
+	scheme := `rtmp`
+	appName := `shy-live-test`
+	// 以下参数正常应由配置读取
+	pushHostName := `livepush-appmedia.zlzgy.org.cn`
+	pullHostName := `live-appmedia.zlzgy.org.cn`
+	pushKey := `N3IC5OfIBg`
+	pullKey := `utPuNSNngo`
+
+	// 视频直播
+	videoStreamName := wph.NextId().String()
+	videoPushUrl := GetLiveSingedURL(scheme, pushHostName, appName,
+		videoStreamName, pushKey, expire, nil)
+	println(`video push url:`, videoPushUrl)
+	videoPullUrl := GetLiveSingedURL(scheme, pullHostName, appName,
+		videoStreamName, pullKey, expire, nil)
+	println(`video pull url:`, videoPullUrl)
+
+	// 音频直播
+	audioStreamName := wph.NextId().String()
+	audioPushUrl := GetLiveSingedURL(scheme, pushHostName, appName,
+		audioStreamName, pushKey, expire, DefaultAudioArgs)
+	println(`audio push url:`, audioPushUrl)
+	audioPullUrl := GetLiveSingedURL(scheme, pullHostName, appName,
+		audioStreamName, pullKey, expire, DefaultAudioArgs)
+	println(`audio pull url:`, audioPullUrl)
+}

alicloud_kafka/ca-cert

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDPDCCAqWgAwIBAgIJAMRsb0DLM1fsMA0GCSqGSIb3DQEBBQUAMHIxCzAJBgNV
+BAYTAkNOMQswCQYDVQQIEwJIWjELMAkGA1UEBxMCSFoxCzAJBgNVBAoTAkFCMRAw
+DgYDVQQDEwdLYWZrYUNBMSowKAYJKoZIhvcNAQkBFht6aGVuZG9uZ2xpdS5semRA
+YWxpYmFiYS5jb20wIBcNMTcwMzA5MTI1MDUyWhgPMjEwMTAyMTcxMjUwNTJaMHIx
+CzAJBgNVBAYTAkNOMQswCQYDVQQIEwJIWjELMAkGA1UEBxMCSFoxCzAJBgNVBAoT
+AkFCMRAwDgYDVQQDEwdLYWZrYUNBMSowKAYJKoZIhvcNAQkBFht6aGVuZG9uZ2xp
+dS5semRAYWxpYmFiYS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBALZV
+bbIO1ULQQN853BTBgRfPiRJaAOWf38u8GC0TNp/E9qtI88A+79ywAP17k5WYJ7XS
+wXMOJ3h1qkQT2TYJVetZ6E69CUJq4BsOvNlNRvmnW6eFymh5QZsEz2MTooxJjVjC
+JQPlI2XRDjIrTVYEQWUDxj2JhB8VVPEed+6u4KQVAgMBAAGjgdcwgdQwHQYDVR0O
+BBYEFHFlOoiqQxXanVi2GUoDiKDD33ujMIGkBgNVHSMEgZwwgZmAFHFlOoiqQxXa
+nVi2GUoDiKDD33ujoXakdDByMQswCQYDVQQGEwJDTjELMAkGA1UECBMCSFoxCzAJ
+BgNVBAcTAkhaMQswCQYDVQQKEwJBQjEQMA4GA1UEAxMHS2Fma2FDQTEqMCgGCSqG
+SIb3DQEJARYbemhlbmRvbmdsaXUubHpkQGFsaWJhYmEuY29tggkAxGxvQMszV+ww
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQBTSz04p0AJXKl30sHw+UM/
+/k1jGFJzI5p0Z6l2JzKQYPP3PfE/biE8/rmiGYEenNqWNy1ZSniEHwa8L/Ux98ci
+4H0ZSpUrMo2+6bfuNW9X35CFPp5vYYJqftilJBKIJX3C3J1ruOuBR28UxE42xx4K
+pQ70wChNi914c4B+SxkGUg==
+-----END CERTIFICATE-----

alicloud_kafka/consumer.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"github.com/Shopify/sarama"
+	"github.com/bsm/sarama-cluster"
+	"io/ioutil"
+	"os"
+	"os/signal"
+)
+
+var consumer *cluster.Consumer
+var sig chan os.Signal
+
+func init() {
+	topics := []string{"chatRoom"}
+	brokers := []string{"47.94.239.212:9093", "47.94.91.146:9093", "47.94.154.247:9093"}
+	certPath := "./alicloud_kafka/ca-cert"
+	fmt.Println("init kafka consumer, it may take a few seconds...")
+
+	var err error
+
+	clusterCfg := cluster.NewConfig()
+
+	clusterCfg.Net.SASL.Enable = true
+	clusterCfg.Net.SASL.User = "wangyangming"
+	clusterCfg.Net.SASL.Password = "WangMy1920"
+	clusterCfg.Net.SASL.Handshake = true
+
+	certBytes, err := ioutil.ReadFile(certPath)
+	clientCertPool := x509.NewCertPool()
+	ok := clientCertPool.AppendCertsFromPEM(certBytes)
+	if !ok {
+		panic("kafka consumer failed to parse root certificate")
+	}
+
+	clusterCfg.Net.TLS.Config = &tls.Config{
+		//Certificates:       []tls.Certificate{},
+		RootCAs:            clientCertPool,
+		InsecureSkipVerify: true,
+	}
+
+	clusterCfg.Net.TLS.Enable = true
+	clusterCfg.Consumer.Return.Errors = true
+	clusterCfg.Consumer.Offsets.Initial = sarama.OffsetOldest
+	clusterCfg.Group.Return.Notifications = true
+	clusterCfg.Version = sarama.V0_10_2_1
+	if err = clusterCfg.Validate(); err != nil {
+		msg := fmt.Sprintf("Kafka consumer config invalidate. config: %v. err: %v", *clusterCfg, err)
+		fmt.Println(msg)
+		panic(msg)
+	}
+
+	// consumer group需要事先建立好，参考：https://help.aliyun.com/document_detail/99952.html?spm=a2c4g.11186623.6.564.2b74754bS6iQbg#title-sf4-w77-d85
+	consumer, err = cluster.NewConsumer(brokers, "consumer_group_test", topics, clusterCfg)
+	if err != nil {
+		msg := fmt.Sprintf("Create kafka consumer error: %v. config: %v", err, clusterCfg)
+		fmt.Println(msg)
+		panic(msg)
+	}
+
+	sig = make(chan os.Signal, 1)
+
+}
+
+func Start() {
+	go consume()
+}
+
+func consume() {
+	for {
+		select {
+		case msg, more := <-consumer.Messages():
+			if more {
+				fmt.Printf("Partition:%d, Offset:%d, Key:%s, Value:%s, Timestamp:%s\n", msg.Partition, msg.Offset, string(msg.Key), string(msg.Value), msg.Timestamp)
+				consumer.MarkOffset(msg, "") // mark message as processed
+			}
+		case err, more := <-consumer.Errors():
+			if more {
+				fmt.Printf("kafka consumer error: %v\n", err.Error())
+			}
+		case ntf, more := <-consumer.Notifications():
+			if more {
+				fmt.Printf("Kafka consumer rebalance: %v\n", ntf)
+			}
+		case <-sig:
+			fmt.Println("Stop consumer server...")
+			_ = consumer.Close()
+			return
+		}
+	}
+
+}
+
+func Stop(s os.Signal) {
+	fmt.Println("Recived kafka consumer stop signal...")
+	sig <- s
+	fmt.Println("kafka consumer stopped!!!")
+}
+
+func main() {
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, os.Interrupt)
+
+	Start()
+
+	select {
+	case s := <-signals:
+		Stop(s)
+	}
+}

alicloud_kafka/producer.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"github.com/Shopify/sarama"
+	"io/ioutil"
+	"strconv"
+	"time"
+)
+
+var producer sarama.SyncProducer
+
+func init() {
+	brokers := []string{"47.94.239.212:9093", "47.94.91.146:9093", "47.94.154.247:9093"}
+	certPath := "./alicloud_kafka/ca-cert"
+	fmt.Print("init kafka producer, it may take a few seconds to init the connection\n")
+
+	var err error
+
+	mqConfig := sarama.NewConfig()
+	mqConfig.Net.SASL.Enable = true
+	mqConfig.Net.SASL.User = "wangyangming"
+	mqConfig.Net.SASL.Password = "WangMy1920"
+	mqConfig.Net.SASL.Handshake = true
+	mqConfig.Version = sarama.V0_10_2_1
+	certBytes, err := ioutil.ReadFile(certPath)
+	clientCertPool := x509.NewCertPool()
+	ok := clientCertPool.AppendCertsFromPEM(certBytes)
+	if !ok {
+		panic("kafka producer failed to parse root certificate")
+	}
+
+	mqConfig.Net.TLS.Config = &tls.Config{
+		//Certificates:       []tls.Certificate{},
+		RootCAs:            clientCertPool,
+		InsecureSkipVerify: true,
+	}
+
+	mqConfig.Net.TLS.Enable = true
+	mqConfig.Producer.Return.Successes = true
+
+	if err = mqConfig.Validate(); err != nil {
+		msg := fmt.Sprintf("Kafka producer config invalidate. config: %v. err: %v", *mqConfig, err)
+		fmt.Println(msg)
+		panic(msg)
+	}
+
+	producer, err = sarama.NewSyncProducer(brokers, mqConfig)
+	if err != nil {
+		msg := fmt.Sprintf("Kafak producer create fail. err: %v", err)
+		fmt.Println(msg)
+		panic(msg)
+	}
+
+}
+
+func produce(topic string, key string, content string) error {
+	msg := &sarama.ProducerMessage{
+		Topic:     topic,
+		Key:       sarama.StringEncoder(key),
+		Value:     sarama.StringEncoder(content),
+		Timestamp: time.Now(),
+	}
+
+	_, _, err := producer.SendMessage(msg)
+	if err != nil {
+		msg := fmt.Sprintf("Send Error topic: %v. key: %v. content: %v", topic, key, content)
+		fmt.Println(msg)
+		return err
+	}
+	fmt.Printf("Send OK topic:%s key:%s value:%s\n", topic, key, content)
+
+	return nil
+}
+
+func main() {
+	//the key of the kafka messages
+	//do not set the same the key for all messages, it may cause partition im-balance
+	key := strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
+	value := "this is a kafka message!"
+	_ = produce("chatRoom", key, value)
+}

aog.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/sclevine/agouti"
+)
+
+func main() {
+	// cd := agouti.ChromeDriver()
+	// cd := agouti.PhantomJS()
+	// cd := agouti.Selenium()
+	cd := agouti.Selendroid("/Users/marion/go/src/github.com/tebeka/selenium/vendor/selenium-server-standalone-3.14.0.jar")
+
+	err := cd.Start()
+	if err != nil {
+		fmt.Println("Start webDriver:", err)
+		return
+	}
+	defer cd.Stop()
+
+	pg, err := cd.NewPage(agouti.Browser("chrome"))
+	if err != nil {
+		fmt.Println("NewPage:", err)
+		return
+	}
+
+	err = pg.SetPageLoad(10000)
+	if err != nil {
+		fmt.Println("SetPageLoad:", err)
+		return
+	}
+
+	err = pg.SetScriptTimeout(1000)
+	if err != nil {
+		fmt.Println("SetScriptTimeout:", err)
+		return
+	}
+
+	err = pg.Navigate("https://www.baidu.com/")
+	if err != nil {
+		fmt.Println("Navigate:", err)
+		return
+	}
+
+	for {
+
+		title, err := pg.Title()
+		if err != nil {
+			fmt.Println("Title:", err)
+			return
+		}
+		fmt.Println(title)
+		sc := pg.FindByLink("新闻")
+
+		// str, err := sc.Text()
+		// if err != nil {
+		//     fmt.Println("str:", err)
+		//     return
+		// }
+		// fmt.Println(str)
+
+		err = sc.Click()
+		if err != nil {
+			fmt.Println("click:", err)
+			return
+		}
+		title, err = pg.Title()
+		if err != nil {
+			fmt.Println("Title2:", err)
+			return
+		}
+		fmt.Println(title)
+
+		//sleep
+		time.Sleep(2 * time.Second)
+
+		err = pg.Back()
+		if err != nil {
+			fmt.Println("Back:", err)
+			return
+		}
+		//sleep
+		time.Sleep(2 * time.Second)
+
+	}
+}

browser_robot/browser_robot.go

@@ -0,0 +1,294 @@
+package browser_robot
+
+import (
+	"fmt"
+	"github.com/go-vgo/robotgo"
+	"github.com/tebeka/selenium"
+	"github.com/tebeka/selenium/chrome"
+	"log"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"time"
+)
+
+// ------------------------------------ 一条纯UI操作的可视化golang爬虫简单示例 ---------------------------------------------
+// TODO 仅为示例程序，需要优化结构，且实际使用缺少代理池、随机User-Agent替换、行为策略处理、页面访问历史和路由记录及处理等
+
+// download selenium standalone server jar
+// https://www.seleniumhq.org/download/
+// download gecko driver
+// https://github.com/mozilla/geckodriver/releases
+// download chrome driver
+// https://sites.google.com/a/chromium.org/chromedriver/downloads
+// or
+// http://npm.taobao.org/mirrors/chromedriver/
+
+// go get github.com/go-vgo/robotgo
+// git clone https://github.com/googleapis/google-api-go-client $GOPATH/src/google.golang.org/api
+// git clone https://github.com/googleapis/google-cloud-go $GOPATH/src/cloud.google.com/go
+// git clone https://github.com/golang/oauth2 $GOPATH/src/golang.org/x/oauth2
+// git clone https://github.com/census-instrumentation/opencensus-go $GOPATH/src/go.opencensus.io
+// cd $GOPATH/src/github.com/tebeka/selenium/vendor
+// go get -d ./...
+// go run init.go --alsologtostderr
+
+const (
+	seleniumPath         = "/Users/marion/go/src/github.com/tebeka/selenium/"
+	standaloneServerPath = seleniumPath + "vendor/selenium-server-standalone-3.141.59.jar"
+	geckoDriverPath      = seleniumPath + "vendor/geckodriver"
+	servicePort          = 9515
+)
+
+var quit = make(chan bool)
+var linkRex = regexp.MustCompile(`(\S*[^.]+\.[^.]+)\s+`)
+
+type osAttrs struct {
+	RunCommand       string
+	ChromeDriverPath string
+}
+type osMap map[string]*osAttrs
+
+var currentOS = osMap{
+	"windows": {RunCommand: "cmd /c start", ChromeDriverPath: seleniumPath + "vendor/chromedriver.exe"},
+	"darwin":  {RunCommand: "open", ChromeDriverPath: seleniumPath + "vendor/chromedriver"},
+	"linux":   {RunCommand: "xdg-open", ChromeDriverPath: seleniumPath + "vendor/chromedriver_linux"},
+}
+
+// 手工打开系统默认浏览器
+func (m osMap) Run(uri string) error {
+	attrs, ok := m[runtime.GOOS]
+	if !ok {
+		return fmt.Errorf("don't know how to open things on %s platform", runtime.GOOS)
+	}
+
+	cmd := exec.Command(attrs.RunCommand, uri)
+	return cmd.Start()
+}
+
+func GetCurrentDirectory() string {
+	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
+	if err != nil {
+		log.Fatal(err)
+
+	}
+	return strings.Replace(dir, "\\", "/", -1) //将\替换成/
+}
+
+func (m osMap) GetChromeDriverPath() string {
+	attrs, ok := m[runtime.GOOS]
+	if !ok {
+		return ""
+	}
+
+	return attrs.ChromeDriverPath
+	// return path.Join(GetCurrentDirectory(), attrs.ChromeDriverPath)
+}
+
+// 鼠标位置随机偏移
+func pos(x int, y int, xRange int, yRange int) (int, int) {
+	var xPixels, yPixels int
+	if xRange > 0 {
+		xPixels = rand.Intn(xRange / 2)
+	}
+	if rand.Intn(2) == 1 {
+		x = x + xPixels
+	} else {
+		x = x - xPixels
+	}
+	if yRange > 0 {
+		yPixels = rand.Intn(yRange / 2)
+	}
+	if rand.Intn(2) == 1 {
+		y = y + yPixels
+	} else {
+		y = y - yPixels
+	}
+	return x, y
+}
+
+func main() {
+	service, webDriver, err := StartChrome()
+	if err != nil {
+		panic(err)
+	}
+	defer func(s *selenium.Service, wd selenium.WebDriver) {
+		_ = wd.Quit()
+		_ = s.Stop()
+	}(service, webDriver)
+
+	sWidth, sHeight := robotgo.GetScreenSize()
+	fmt.Println("screen size:", sWidth, sHeight)
+
+	// 随机鼠标起始位置
+	robotgo.MoveMouseSmooth(pos(sWidth/2, sHeight/2, sWidth, sHeight)) // TODO 模拟人移动鼠标
+
+	// 打开百度首页
+	fmt.Println("1. open baidu home page")
+	err = webDriver.Get("https://www.baidu.com")
+	if err != nil {
+		fmt.Printf("Failed to load page: %s\n", err)
+		return
+	}
+	_ = webDriver.MaximizeWindow("")
+
+	// 搜索关键词，进入搜索结果列表页
+	fmt.Println("2. search keyword")
+	time.Sleep(time.Second * 2)
+
+	searchBar, _ := webDriver.FindElement(selenium.ByID, "kw")
+	if searchBar == nil {
+		fmt.Println("there no search bar in home page")
+		return
+	}
+	loc, _ := searchBar.Location()
+	// 160 是浏览器头部高度？
+	robotgo.MoveMouseSmooth(pos(sWidth/2, 160+loc.Y, 540, 36))
+	robotgo.Click()
+	time.Sleep(time.Second * 2)
+	robotgo.TypeString("LED广告屏") // TODO 模拟人打字
+	time.Sleep(time.Second * 3)
+
+	leftList, _ := webDriver.FindElement(selenium.ByID, "content_left")
+	if leftList == nil {
+		fmt.Println("there no links list in target page")
+		return
+	}
+
+	// 获取所有超链接豆腐块，但是后续需要注意筛除掉反爬的蜜罐超链接（例如display=none的）
+	items, _ := leftList.FindElements(selenium.ByTagName, "h3")
+	if len(items) == 0 {
+		fmt.Println("there no links in target page")
+		return
+	}
+	for _, item := range items {
+		if isDisplayed, _ := item.IsDisplayed(); !isDisplayed {
+			continue
+		}
+		a, _ := item.FindElement(selenium.ByTagName, "a")
+		if a == nil {
+			continue
+		}
+		href, _ := a.GetAttribute("href")
+
+		parent, err := item.FindElement(selenium.ByXPATH, "..")
+		if parent == nil || err != nil {
+			continue
+		}
+		if className, _ := parent.GetAttribute("class"); !strings.Contains(className, "c-container") {
+			parent, _ = parent.FindElement(selenium.ByXPATH, "..")
+		}
+		sammyLinks, _ := parent.FindElements(selenium.ByXPATH, "div[last()]//a")
+		if len(sammyLinks) > 0 {
+			isTargetSite := false
+			for _, a := range sammyLinks {
+				t, _ := a.Text()
+				if t == "" {
+					continue
+				}
+
+				arr := linkRex.FindStringSubmatch(t)
+				if len(arr) > 1 {
+					siteHost := strings.ToLower(strings.Trim(arr[1], "/"))
+					if siteHost == "www.novisled.com" { // 随便写了个网站测试用，后续处理，如果是目标网站，则继续移动鼠标到它对链接上点击处理，如果未显示先滚动到它的位置
+						println("found target site,host", siteHost, " href:", href)
+						isTargetSite = true
+						break
+					}
+				}
+			}
+			if isTargetSite {
+				p, _ := a.Location()
+				fmt.Println("link element position:", p.X, p.Y)
+				y := p.Y
+
+				// TODO 平滑滚动
+				if p.Y > sHeight {
+					rd := rand.Intn(50)
+					gap := p.Y - sHeight - rd
+					robotgo.ScrollMouse(gap, "down")
+					y = rd + 234
+				}
+
+				// 继续干点啥
+				fmt.Println("link element new position:", p.X, y)
+				robotgo.MoveMouseSmooth(p.X+30, y)
+				fmt.Println("3. go to target site")
+				robotgo.Click()
+			}
+		}
+	}
+
+	<-quit
+}
+
+// 启动火狐浏览器
+func StartFirefox() (service *selenium.Service, webDriver selenium.WebDriver, err error) {
+	opts := []selenium.ServiceOption{
+		// selenium.StartFrameBuffer(),           // Start an X frame buffer for the browser to run in.
+		selenium.GeckoDriver(geckoDriverPath), // Specify the path to GeckoDriver in order to use Firefox.
+		selenium.Output(os.Stderr),            // Output debug information to STDERR.
+	}
+
+	caps := selenium.Capabilities{
+		"browserName": "firefox",
+	}
+
+	// 启动GeckoDriver
+	service, err = selenium.NewGeckoDriverService(standaloneServerPath, servicePort, opts...)
+	if err != nil {
+		log.Printf("Error starting the GeckoDriver server: %v", err)
+		return
+	}
+
+	// 调起浏览器
+	webDriver, err = selenium.NewRemote(caps, fmt.Sprintf("http://localhost:%d/wd/hub", servicePort))
+	if err != nil {
+		log.Printf("Error starting the GeckoDriver web driver: %v", err)
+		return
+	}
+	return
+}
+
+// 启动谷歌浏览器
+func StartChrome() (service *selenium.Service, webDriver selenium.WebDriver, err error) {
+	var opts []selenium.ServiceOption
+
+	caps := selenium.Capabilities{
+		"browserName": "chrome",
+	}
+	// 禁止加载图片，加快渲染速度
+	//imagCaps := map[string]interface{}{
+	//	"profile.managed_default_content_settings.images": 2,
+	//}
+	chromeCaps := chrome.Capabilities{
+		// Prefs: imagCaps,
+		Path: "",
+		Args: []string{
+			"--start-maximized",
+			// "--headless", // 设置Chrome无头模式
+			"--no-sandbox",
+			"--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/604.4.7 (KHTML, like Gecko) Version/11.0.2 Safari/604.4.7", // 模拟user-agent，防反爬
+		},
+	}
+	caps.AddChrome(chromeCaps)
+
+	// 启动ChromeDriver
+	service, err = selenium.NewChromeDriverService(currentOS.GetChromeDriverPath(), servicePort, opts...)
+	if err != nil {
+		log.Printf("Error starting the ChromeDriver server: %v", err)
+		return
+	}
+
+	// 调起chrome浏览器
+	webDriver, err = selenium.NewRemote(caps, fmt.Sprintf("http://localhost:%d/wd/hub", servicePort))
+	if err != nil {
+		log.Printf("Error starting the ChromeDriver web driver: %v", err)
+		return
+	}
+	return
+}

ctx_cancel.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+var c chan bool
+
+func init() {
+	c = make(chan bool, 10)
+	for i := 0; i < 10; i++ {
+		c <- true
+	}
+}
+
+func main() {
+	ctx, cancel := context.WithCancel(context.Background())
+	close(c)
+	println(1)
+	cancel()
+	println(2)
+exit:
+	for {
+		select {
+		case data, ok := <-c:
+			fmt.Println(data, ok)
+			time.Sleep(1 * time.Second)
+		case <-ctx.Done():
+			println(3)
+			break exit
+		}
+	}
+	println(4)
+}

deploy/Dockerfile

@@ -0,0 +1,15 @@
+FROM ccr.ccs.tencentyun.com/aionnect/public:3.11
+
+RUN     mkdir -p /opt/hello-go /log/error /log/warn /log/info
+WORKDIR /opt/hello-go
+COPY    ./bin/ ./
+RUN     chmod +x ./hello-go
+
+ENV GIN_MODE release
+ENV PATH $PATH:/sbin:/opt/hello-go
+ENV LANG zh_CN.UTF-8
+
+EXPOSE 8080 10080
+
+ENTRYPOINT ["tini", "--"]
+CMD ["hello-go", "-c", "config", "2>&1", "&"]

entity/person.go

@@ -0,0 +1,7 @@
+package entity
+
+type Person struct {
+	Name    string
+	Address string
+	Age     int
+}

ffmpeg/ffmpeg_alac_mp3.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"git.wanpinghui.com/WPH/go_common/wph"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	inputFiles, err := listFiles(`./ffmpeg/mp3`, `.mp3`)
+	if nil != err {
+		println(`查找输入文件出错:`, err.Error())
+	}
+	if nil == inputFiles || len(inputFiles) == 0 {
+		return
+	}
+
+	var command, out string
+	for _, f := range inputFiles {
+		o := fmt.Sprintf(`./ffmpeg/output/%s.m4a`, strings.TrimRight(strings.ToLower(path.Base(f)), ".mp3"))
+		command, out, err = MP3ToM4A(f, o)
+		if nil != err {
+			println("error:", err.Error())
+			println("command:", command)
+			println("out:\n", out)
+		}
+	}
+}
+
+const (
+	wrongMP3Output = `Estimating duration from bitrate, this may be inaccurate`
+	infoCmdFormat  = `ffmpeg -i %s`
+	tempCmdFormat  = `ffmpeg -i %s -c:a alac -b:a 80k -c:v copy %s -y`
+	audioCmdFormat = `ffmpeg -i %s -movflags faststart -c:a aac -b:a 80k -vn %s -y`
+	transCmdFormat = `ffmpeg -i %s -movflags faststart -c:a aac -b:a 80k %s -y`
+)
+
+// mp3转m4a
+func MP3ToM4A(inputFileName, outputFileName string) (string, string, error) {
+	var err error
+	var command string
+
+	// 先查看文件信息
+	command = fmt.Sprintf(infoCmdFormat, inputFileName)
+	out, _ := RunCmd(command)
+	if strings.Contains(out, wrongMP3Output) { // 处理特殊编码的mp3文件
+		// 选用原始编码器先复写一份临时m4a，此文件含图像，体积特别大
+		tempFileName := fmt.Sprintf(`./%d.m4a`, wph.NextId())
+		command = fmt.Sprintf(tempCmdFormat, inputFileName, tempFileName)
+		out, err = RunCmd(command)
+		if nil != err {
+			return command, out, err
+		}
+		// 从临时文件中提取音频
+		command = fmt.Sprintf(audioCmdFormat, tempFileName, outputFileName)
+		out, err = RunCmd(command)
+		if nil != err {
+			return command, out, err
+		}
+		// 删除临时文件
+		_ = os.Remove(tempFileName)
+	} else { // 正常编码的mp3
+		command := fmt.Sprintf(transCmdFormat, inputFileName, outputFileName)
+		out, err = RunCmd(command)
+		if nil != err {
+			return command, out, err
+		}
+	}
+	return command, out, nil
+}
+
+func RunCmd(command string) (string, error) {
+	command = strings.TrimSpace(command)
+	if command == "" {
+		return "", nil
+	}
+
+	var cmdAndArgs []string
+	arr := strings.Split(command, " ")
+	for i := 0; i < len(arr); i++ {
+		part := strings.TrimSpace(arr[i])
+		if part == "" {
+			continue
+		}
+
+		cmdAndArgs = append(cmdAndArgs, part)
+	}
+	if nil == cmdAndArgs || len(cmdAndArgs) == 0 {
+		return "", nil
+	}
+	var cmd *exec.Cmd
+	if len(cmdAndArgs) == 1 {
+		cmd = exec.Command(cmdAndArgs[0])
+	} else {
+		cmd = exec.Command(cmdAndArgs[0], cmdAndArgs[1:]...)
+	}
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	err := cmd.Start()
+	if nil != err {
+		return out.String(), err
+	}
+	err = cmd.Wait()
+	return out.String(), err
+}
+
+func listFiles(dirPath string, extList ...string) ([]string, error) {
+	dir, err := os.Stat(dirPath)
+	if (nil != err && os.IsNotExist(err)) || !dir.IsDir() {
+		return nil, nil
+	}
+	err = nil
+
+	var files []string
+	err = filepath.Walk(dirPath, func(name string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		ext := strings.ToLower(path.Ext(name))
+		if wph.Contains(extList, ext) {
+			files = append(files, name)
+		}
+		return nil
+	})
+	if nil != err {
+		return nil, err
+	}
+	return files, nil
+}

ffmpeg/ffmpeg_cmd.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+const (
+	CMD                    = `ffmpeg`
+	FileName               = `./ffmpeg/mp4/wph.mp4`
+	TEMPDir                = `./ffmpeg/output/`
+	AudioPickUpArgs        = `-i %s -y -vn -acodec copy %s`                                          // 音频提取
+	VideoPickUpArgs        = `-i %s -y -an -vcodec copy %s`                                          // 视频提取
+	SRTPickUpArgs          = `-i %s -map 0:s:0 %s`                                                   // 文本字幕提取 or `-i %s  -an -vn -bsf:s mov2textsub -scodec copy -f rawvideo %s` or `-i %s -an -vn -c:s copy -f rawvideo -map 0:s %s`
+	ASSPickUpArgs          = `-i %s -an -vn -scodec copy %s`                                         // ASS字幕提取
+	VideoAndAudioMergeArgs = `-i %s -i %s -y -vcodec copy -acodec copy %s`                           // 音视频混编
+	SRTMapArgs             = `-i %s -i %s -c copy -c:s mov_text %s`                                  // 增加SRT字幕
+	ASSMapArgs             = `-i %s -i %s -map 0:0 -map 0:1 -map 1 -c:a copy -c:v copy -c:s copy %s` // 增加ASS字幕
+	PushFlv                = `-re -i %s -c copy -f flv %s`                                           // 推Flv流
+)
+
+func main() {
+	//_ = os.RemoveAll(TEMPDir)
+	//_ = os.Mkdir(TEMPDir, os.ModePerm)
+	//runCmd(CMD, strings.Split(fmt.Sprintf(AudioPickUpArgs, FileName, TEMPDir+`wph.aac`), " "))
+	//runCmd(CMD, strings.Split(fmt.Sprintf(VideoPickUpArgs, FileName, TEMPDir+`wph.mp4`), " "))
+	//if path.Ext(FileName) == "mkv" { // mkv文件才能提取出字幕流，mp4通常会报错
+	//	runCmd(CMD, strings.Split(fmt.Sprintf(SRTPickUpArgs, FileName, TEMPDir+`wph.txt`), " "))
+	//}
+	//runCmd(CMD, strings.Split(fmt.Sprintf(VideoAndAudioMergeArgs, TEMPDir+`wph.mp4`, TEMPDir+`wph.aac`, TEMPDir+`wph1.mp4`), " "))
+	runCmd(CMD, strings.Split(fmt.Sprintf(PushFlv, FileName, `rtmp://localhost:1935/live/test`), ` `))
+}
+
+func runCmd(name string, args []string) {
+	cmd := exec.Command(name, args...)
+	buf := new(bytes.Buffer)
+	cmd.Stdout = buf
+	err := cmd.Run()
+	if nil != err {
+		panic(err.Error())
+	} else {
+		println(buf.String())
+	}
+}

go.mod

@@ -0,0 +1,32 @@
+module git.aionnect.com/hello-go
+
+go 1.14
+
+require (
+	git.aionnect.com/aionnect/go-common v0.0.0-20200522082556-9e4068ef1418
+	git.wanpinghui.com/WPH/go_common v1.2.4
+	github.com/PuerkitoBio/goquery v1.5.1
+	github.com/Shopify/sarama v1.26.3
+	github.com/bsm/sarama-cluster v2.1.15+incompatible
+	github.com/cbroglie/mustache v1.0.1
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/eclipse/paho.mqtt.golang v1.2.0
+	github.com/gin-gonic/gin v1.6.3
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/go-vgo/robotgo v0.0.0-20200509164312-55a4babb8ca7
+	github.com/golang/protobuf v1.4.1
+	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/mattn/go-gtk v0.0.0-20191030024613-af2e013261f5
+	github.com/mattn/go-pointer v0.0.0-20190911064623-a0a44394634f // indirect
+	github.com/sclevine/agouti v3.0.0+incompatible
+	github.com/sirupsen/logrus v1.6.0
+	github.com/smallnest/rpcx v0.0.0-20200512151426-9e5976a9d1d6
+	github.com/sony/sonyflake v1.0.0
+	github.com/spf13/viper v1.7.0
+	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
+	github.com/tebeka/selenium v0.9.9
+	golang.org/x/net v0.0.0-20200421231249-e086a090c8fd
+	golang.org/x/text v0.3.2
+	google.golang.org/grpc v1.29.1
+	xorm.io/xorm v1.0.1
+)

gogtk.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"github.com/mattn/go-gtk/glib"
+	"github.com/mattn/go-gtk/gtk"
+	"os"
+)
+
+func main() {
+	gtk.Init(&os.Args) //环境初始化
+
+	window := gtk.NewWindow(gtk.WINDOW_TOPLEVEL) //创建窗口
+	window.SetPosition(gtk.WIN_POS_CENTER)       //设置窗口居中显示
+	window.SetTitle("GTK Go!")                   //设置标题
+	window.SetSizeRequest(300, 200)              //设置窗口的宽度和高度
+
+	layout := gtk.NewFixed() //创建固定布局
+	window.Add(layout)       //把布局添加到主窗口中
+
+	b1 := gtk.NewButton()       //新建按钮
+	b1.SetLabel("Hello world!") //设置内容
+	b1.SetSizeRequest(100, 50)  //设置按钮大小
+	layout.Put(b1, 0, 0)        //设置按钮在容器的位置
+	layout.Move(b1, 100, 75)    //移动按钮的位置，必须先put，再用move
+
+	tmp := 10
+	b1.Connect("pressed", HandleButton, &tmp)
+
+	window.ShowAll() //显示所有的控件
+
+	gtk.Main() //主事件循环，等待用户操作
+}
+
+func HandleButton(ctx *glib.CallbackContext) {
+	arg := ctx.Data()   //获取用户传递的参数，是空接口类型
+	p, ok := arg.(*int) //类型断言
+	if ok {             //如果ok为true，说明类型断言正确
+		fmt.Println("*p = ", *p) //用户传递传递的参数为&tmp，是一个变量的地址
+		*p = 250                 //操作指针所指向的内存
+	}
+
+	fmt.Println("按钮b1被按下")
+
+	d1 := gtk.NewDialog()
+	d1.SetTitle("Test Dialog")
+	d1.SetPosition(gtk.WIN_POS_CENTER)
+	d1.SetSizeRequest(200, 100)
+	d1.Show()
+
+	//gtk.MainQuit() //关闭gtk程序
+}

group_filter.go

@@ -0,0 +1,140 @@
+package main
+
+// 数据复杂分组测试
+
+func main() {
+	// 分组策略列表
+	policies := []GroupPolicy{
+		{Area, Group},
+		{Company, Hash},
+		{Title, Group},
+	}
+	// 待分组数据记录列表
+	items := []RecordItem{
+		{Area: "", Company: "", Title: "", IsLeader: ""},
+		{Area: "", Company: "", Title: "", IsLeader: ""},
+	}
+	// 根级，外面再加一层是为了适配下面的循环处理
+	rootClusters := FilterClusters{
+		{Seq: 0, Items: items},
+	}
+	// 循环变量
+	clusters := rootClusters
+	for _, policy := range policies {
+		// 依次按照分组策略处理
+		clusters = clusters.filter(policy)
+		if nil == clusters || len(clusters) == 0 {
+			return
+		}
+	}
+	if len(clusters) == 0 {
+		return
+	}
+
+	// 获得平铺的叶子节点
+	var results []RecordItem
+	for _, cluster := range clusters {
+		if nil == cluster.Items || len(cluster.Items) == 0 {
+			continue
+		}
+
+		results = append(results, cluster.Items...)
+	}
+	if nil == results || len(results) == 0 {
+		return
+	}
+
+	// 求箱子数量
+	length := len(results)
+	groupSize := 50
+	groupCount := length / groupSize
+	if length%groupSize > 0 {
+		groupCount++
+	}
+
+	// TODO 最后把results塞进箱子里就是
+}
+
+type RecordItem map[string]string // 数据记录
+// 聚类，多叉树形结构
+type FilterCluster struct {
+	Seq         int            // 序号
+	Items       []RecordItem   // 组内记录列表
+	SubClusters FilterClusters // 子级聚类集合
+}
+type FilterClusters []FilterCluster // 聚类集合
+
+// 分组策略字段
+const (
+	Area     = "area"
+	Company  = "company"
+	Title    = "title"
+	IsLeader = "isLeader"
+)
+
+// 分组策略类型
+type FilterType int8
+
+const (
+	Random FilterType = iota // 随机
+	Group                    // 聚合
+	Hash                     // 散列
+)
+
+// 分组策略
+type GroupPolicy struct {
+	Name string
+	Type FilterType
+}
+
+// 策略模式，分支处理，返回当前聚类集合再进一步处理的子级聚类集合
+func (clusters *FilterClusters) filter(policy GroupPolicy) FilterClusters {
+	if policy.Type == Random {
+		return clusters.mapping(policy.Name, random)
+	} else if policy.Type == Group {
+		return clusters.mapping(policy.Name, group)
+	} else if policy.Type == Hash {
+		return clusters.mapping(policy.Name, hash)
+	}
+	return nil
+}
+
+// 将子级聚类集合映射关联到父级，如此，虽然仅返回子级聚类集合，但通过根级对象可以遍历整棵树
+func (clusters *FilterClusters) mapping(name string, fn func(string, FilterCluster) FilterClusters) FilterClusters {
+	var allSubClusters FilterClusters
+	for i := 0; i < len(*clusters); i++ {
+		subClusters := fn(name, (*clusters)[i])
+		if nil == subClusters || len(subClusters) == 0 {
+			continue
+		}
+		(*clusters)[i].SubClusters = subClusters
+		allSubClusters = append(allSubClusters, subClusters...)
+	}
+	if nil == allSubClusters || len(allSubClusters) == 0 {
+		return nil
+	}
+	return allSubClusters
+}
+
+// TODO 随机算法实现
+func random(name string, cluster FilterCluster) FilterClusters {
+	// 建议子级聚类数量 = 当前聚类中数据记录内，指定name字段的取值数
+	// 记录随机排列即可
+	return nil
+}
+
+// TODO 聚合算法实现
+func group(name string, cluster FilterCluster) FilterClusters {
+	// 建议子级聚类数量 = 当前聚类中数据记录内，指定name字段的取值数
+	// 简单近似处理，group by即可
+	// 求最优解处理，建议多次迭代的kmeans
+	return nil
+}
+
+// TODO 散列算法实现
+func hash(name string, cluster FilterCluster) FilterClusters {
+	// 建议子级聚类数量 = 当前聚类中数据记录内，指定name字段的取值数
+	// 简单近似处理，写个循环分布一下数据即可
+	// 求最优解处理，建议多次迭代的kmeans
+	return nil
+}

grpc-demo/README.md

@@ -0,0 +1,208 @@
+# gRPC原生负载均衡原型
+
+目录example中是proto定义
+
+目录server中是服务实现
+
+目录client中是客户端实现
+
+目录deploy中是k8s相关部署文件
+
+目录py-example中是一个python的示例gRPC服务
+
+## 测试结论
+
+**gRPC原生库负载均衡实现**
+
+1. gRPC原生库在客户端用resolver支持负载均衡，验证通过，代码灵活可配置，但需要有额外的机制维护服务端暴露的动态gRPC节点列表
+
+**不使用resolver相关代码库，尝试k8s原生环境的负载均衡实现**
+
+2. 基于NodePort、ClusterIP的Service，可以访问通，但无负载均衡效果
+
+3. 基于LoadBalance的Service和相应4层负载均衡，或手工创建的4层负载均衡，可以访问通，但无负载均衡效果
+
+4. 腾讯云容器环境TKE默认的gcloud类型的ingress和相应7层负载均衡，无法访问通
+
+5. TKE需另外安装nginx controller，安装方法下面有说明，443 tls连接能访问通，但不是必须做证书验证，且正确实现负载均衡（目前推荐使用此方式，相关更改都是部署层面的，代码几乎没有修改）
+
+**使用服务中间件的负载均衡实现**
+
+6. Ambassador边缘网关中间件，安装方法下面有说明，http 1.1请求能正常转发，gRPC未调通
+
+7. Istio服务网格，每个Pod前端都加了网格代理，gRPC负载均衡自动能够实现，无需做其它额外配置
+
+grpcurl连接测试工具：https://github.com/fullstorydev/grpcurl
+
+## nginx ingress方式访问gRPC
+
+### nginx ingress安装
+
+先新建命名空间 ingress-nginx
+
+到 [ingress-nginx项目网站](https://github.com/kubernetes/ingress-nginx/tree/master/deploy/static) 找到最新的mandatory.yaml文件的源文件下载安装即可
+
+> kubectl apply -f mandatory.yaml
+
+时间可能较久，耐心等待，安装成功后会在ingress-nginx命名空间下新增nginx controller的pod
+
+建立一个指向此pod的service以便访问，注意一定同时映射80和443端口（nginx controller的gRPC支持实际在443上），且type推荐为ClusterIP以便最小化可访问范围和路由长度，参考ingress-nginx-service.yaml
+
+客户端访问nginx controller会按照域名分发请求，所以需要给上述pod或service配置域名解析，到外部dns服务里去配置，推荐指向上述service内网ip
+
+另外，经测试，在客户端pod的yaml文件中配置本地hosts的方式无效，而在kube-dns的configmap配置文件中添加配置可能导致整个集群dns失效
+
+### nginx ingress使用
+
+创建nginx类型的ingress即可，可参考本项目中的ingress.yaml
+
+ingress创建或更新后，会自动同步更新nginx controller中的配置，可以进入nginx controller pod查看nginx.conf
+
+> kubectl exec -n ingress-nginx -it <pod名> cat nginx.conf
+
+
+### 关于k8s证书创建
+
+> openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ex_tls.key -out ex_tls.crt -subj "/CN=example.wanpinghui.com/O=example.wanpinghui.com"
+
+> openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout gw_tls.key -out gw_tls.crt -subj "/CN=gw.wanpinghui.com/O=gw.wanpinghui.com"
+
+其中两个example.wanpinghui.com或gw.wanpinghui.com之类是证书对应域名
+
+> kubectl create secret tls ex-wph-secret --key ex_tls.key --cert ex_tls.crt
+
+> kubectl create secret tls gw-wph-secret --key gw_tls.key --cert gw_tls.crt
+
+将刚刚生成的证书加入k8s的secret
+
+如果具体某个nginx ingress不放在ingress-nginx命名空间，那么最好ingress-nginx命名空间和nginx ingress所在命名空间中都加一下secret
+
+## Ambassador网关方式访问gRPC（gRPC重定向未通）
+
+### Ambassador安装
+
+**下载edgectl**
+
+edgectl是必备的
+
+https://www.getambassador.io/reference/edgectl-download/
+
+但尝试用推荐的edgectl install命令安装未完全成功且运行不稳定，改用次选的Helm应用管理器方式安装
+
+**通过Helm安装**
+
+先在k8s集群上安装helm，腾讯云到集群管理后台打开即可
+
+下载Helm客户端
+
+```
+curl -O https://storage.googleapis.com/kubernetes-helm/helm-v2.10.0-linux-amd64.tar.gz
+tar xzvf helm-v2.10.0-linux-amd64.tar.gz
+sudo cp linux-amd64/helm /usr/local/bin/helm
+helm init --client-only
+```
+
+如果是在k8s集群中的节点机上，可以直接使用，推荐直接使用
+
+否则再执行如下操作
+
+以下是针对腾讯云环境，配置Helm客户端访问集群
+
+将helm对应的tiller service转换为内网负载均衡模式
+
+> kubectl patch svc $(kubectl get svc -l app=helm,name=tiller -n kube-system -o=jsonpath={.items[0].metadata.name}) -n kube-system -p '{"metadata":{"annotations":{"service.kubernetes.io/qcloud-loadbalancer-internal-subnetid":"subnet-88888888"}},"spec":{"type":"LoadBalancer"}}'
+
+等待负载均衡配置完成后，获取ip和端口信息
+
+> kubectl get svc -l app=helm,name=tiller -n kube-system -o=jsonpath={.items[0].status.loadBalancer.ingress[0].ip}
+
+> kubectl get svc -l app=helm,name=tiller -n kube-system -o=jsonpath={.items[0].spec.ports[0].port}
+
+以后，在helm客户端所在机器，即可执行如下命令导入环境变量，以切换连接指定集群
+
+可以将这个语句存成一个脚本，以便使用
+
+> export HELM_HOST=$EXTERNALIP:$PORT
+
+安装Ambassador
+
+> helm repo add datawire https://www.getambassador.io
+
+> kubectl create namespace ambassador
+
+> helm install --name ambassador --namespace ambassador datawire/ambassador
+
+安装完毕后会在ambassador命名空间中创建一堆crds，以及deployment、service以及service相应的4层公网负载均衡
+
+经测试，手工创建http 7层负载均衡指向node port无效，且公网访问仅用于管理，内网访问用service的内网ip即可，也并不仅限于http协议请求处理，所以，保留自动建立的负载均衡
+
+**获取Community Edition License**
+
+1. 新建一个域名指向ambassador service的公网ip（后续如更换域名，需要重复执行下面2、3步）
+
+2. 命令行工具登录
+
+> edgectl login --namespace=ambassador <访问域名>
+
+可能会提示缺少xdg-open，如果在CentOS上，通过以下命令安装
+
+> yum install -y xdg-utils
+
+3. 访问命令输出的admin网址
+
+4. 该网站首页中会有一个获取授权的豆腐块，在其中输入一个邮箱然后点击sign up
+
+5. 邮箱会收到邮件，根据邮件中的提示，执行edgectl命令行工具应用授权即可
+
+随后即可正常使用Ambassador，几个主要网址：
+
+/edge_stack/admin/ 或首页点击空白处即可进入 管理端，提供服务监控及Mapping等的图形化配置
+
+/docs/ 本地文档，这个网址就是通过Mapping配置的请求转发，关于用yaml配置Mapping做普通的http转发的示例，可见Ambassador官网Get Start
+
+## Istio服务网格
+
+参照Istio官网Get Start https://istio.io/docs/setup/getting-started/
+
+下载istioctl客户端后，kubectl切换到目标集群，执行一下命令即可安装Istio
+
+> istioctl manifest apply --set profile=demo
+
+对希望开启自动istio sidecar代理注入的k8s命名空间，执行一下命令
+
+> kubectl label namespace <命名空间名称> istio-injection=enabled
+
+然后，正常部署gRPC服务，正常访问即可，代码无需做任何改动，即可实现gGPC负载均衡
+
+其它Istio标签配置，路由配置，TLS证书，请求头处理，服务跟踪，金丝雀发布等，参考本项目中deloy中相关内容即可
+
+更多内容参考Istio文档
+
+## 相关资源清理语句
+
+```
+# ambassador相关
+kubectl delete service ambassador ambassador-admin ambassador-redis -n ambassador
+kubectl delete deployment ambassador ambassador-redis -n ambassador
+kubectl delete crds authservices.getambassador.io consulresolvers.getambassador.io filterpolicies.getambassador.io filters.getambassador.io hosts.getambassador.io kubernetesendpointresolvers.getambassador.io kubernetesserviceresolvers.getambassador.io logservices.getambassador.io mappings.getambassador.io ratelimits.getambassador.io ratelimitservices.getambassador.io tcpmappings.getambassador.io tlscontexts.getambassador.io tracingservices.getambassador.io modules.getambassador.io -n ambassador
+kubectl delete secret ambassador-edge-stack ambassador-internal -n ambassador
+kubectl delete namespace ambassador
+
+# 后端服务
+kubectl delete service grpc-example py-grpc-example quote -n prd
+kubectl delete deployment grpc-example py-grpc-example quote -n prd
+
+# helm相关
+kubectl delete service helm-api tiller-deploy -n kube-system
+kubectl delete deployment tiller-deploy -n kube-system
+
+# ingress-nginx相关
+kubectl delete secret ex-wph-secret gw-wph-secret -n prd
+kubectl delete secret ex-wph-secret gw-wph-secret -n ingress-nginx
+kubectl delete ingress py-grpc-example grpc-example -n prd
+kubectl delete deployment nginx-ingress-controller -n ingress-nginx
+kubectl delete service ingress-nginx-service -n ingress-nginx
+kubectl delete namespace ingress-nginx
+```
+
+

grpc-demo/client/load_balancing.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"../clientImpl"
+	"../constants"
+	"fmt"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/resolver"
+	"log"
+	"time"
+)
+
+// gRPC官方库负载均衡实现示例，客户端，参考 https://grpc-up-and-running.github.io/
+
+func main() {
+	// 以服务命名来建立与实际服务终结点的连接，默认仅连接第一个终结点
+	pickFirstConn, err := grpc.Dial(
+		fmt.Sprintf("%s:///%s", constants.ExampleScheme, constants.ExampleServiceName), // "grpc:///go-example" 注意必须是三个斜杠
+		grpc.WithInsecure(),
+	)
+	if err != nil {
+		log.Fatalf("did not connect: %v", err)
+	}
+	defer func() {
+		_ = pickFirstConn.Close()
+	}()
+
+	log.Println("==== Calling go-example with pick_first ====")
+	clientImpl.MakeCalls(pickFirstConn, 10)
+
+	// 以服务命名来建立与实际服务终结点的连接，设置以轮询策略连接
+	roundRobinConn, err := grpc.Dial(
+		fmt.Sprintf("%s:///%s", constants.ExampleScheme, constants.ExampleServiceName), // "grpc:///go-example" 注意必须是三个斜杠
+		//grpc.WithBalancerName(roundrobin.Name), // 官方仅提供pick first和round robin两种策略，更多负载均衡策略找第三方或自己实现
+		// 因为WithBalancerName过时了，推荐用WithDefaultServiceConfig，其参数是个服务配置的JSON
+		// 可配置的项目包括超时时间、最大请求响应大小等，参考 https://github.com/grpc/grpc/blob/master/doc/service_config.md
+		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingPolicy":"%s"}`, roundrobin.Name)),
+		grpc.WithInsecure(),
+	)
+	if err != nil {
+		log.Fatalf("did not connect: %v", err)
+	}
+	defer func() {
+		_ = roundRobinConn.Close()
+	}()
+
+	log.Println("==== Calling go-example with round_robin ====")
+	clientImpl.MakeCalls(roundRobinConn, 10)
+	// 停顿10秒再请求，此时可手工增减服务端启动的端口监听，以测试服务端部分节点掉线或新节点上线的情况
+	println(`sleeping ------------------------------------------------------->`)
+	time.Sleep(10 * time.Second)
+	clientImpl.MakeCalls(roundRobinConn, 10)
+}
+
+// 将resolver添加进resolvers map，必须在init函数中
+// 服务命名和实际服务终结点列表的映射关系即在此时建立
+// 此方案最不可行之处在于，在k8s这样的动态环境上，需要花费额外的成本和复杂度来维护这个映射
+func init() {
+	resolver.Register(&clientImpl.ExampleResolverBuilder{})
+}

grpc-demo/client/simple_client.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"../clientImpl"
+	"crypto/tls"
+	"git.wanpinghui.com/WPH/go_common/wph/logger"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/grpclog"
+	"log"
+	"os"
+	"time"
+)
+
+// 简单请求客户端，用于测试k8s环境本身的请求分发能力
+
+func main() {
+	//以k8s service名来建立连接
+
+	//serviceConn, err := grpc.Dial("grpc-example:80", grpc.WithInsecure())
+	//if err != nil {
+	//	log.Fatalf("did not connect: %v", err)
+	//}
+	//defer func() {
+	//	_ = serviceConn.Close()
+	//}()
+	//
+	//log.Println("==== Calling grpc-example by k8s service ====")
+	//clientImpl.MakeCalls(serviceConn, 10)
+
+	// 以k8s ingress来建立连接
+
+	// 用qcloud ingress访问，不论是在service中网络type设为负载均衡所添加的，或手工添加指向NodePort的4层负载均衡，对grpc均不能真正起到负载均衡的效果
+	// qcloud类型7层http/https ingress，直接无法访问通grpc服务
+	//ingressConn, err := grpc.Dial("10.7.255.37:80", grpc.WithInsecure())
+
+	// 用nginx ingress访问，通过配置DNS服务器做域名解析（推荐指向nginx controller内网service ip），在客户端的yaml文件中配置本地hosts的方式无效
+	// 详细访问路径：解析域名 -> nginx controller的service的ip -> nginx controller pod（TKE中额外手工部署ingress-nginx后添加的）->
+	// 转发到具体nginx ingres的yaml配置文件中指向的后端service（创建或更新nginx ingress时，其中的backend配置都会自动同步到nginx controller pod里的nginx.conf中，可进入该pod查看）
+	// 读文件证书
+	//certFile := `./ex_tls.crt` // 服务器上的证书路径，可以在构件docker镜像时复制，但k8s环境更建议从Secret中挂载
+	//isExisted, _ := pathExists(certFile)
+	//if !isExisted {
+	//	certFile = `/Users/marion/Documents/project/test/gotest/grpc-demo/tls/ex_tls.crt` // 本地测试时的证书文件绝对路径
+	//}
+	//crt, _ := credentials.NewClientTLSFromFile(certFile, "example.wanpinghui.com")
+	// 跳过证书验证，但直接grpc.WithInsecure()不通
+	var tlsConf tls.Config
+	tlsConf.InsecureSkipVerify = true
+	crt := credentials.NewTLS(&tlsConf)
+	ingressConn, err := grpc.Dial("example.wanpinghui.com:443", grpc.WithTransportCredentials(crt)) // 推荐把域名解析到nginx controller service内网ip
+	//ingressConn, err := grpc.Dial("example.wanpinghui.com:443", grpc.WithInsecure())
+	if err != nil {
+		log.Fatalf("did not connect: %v", err)
+	}
+	defer func() {
+		_ = ingressConn.Close()
+	}()
+	// 打印更详细的日志
+	l := logger.New()
+	grpclog.SetLoggerV2(l)
+
+	log.Println("==== Calling grpc-example by k8s ingress ====")
+	clientImpl.MakeCalls(ingressConn, 10)
+	// 停顿10秒再请求，此时可手工增减pod，以测试pod掉线或新节点上线的情况
+	println(`sleeping ------------------------------------------------------->`)
+	time.Sleep(10 * time.Second)
+	clientImpl.MakeCalls(ingressConn, 10)
+}
+
+func pathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}

grpc-demo/clientImpl/example_client.go

@@ -0,0 +1,29 @@
+package clientImpl
+
+import (
+	"../example"
+	"context"
+	"google.golang.org/grpc"
+	"time"
+)
+
+func Call(c example.FormatDataClient, input string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	r, err := c.DoFormat(ctx, &example.Data{Text: input})
+	if err != nil {
+		println("could not greet", err.Error())
+		return "", err
+	}
+	return r.Text, nil
+}
+
+func MakeCalls(cc *grpc.ClientConn, n int) {
+	hwc := example.NewFormatDataClient(cc)
+	for i := 0; i < n; i++ {
+		r, err := Call(hwc, "this is load balancing example")
+		if nil == err {
+			println(r)
+		}
+	}
+}

grpc-demo/clientImpl/example_resolver.go

@@ -0,0 +1,42 @@
+package clientImpl
+
+import (
+	"../constants"
+	"google.golang.org/grpc/resolver"
+)
+
+// resolver工厂实现类
+
+type ExampleResolverBuilder struct{}
+
+func (*ExampleResolverBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
+	r := &exampleResolver{
+		target: target,
+		cc:     cc,
+		addressStore: map[string][]string{
+			constants.ExampleServiceName: constants.ExampleEndpoints, // 服务命名与实际服务终结点列表的对应关系 "go-example": ["localhost:50051", "localhost:50052"]
+		},
+	}
+	r.start()
+	return r, nil
+}
+func (*ExampleResolverBuilder) Scheme() string { return constants.ExampleScheme } // "grpc"
+
+// resolver实现类
+
+type exampleResolver struct {
+	target       resolver.Target
+	cc           resolver.ClientConn
+	addressStore map[string][]string
+}
+
+func (r *exampleResolver) start() {
+	addrList := r.addressStore[r.target.Endpoint]
+	addresses := make([]resolver.Address, len(addrList))
+	for i, s := range addrList {
+		addresses[i] = resolver.Address{Addr: s}
+	}
+	r.cc.UpdateState(resolver.State{Addresses: addresses})
+}
+func (*exampleResolver) ResolveNow(resolver.ResolveNowOptions) {}
+func (*exampleResolver) Close()                                {}

grpc-demo/constants/constants.go

@@ -0,0 +1,12 @@
+package constants
+
+// 服务命名中的scheme和服务名，都可随意定义
+const (
+	ExampleScheme      = "grpc"
+	ExampleServiceName = "go-example"
+)
+
+// 实际的服务终结点地址
+//var ExampleEndpoints = []string{"localhost:50051", "localhost:50052"}
+//var ExampleEndpoints = []string{"localhost:50051", "localhost:50052", "localhost:50053"}
+var ExampleEndpoints = []string{"localhost:50051", "localhost:50052", "localhost:50053"}

grpc-demo/deploy/Dockerfile

@@ -0,0 +1,13 @@
+FROM ccr.ccs.tencentyun.com/wanpinghui/alpine
+
+RUN     mkdir -p /opt/go-example /log
+WORKDIR /opt/go-example
+COPY    bin ./
+RUN     chmod +x ./go-example
+
+EXPOSE 8080
+EXPOSE 50051
+ENV PATH $PATH:/sbin:/opt/go-example
+
+ENTRYPOINT ["tini", "--"]
+CMD ["go-example", ">>", "/log/go-example.log", "2>&1", "&"]

grpc-demo/deploy/client.yaml

@@ -0,0 +1,59 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: grpc-example-client
+  namespace: prd
+spec:
+  backoffLimit: 4
+  completions: 1
+  parallelism: 1
+  template:
+    spec:
+      containers:
+      - env:
+        - name: PATH
+          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/sbin:/sbin:/opt/go-example
+        - name: LANG
+          value: zh_CN.UTF-8
+        image: ccr.ccs.tencentyun.com/wanpinghui/go-example-client:1582972748
+        name: grpc-example-client
+        workingDir: /opt/go-example
+      imagePullSecrets:
+        - name: qcloudregistrykey
+        - name: tencenthubkey
+      restartPolicy: Never
+
+# 改变pod hosts文件，与containers同级，对ingress-nginx代理gRPC无效
+#      hostAliases:
+#      - ip: "10.7.255.164"
+#        hostnames:
+#        - "example.wanpinghui.com"
+#        - "gw.wanpinghui.com"
+
+# 挂载证书，与containers同级
+#        volumeMounts:
+#        - mountPath: /opt/go-example/ex_tls.crt
+#          name: ex-wph-secret
+#          readOnly: true
+#          subPath: ex_tls.crt
+#        - mountPath: /opt/go-example/gw_tls.crt
+#          name: gw-wph-secret
+#          readOnly: true
+#          subPath: gw_tls.crt
+#      volumes:
+#      - name: ex-wph-secret
+#        secret:
+#          defaultMode: 420
+#          items:
+#          - key: tls.crt
+#            mode: 420
+#            path: ex_tls.crt
+#          secretName: ex-wph-secret
+#      - name: gw-wph-secret
+#        secret:
+#          defaultMode: 420
+#          items:
+#          - key: tls.crt
+#            mode: 420
+#            path: gw_tls.crt
+#          secretName: gw-wph-secret

grpc-demo/deploy/client_build.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+registry_host="ccr.ccs.tencentyun.com/wanpinghui"
+project_name="go-example-client"
+exec_name="go-example"
+
+rm -rf ./bin
+mkdir -p ./bin
+#cp ../tls/*.crt ./bin/
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s" -v -o ./bin/$exec_name ../client/simple_client.go
+
+for item in $(docker images | grep -E "$registry_host/$project_name\s+\S*" -o);
+do
+    if [[ $item != "$registry_host/$project_name" ]];
+    then
+        echo "remove $registry_host/$project_name:$item"
+        docker rmi "$registry_host/$project_name:$item"
+    fi
+done
+
+tag=$(date +%s)
+docker login -u=100010573203  ccr.ccs.tencentyun.com -p=wanpinghui151217
+docker build --no-cache --disable-content-trust=true -t $registry_host/$project_name:$tag .
+
+docker push $registry_host/$project_name:$tag

grpc-demo/deploy/destination.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: DestinationRule
+metadata:
+  name: grpc-example
+  namespace: prd
+spec:
+  host: grpc-example
+  subsets:
+  - name: v1
+    labels:
+      version: v1
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: DestinationRule
+metadata:
+  name: http-backend
+  namespace: prd
+spec:
+  host: http-backend
+  subsets:
+  - name: v1
+    labels:
+      version: v1
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: DestinationRule
+metadata:
+  name: http-endpoint
+  namespace: prd
+spec:
+  host: http-endpoint
+  subsets:
+  - name: v1
+    labels:
+      version: v1
+  - name: v2
+    labels:
+      version: v2

grpc-demo/deploy/gateway.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: http-endpoint-gateway
+  namespace: prd
+spec:
+  selector:
+    istio: ingressgateway # use istio default controller
+  servers:
+  - port:
+      number: 80
+      name: http
+      protocol: HTTP
+    hosts:
+    - "*"
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: http-endpoint
+  namespace: prd
+spec:
+  hosts:
+  - "*"
+  gateways:
+  - http-endpoint-gateway
+  http:
+  - match:
+    - uri:
+        prefix: /test
+    route:
+    - destination:
+        host: http-endpoint
+#        subset: v1
+        port:
+          number: 8080
+#  - match:
+#    - headers:
+#        cookie:
+#          regex: "^(.*?;)?(user=jason)(;.*)?$"
+#    - uri:
+#        prefix: /test
+#    route:
+#    - destination:
+#        host: http-endpoint
+#        subset: v2
+#        port:
+#          number: 8080

grpc-demo/deploy/gateway_backend.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: grpc-example
+  namespace: prd
+spec:
+  hosts:
+  - grpc-example
+  http:
+  - route:
+    - destination:
+        host: grpc-example
+        subset: v1
+        port:
+          number: 50051
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: http-backend
+  namespace: prd
+spec:
+  hosts:
+  - http-backend
+  http:
+  - route:
+    - destination:
+        host: http-backend
+        subset: v1
+        port:
+          number: 8080

grpc-demo/deploy/grpc-example-mapping.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: getambassador.io/v2
+kind: Mapping
+metadata:
+  name: grpc-example-mapping
+  namespace: prd
+spec:
+  grpc: True
+  prefix: /example.FormatData/
+  rewrite: /example.FormatData/
+  service: grpc-example.prd:50051

grpc-demo/deploy/http_backend.yaml

@@ -0,0 +1,76 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: http-backend
+    service: http-backend
+  name: http-backend
+  namespace: prd
+spec:
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: http-backend
+  type: ClusterIP
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: http-backend
+  namespace: prd
+  labels:
+    account: http-backend
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: http-backend
+    version: v1
+  name: http-backend-v1
+  namespace: prd
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: http-backend
+      version: v1
+  template:
+    metadata:
+      labels:
+        app: http-backend
+        version: v1
+    spec:
+      serviceAccountName: http-backend
+      containers:
+      - env:
+        - name: PATH
+          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/sbin:/sbin:/opt/go-example
+        - name: LANG
+          value: zh_CN.UTF-8
+        - name: VERSION
+          value: v1
+        image: ccr.ccs.tencentyun.com/wanpinghui/http-backend:1583552944
+        name: http-backend
+        ports:
+        - containerPort: 8080
+          protocol: TCP
+          name: http-api
+        workingDir: /opt/go-example
+      initContainers:
+      - image: busybox
+        command:
+        - sh
+        - -c
+        - echo 65535 > /proc/sys/net/core/somaxconn
+        imagePullPolicy: Always
+        name: setsysctl
+        securityContext:
+          privileged: true
+      imagePullSecrets:
+      - name: qcloudregistrykey
+      - name: tencenthubkey

grpc-demo/deploy/http_backend_build.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+registry_host="ccr.ccs.tencentyun.com/wanpinghui"
+project_name="http-backend"
+exec_name="go-example"
+
+rm -rf ./bin
+mkdir -p ./bin
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s" -v -o ./bin/$exec_name ../server/http_backend.go
+
+for item in $(docker images | grep -E "$registry_host/$project_name\s+\S*" -o);
+do
+    if [[ $item != "$registry_host/$project_name" ]];
+    then
+        echo "remove $registry_host/$project_name:$item"
+        docker rmi "$registry_host/$project_name:$item"
+    fi
+done
+
+tag=$(date +%s)
+docker login -u=100010573203  ccr.ccs.tencentyun.com -p=wanpinghui151217
+docker build --no-cache --disable-content-trust=true -t $registry_host/$project_name:$tag .
+
+docker push $registry_host/$project_name:$tag

grpc-demo/deploy/http_endpoint.yaml

@@ -0,0 +1,126 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: http-endpoint
+    service: http-endpoint
+  name: http-endpoint
+  namespace: prd
+spec:
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: http-endpoint
+  type: ClusterIP
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: http-endpoint
+  namespace: prd
+  labels:
+    account: http-endpoint
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: http-endpoint
+    version: v1
+  name: http-endpoint-v1
+  namespace: prd
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: http-endpoint
+      version: v1
+  template:
+    metadata:
+      labels:
+        app: http-endpoint
+        version: v1
+    spec:
+      serviceAccountName: http-endpoint
+      containers:
+      - env:
+        - name: PATH
+          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/sbin:/sbin:/opt/go-example
+        - name: LANG
+          value: zh_CN.UTF-8
+        - name: VERSION
+          value: v1
+        image: ccr.ccs.tencentyun.com/wanpinghui/http-endpoint:1583722708
+        name: http-endpoint
+        ports:
+        - containerPort: 8080
+          protocol: TCP
+          name: http-api
+        workingDir: /opt/go-example
+      initContainers:
+      - image: busybox
+        command:
+        - sh
+        - -c
+        - echo 65535 > /proc/sys/net/core/somaxconn
+        imagePullPolicy: Always
+        name: setsysctl
+        securityContext:
+          privileged: true
+      imagePullSecrets:
+      - name: qcloudregistrykey
+      - name: tencenthubkey
+#---
+#apiVersion: apps/v1
+#kind: Deployment
+#metadata:
+#  labels:
+#    app: http-endpoint
+#    version: v2
+#  name: http-endpoint-v2
+#  namespace: prd
+#spec:
+#  replicas: 1
+#  selector:
+#    matchLabels:
+#      app: http-endpoint
+#      version: v2
+#  template:
+#    metadata:
+#      labels:
+#        app: http-endpoint
+#        version: v2
+#    spec:
+#      serviceAccountName: http-endpoint
+#      containers:
+#      - env:
+#        - name: PATH
+#          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/sbin:/sbin:/opt/go-example
+#        - name: LANG
+#          value: zh_CN.UTF-8
+#        - name: VERSION
+#          value: v2
+#        image: ccr.ccs.tencentyun.com/wanpinghui/http-endpoint:1583722730
+#        name: http-endpoint
+#        ports:
+#        - containerPort: 8080
+#          protocol: TCP
+#          name: http-api
+#        workingDir: /opt/go-example
+#      initContainers:
+#      - image: busybox
+#        command:
+#        - sh
+#        - -c
+#        - echo 65535 > /proc/sys/net/core/somaxconn
+#        imagePullPolicy: Always
+#        name: setsysctl
+#        securityContext:
+#          privileged: true
+#      imagePullSecrets:
+#      - name: qcloudregistrykey
+#      - name: tencenthubkey

grpc-demo/deploy/http_endpoint_blue.yaml

@@ -0,0 +1,73 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: http-endpoint-blue
+    service: http-endpoint-blue
+  name: http-endpoint-blue
+  namespace: prd
+spec:
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: http-endpoint-blue
+  type: NodePort
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: http-endpoint-blue
+  namespace: prd
+  labels:
+    account: http-endpoint-blue
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: http-endpoint-blue
+  name: http-endpoint-blue
+  namespace: prd
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: http-endpoint-blue
+  template:
+    metadata:
+      labels:
+        app: http-endpoint-blue
+    spec:
+      serviceAccountName: http-endpoint-blue
+      containers:
+      - env:
+        - name: PATH
+          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/sbin:/sbin:/opt/go-example
+        - name: LANG
+          value: zh_CN.UTF-8
+        - name: VERSION
+          value: blue
+        image: ccr.ccs.tencentyun.com/wanpinghui/http-endpoint:1583742756
+        name: http-endpoint-blue
+        ports:
+        - containerPort: 8080
+          protocol: TCP
+          name: http-api
+        workingDir: /opt/go-example
+      initContainers:
+      - image: busybox
+        command:
+        - sh
+        - -c
+        - echo 65535 > /proc/sys/net/core/somaxconn
+        imagePullPolicy: Always
+        name: setsysctl
+        securityContext:
+          privileged: true
+      imagePullSecrets:
+      - name: qcloudregistrykey
+      - name: tencenthubkey

grpc-demo/deploy/http_endpoint_blue_build.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+registry_host="ccr.ccs.tencentyun.com/wanpinghui"
+project_name="http-endpoint"
+exec_name="go-example"
+
+rm -rf ./bin
+mkdir -p ./bin
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s" -v -o ./bin/$exec_name ../server/http_endpoint_blue.go
+
+for item in $(docker images | grep -E "$registry_host/$project_name\s+\S*" -o);
+do
+    if [[ $item != "$registry_host/$project_name" ]];
+    then
+        echo "remove $registry_host/$project_name:$item"
+        docker rmi "$registry_host/$project_name:$item"
+    fi
+done
+
+tag=$(date +%s)
+docker login -u=100010573203  ccr.ccs.tencentyun.com -p=wanpinghui151217
+docker build --no-cache --disable-content-trust=true -t $registry_host/$project_name:$tag .
+
+docker push $registry_host/$project_name:$tag

grpc-demo/deploy/http_endpoint_v1_build.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+registry_host="ccr.ccs.tencentyun.com/wanpinghui"
+project_name="http-endpoint"
+exec_name="go-example"
+
+rm -rf ./bin
+mkdir -p ./bin
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s" -v -o ./bin/$exec_name ../server/http_endpoint_v1.go
+
+for item in $(docker images | grep -E "$registry_host/$project_name\s+\S*" -o);
+do
+    if [[ $item != "$registry_host/$project_name" ]];
+    then
+        echo "remove $registry_host/$project_name:$item"
+        docker rmi "$registry_host/$project_name:$item"
+    fi
+done
+
+tag=$(date +%s)
+docker login -u=100010573203  ccr.ccs.tencentyun.com -p=wanpinghui151217
+docker build --no-cache --disable-content-trust=true -t $registry_host/$project_name:$tag .
+
+docker push $registry_host/$project_name:$tag

grpc-demo/deploy/http_endpoint_v2_build.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+registry_host="ccr.ccs.tencentyun.com/wanpinghui"
+project_name="http-endpoint"
+exec_name="go-example"
+
+rm -rf ./bin
+mkdir -p ./bin
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s" -v -o ./bin/$exec_name ../server/http_endpoint_v2.go
+
+for item in $(docker images | grep -E "$registry_host/$project_name\s+\S*" -o);
+do
+    if [[ $item != "$registry_host/$project_name" ]];
+    then
+        echo "remove $registry_host/$project_name:$item"
+        docker rmi "$registry_host/$project_name:$item"
+    fi
+done
+
+tag=$(date +%s)
+docker login -u=100010573203  ccr.ccs.tencentyun.com -p=wanpinghui151217
+docker build --no-cache --disable-content-trust=true -t $registry_host/$project_name:$tag .
+
+docker push $registry_host/$project_name:$tag

grpc-demo/deploy/ingress-nginx-service.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.kubernetes.io/qcloud-loadbalancer-internal-subnetid: subnet-bkmrev02
+  name: ingress-nginx-service
+  namespace: ingress-nginx
+spec:
+  externalTrafficPolicy: Cluster
+  ports:
+    - name: 80-80-tcp
+      port: 80
+      protocol: TCP
+      targetPort: 80
+    - name: 443-443-tcp
+      port: 443
+      protocol: TCP
+      targetPort: 443
+  selector:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+  sessionAffinity: None
+  type: ClusterIP

grpc-demo/deploy/ingress.yaml

@@ -0,0 +1,52 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+  name: grpc-example
+  namespace: prd
+spec:
+  tls:
+  - hosts:
+    - example.wanpinghui.com
+    secretName: ex-wph-secret
+  rules:
+  - host: example.wanpinghui.com
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: grpc-example
+          servicePort: grpc
+
+# nginx ingress controller 1.8之前的版本使用
+# nginx.ingress.kubernetes.io/grpc-backend: "true"
+
+# qcloud类型ingress转发配置，不支持gRPC
+#apiVersion: extensions/v1beta1
+#kind: Ingress
+#metadata:
+#  annotations:
+#    kubernetes.io/ingress.class: qcloud
+#    kubernetes.io/ingress.http-rules: '[{"path":"/","backend":{"serviceName":"grpc-example","servicePort":"50051"}}]'
+#    kubernetes.io/ingress.https-rules: "null"
+#    kubernetes.io/ingress.rule-mix: "false"
+#    kubernetes.io/ingress.subnetId: subnet-bkmrev02
+#  name: grpc-example
+#  namespace: prd
+#spec:
+#  rules:
+#    - http:
+#        paths:
+#          - path: /
+#            backend:
+#              serviceName: grpc-example
+#              servicePort: grpc
+#
+#status: {}

grpc-demo/deploy/server.yaml

@@ -0,0 +1,76 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: grpc-example
+    service: grpc-example
+  name: grpc-example
+  namespace: prd
+spec:
+  ports:
+    - name: grpc
+      port: 50051
+      protocol: TCP
+      targetPort: 50051
+  selector:
+    app: grpc-example
+  type: ClusterIP
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: grpc-example
+  namespace: prd
+  labels:
+    account: grpc-example
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: grpc-example
+    version: v1
+  name: grpc-example-v1
+  namespace: prd
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: grpc-example
+      version: v1
+  template:
+    metadata:
+      labels:
+        app: grpc-example
+        version: v1
+    spec:
+      serviceAccountName: grpc-example
+      containers:
+      - env:
+        - name: PATH
+          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/sbin:/sbin:/opt/go-example
+        - name: LANG
+          value: zh_CN.UTF-8
+        - name: VERSION
+          value: v1
+        image: ccr.ccs.tencentyun.com/wanpinghui/go-example-server:1583552883
+        name: grpc-example
+        ports:
+        - containerPort: 50051
+          protocol: TCP
+          name: grpc-api
+        workingDir: /opt/go-example
+      initContainers:
+      - image: busybox
+        command:
+        - sh
+        - -c
+        - echo 65535 > /proc/sys/net/core/somaxconn
+        imagePullPolicy: Always
+        name: setsysctl
+        securityContext:
+          privileged: true
+      imagePullSecrets:
+      - name: qcloudregistrykey
+      - name: tencenthubkey

grpc-demo/deploy/server_build.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+registry_host="ccr.ccs.tencentyun.com/wanpinghui"
+project_name="go-example-server"
+exec_name="go-example"
+
+rm -rf ./bin
+mkdir -p ./bin
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s" -v -o ./bin/$exec_name ../server/load_balancing1.go
+
+for item in $(docker images | grep -E "$registry_host/$project_name\s+\S*" -o);
+do
+    if [[ $item != "$registry_host/$project_name" ]];
+    then
+        echo "remove $registry_host/$project_name:$item"
+        docker rmi "$registry_host/$project_name:$item"
+    fi
+done
+
+tag=$(date +%s)
+docker login -u=100010573203  ccr.ccs.tencentyun.com -p=wanpinghui151217
+docker build --no-cache --disable-content-trust=true -t $registry_host/$project_name:$tag .
+
+docker push $registry_host/$project_name:$tag

grpc-demo/example/data.pb.go

@@ -0,0 +1,151 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: data.proto
+
+package example
+
+import (
+	context "context"
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type Data struct {
+	Text                 string   `protobuf:"bytes,1,opt,name=text,proto3" json:"text,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Data) Reset()         { *m = Data{} }
+func (m *Data) String() string { return proto.CompactTextString(m) }
+func (*Data) ProtoMessage()    {}
+func (*Data) Descriptor() ([]byte, []int) {
+	return fileDescriptor_871986018790d2fd, []int{0}
+}
+
+func (m *Data) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Data.Unmarshal(m, b)
+}
+func (m *Data) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Data.Marshal(b, m, deterministic)
+}
+func (m *Data) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Data.Merge(m, src)
+}
+func (m *Data) XXX_Size() int {
+	return xxx_messageInfo_Data.Size(m)
+}
+func (m *Data) XXX_DiscardUnknown() {
+	xxx_messageInfo_Data.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Data proto.InternalMessageInfo
+
+func (m *Data) GetText() string {
+	if m != nil {
+		return m.Text
+	}
+	return ""
+}
+
+func init() {
+	proto.RegisterType((*Data)(nil), "example.Data")
+}
+
+func init() { proto.RegisterFile("data.proto", fileDescriptor_871986018790d2fd) }
+
+var fileDescriptor_871986018790d2fd = []byte{
+	// 106 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4a, 0x49, 0x2c, 0x49,
+	0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x4f, 0xad, 0x48, 0xcc, 0x2d, 0xc8, 0x49, 0x55,
+	0x92, 0xe2, 0x62, 0x71, 0x49, 0x2c, 0x49, 0x14, 0x12, 0xe2, 0x62, 0x29, 0x49, 0xad, 0x28, 0x91,
+	0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x02, 0xb3, 0x8d, 0x2c, 0xb8, 0xb8, 0xdc, 0xf2, 0x8b, 0x72,
+	0x13, 0x4b, 0xc0, 0x2a, 0xb4, 0xb8, 0x38, 0x5c, 0xf2, 0x21, 0x7c, 0x21, 0x5e, 0x3d, 0xa8, 0x7e,
+	0x3d, 0x90, 0x94, 0x14, 0x2a, 0x57, 0x89, 0x21, 0x89, 0x0d, 0x6c, 0x8b, 0x31, 0x20, 0x00, 0x00,
+	0xff, 0xff, 0x33, 0xcb, 0xa7, 0xd5, 0x73, 0x00, 0x00, 0x00,
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// FormatDataClient is the client API for FormatData service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type FormatDataClient interface {
+	DoFormat(ctx context.Context, in *Data, opts ...grpc.CallOption) (*Data, error)
+}
+
+type formatDataClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewFormatDataClient(cc *grpc.ClientConn) FormatDataClient {
+	return &formatDataClient{cc}
+}
+
+func (c *formatDataClient) DoFormat(ctx context.Context, in *Data, opts ...grpc.CallOption) (*Data, error) {
+	out := new(Data)
+	err := c.cc.Invoke(ctx, "/example.FormatData/DoFormat", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// FormatDataServer is the server API for FormatData service.
+type FormatDataServer interface {
+	DoFormat(context.Context, *Data) (*Data, error)
+}
+
+func RegisterFormatDataServer(s *grpc.Server, srv FormatDataServer) {
+	s.RegisterService(&_FormatData_serviceDesc, srv)
+}
+
+func _FormatData_DoFormat_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Data)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(FormatDataServer).DoFormat(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/example.FormatData/DoFormat",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(FormatDataServer).DoFormat(ctx, req.(*Data))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _FormatData_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "example.FormatData",
+	HandlerType: (*FormatDataServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "DoFormat",
+			Handler:    _FormatData_DoFormat_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "data.proto",
+}

grpc-demo/example/data.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+package example;
+
+message Data {
+    string text = 1;
+}
+
+service FormatData {
+    rpc DoFormat(Data) returns (Data){}
+}
+
+// protoc -I=. --go_out=plugins=grpc:. ./data.proto

grpc-demo/py-example/.gitignore

@@ -0,0 +1 @@
+*.pyc

grpc-demo/py-example/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:2.7
+WORKDIR /grpc
+ENV PATH "$PATH:/grpc"
+COPY greeter_server.py /grpc
+COPY helloworld_pb2.py /grpc
+COPY helloworld_pb2_grpc.py /grpc
+RUN python -m pip install grpcio
+RUN python -m pip install grpcio-tools googleapis-common-protos
+CMD ["python", "./greeter_server.py"]
+EXPOSE 50051

grpc-demo/py-example/README.md

@@ -0,0 +1 @@
+[This code's documentation lives on the grpc.io site.](https://grpc.io/docs/quickstart/python/)

grpc-demo/py-example/greeter_client.py

@@ -0,0 +1,38 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The Python implementation of the GRPC helloworld.Greeter client."""
+
+from __future__ import print_function
+import logging
+
+import grpc
+
+import helloworld_pb2
+import helloworld_pb2_grpc
+
+
+def run():
+    # NOTE(gRPC Python Team): .close() is possible on a channel and should be
+    # used in circumstances in which the with statement does not fit the needs
+    # of the code.
+    with grpc.insecure_channel('gw.wanpinghui.com:443') as channel:
+    #with grpc.insecure_channel('localhost:50051') as channel:
+        stub = helloworld_pb2_grpc.GreeterStub(channel)
+        response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
+    print("Greeter client received: " + response.message)
+
+
+if __name__ == '__main__':
+    logging.basicConfig()
+    run()

grpc-demo/py-example/greeter_client_with_options.py

@@ -0,0 +1,46 @@
+# Copyright 2018 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""gRPC Python helloworld.Greeter client with channel options and call timeout parameters."""
+
+from __future__ import print_function
+import logging
+
+import grpc
+
+import helloworld_pb2
+import helloworld_pb2_grpc
+
+
+def run():
+    # NOTE(gRPC Python Team): .close() is possible on a channel and should be
+    # used in circumstances in which the with statement does not fit the needs
+    # of the code.
+    #
+    # For more channel options, please see https://grpc.io/grpc/core/group__grpc__arg__keys.html
+    with grpc.insecure_channel(target='localhost:50051',
+                               options=[('grpc.lb_policy_name', 'pick_first'),
+                                        ('grpc.enable_retries', 0),
+                                        ('grpc.keepalive_timeout_ms', 10000)
+                                       ]) as channel:
+        stub = helloworld_pb2_grpc.GreeterStub(channel)
+        # Timeout in seconds.
+        # Please refer gRPC Python documents for more detail. https://grpc.io/grpc/python/grpc.html
+        response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'),
+                                 timeout=10)
+    print("Greeter client received: " + response.message)
+
+
+if __name__ == '__main__':
+    logging.basicConfig()
+    run()

grpc-demo/py-example/greeter_server.py

@@ -0,0 +1,41 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The Python implementation of the GRPC helloworld.Greeter server."""
+
+from concurrent import futures
+import logging
+
+import grpc
+
+import helloworld_pb2
+import helloworld_pb2_grpc
+
+
+class Greeter(helloworld_pb2_grpc.GreeterServicer):
+
+    def SayHello(self, request, context):
+        return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)
+
+
+def serve():
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
+    server.add_insecure_port('[::]:50051')
+    server.start()
+    server.wait_for_termination()
+
+
+if __name__ == '__main__':
+    logging.basicConfig()
+    serve()

grpc-demo/py-example/greeter_server_with_reflection.py

@@ -0,0 +1,47 @@
+# Copyright 2018 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The reflection-enabled version of gRPC helloworld.Greeter server."""
+
+from concurrent import futures
+import logging
+
+import grpc
+from grpc_reflection.v1alpha import reflection
+
+import helloworld_pb2
+import helloworld_pb2_grpc
+
+
+class Greeter(helloworld_pb2_grpc.GreeterServicer):
+
+    def SayHello(self, request, context):
+        return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)
+
+
+def serve():
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
+    SERVICE_NAMES = (
+        helloworld_pb2.DESCRIPTOR.services_by_name['Greeter'].full_name,
+        reflection.SERVICE_NAME,
+    )
+    reflection.enable_server_reflection(SERVICE_NAMES, server)
+    server.add_insecure_port('[::]:50051')
+    server.start()
+    server.wait_for_termination()
+
+
+if __name__ == '__main__':
+    logging.basicConfig()
+    serve()

grpc-demo/py-example/helloworld_pb2.py

@@ -0,0 +1,134 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: helloworld.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='helloworld.proto',
+  package='helloworld',
+  syntax='proto3',
+  serialized_pb=_b('\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2I\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x42\x36\n\x1bio.grpc.examples.helloworldB\x0fHelloWorldProtoP\x01\xa2\x02\x03HLWb\x06proto3')
+)
+
+
+
+
+_HELLOREQUEST = _descriptor.Descriptor(
+  name='HelloRequest',
+  full_name='helloworld.HelloRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='name', full_name='helloworld.HelloRequest.name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=32,
+  serialized_end=60,
+)
+
+
+_HELLOREPLY = _descriptor.Descriptor(
+  name='HelloReply',
+  full_name='helloworld.HelloReply',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='message', full_name='helloworld.HelloReply.message', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=62,
+  serialized_end=91,
+)
+
+DESCRIPTOR.message_types_by_name['HelloRequest'] = _HELLOREQUEST
+DESCRIPTOR.message_types_by_name['HelloReply'] = _HELLOREPLY
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+HelloRequest = _reflection.GeneratedProtocolMessageType('HelloRequest', (_message.Message,), dict(
+  DESCRIPTOR = _HELLOREQUEST,
+  __module__ = 'helloworld_pb2'
+  # @@protoc_insertion_point(class_scope:helloworld.HelloRequest)
+  ))
+_sym_db.RegisterMessage(HelloRequest)
+
+HelloReply = _reflection.GeneratedProtocolMessageType('HelloReply', (_message.Message,), dict(
+  DESCRIPTOR = _HELLOREPLY,
+  __module__ = 'helloworld_pb2'
+  # @@protoc_insertion_point(class_scope:helloworld.HelloReply)
+  ))
+_sym_db.RegisterMessage(HelloReply)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW'))
+
+_GREETER = _descriptor.ServiceDescriptor(
+  name='Greeter',
+  full_name='helloworld.Greeter',
+  file=DESCRIPTOR,
+  index=0,
+  options=None,
+  serialized_start=93,
+  serialized_end=166,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='SayHello',
+    full_name='helloworld.Greeter.SayHello',
+    index=0,
+    containing_service=None,
+    input_type=_HELLOREQUEST,
+    output_type=_HELLOREPLY,
+    options=None,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_GREETER)
+
+DESCRIPTOR.services_by_name['Greeter'] = _GREETER
+
+# @@protoc_insertion_point(module_scope)

grpc-demo/py-example/helloworld_pb2_grpc.py

@@ -0,0 +1,45 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+import grpc
+
+import helloworld_pb2 as helloworld__pb2
+
+
+class GreeterStub(object):
+  """The greeting service definition.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.SayHello = channel.unary_unary(
+        '/helloworld.Greeter/SayHello',
+        request_serializer=helloworld__pb2.HelloRequest.SerializeToString,
+        response_deserializer=helloworld__pb2.HelloReply.FromString,
+        )
+
+
+class GreeterServicer(object):
+  """The greeting service definition.
+  """
+
+  def SayHello(self, request, context):
+    """Sends a greeting
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_GreeterServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'SayHello': grpc.unary_unary_rpc_method_handler(
+          servicer.SayHello,
+          request_deserializer=helloworld__pb2.HelloRequest.FromString,
+          response_serializer=helloworld__pb2.HelloReply.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'helloworld.Greeter', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))

grpc-demo/py-example/py-grpc-example-mapping.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: getambassador.io/v2
+kind: Mapping
+metadata:
+  name: grpc-py
+  namespace: prd
+spec:
+  grpc: True
+  prefix: /helloworld.Greeter/
+  rewrite: /helloworld.Greeter/
+  service: py-grpc-example.prd:50051

grpc-demo/py-example/py-grpc-example.yaml

@@ -0,0 +1,48 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: py-grpc-example
+    version: v1
+  name: py-grpc-example
+  namespace: prd
+spec:
+  type: ClusterIP
+  ports:
+  - name: grpc-greet
+    port: 50051
+    protocol: TCP
+    targetPort: 50051
+  selector:
+    app: py-grpc-example
+    version: v1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: py-grpc-example
+  namespace: prd
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: py-grpc-example
+      version: v1
+  template:
+    metadata:
+      labels:
+        app: py-grpc-example
+        version: v1
+    spec:
+      containers:
+      - name: py-grpc-example
+        image: ccr.ccs.tencentyun.com/wanpinghui/py-example-server:1582956676
+        ports:
+        - name: grpc-api
+          protocol: TCP
+          containerPort: 50051
+      imagePullSecrets:
+        - name: qcloudregistrykey
+        - name: tencenthubkey
+      restartPolicy: Always

grpc-demo/py-example/py-ingress.yaml

@@ -0,0 +1,26 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+  name: py-grpc-example
+  namespace: prd
+spec:
+  tls:
+  - hosts:
+    - gw.wanpinghui.com
+    secretName: gw-wph-secret
+  rules:
+  - host: gw.wanpinghui.com
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: py-grpc-example
+          servicePort: grpc-greet

grpc-demo/py-example/server_build.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+registry_host="ccr.ccs.tencentyun.com/wanpinghui"
+project_name="py-example-server"
+
+for item in $(docker images | grep -E "$registry_host/$project_name\s+\S*" -o);
+do
+    if [[ $item != "$registry_host/$project_name" ]];
+    then
+        echo "remove $registry_host/$project_name:$item"
+        docker rmi "$registry_host/$project_name:$item"
+    fi
+done
+
+tag=$(date +%s)
+docker login -u=100010573203  ccr.ccs.tencentyun.com -p=wanpinghui151217
+docker build --no-cache --disable-content-trust=true -t $registry_host/$project_name:$tag .
+
+docker push $registry_host/$project_name:$tag

grpc-demo/server/http_backend.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"../serverImpl"
+	"fmt"
+	"git.wanpinghui.com/WPH/go_common/wph"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+// Istio测试用，HTTP后端服务
+
+func main() {
+	router := gin.New()
+
+	// 添加 GIN 请求处理中间件
+	//router.Use(
+	//	gzip.Gzip(gzip.DefaultCompression),
+	//	mw.Favicon("./static/favicon.ico"), // 请求图标时返回favicon.ico
+	//	mw.AddCrossOriginHeaders(),         // 添加跨域头
+	//	mw.HandleOptionsMethod(),           // Options和Head请求处理
+	//)
+	// 设置404、405响应
+	router.NoMethod(func(c *gin.Context) {
+		resp := wph.E(http.StatusMethodNotAllowed, "Method Not Allowed", nil)
+		resp.HttpStatus = http.StatusMethodNotAllowed
+		panic(resp)
+	})
+	router.NoRoute(func(c *gin.Context) {
+		resp := wph.E(http.StatusMethodNotAllowed, "Endpoint Not Found", nil)
+		resp.HttpStatus = http.StatusMethodNotAllowed
+		panic(resp)
+	})
+
+	// 注册路由
+	router.GET("/backend", backend)
+
+	// 开始监听HTTP请求
+	httpServer := &http.Server{Addr: ":8080", Handler: router}
+	go func(httpServer *http.Server) {
+		println("Start http server")
+		if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			println("http server failed to listen and serve", err.Error())
+		}
+	}(httpServer)
+	serverImpl.SignalHandler(httpServer)
+}
+
+func backend(ctxt *gin.Context) {
+	input, _ := ctxt.GetQuery("in")
+	var ipStr string
+	ip, err := wph.PrivateIPv4()
+	if nil == err && nil != ip {
+		ipStr = ip.String()
+	}
+	ctxt.JSON(http.StatusOK, wph.R(fmt.Sprintf("%s (from %s:8080)", input, ipStr)))
+}

grpc-demo/server/http_endpoint_blue.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"../clientImpl"
+	"../example"
+	"../serverImpl"
+	"fmt"
+	"git.wanpinghui.com/WPH/go_common/wph"
+	"git.wanpinghui.com/WPH/go_common/wph/logger"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	"net/http"
+)
+
+// HTTP边界服务，同时也是后端服务的客户端
+// Istio对比测试用，部署于无Istio的环境
+
+var logB *logger.Logger
+
+func main() {
+	logB = logger.New()
+	logB.Level = logrus.WarnLevel
+
+	router := gin.New()
+
+	// 添加 GIN 请求处理中间件
+	//router.Use(
+	//	gzip.Gzip(gzip.DefaultCompression),
+	//	mw.Favicon("./static/favicon.ico"), // 请求图标时返回favicon.ico
+	//	mw.AddCrossOriginHeaders(),         // 添加跨域头
+	//	mw.HandleOptionsMethod(),           // Options和Head请求处理
+	//)
+	// 设置404、405响应
+	router.NoMethod(func(c *gin.Context) {
+		resp := wph.E(http.StatusMethodNotAllowed, "Method Not Allowed", nil)
+		resp.HttpStatus = http.StatusMethodNotAllowed
+		c.JSON(http.StatusBadRequest, wph.E(400, "Method Not Allowed", resp))
+		return
+	})
+	router.NoRoute(func(c *gin.Context) {
+		resp := wph.E(http.StatusMethodNotAllowed, "Endpoint Not Found", nil)
+		resp.HttpStatus = http.StatusMethodNotAllowed
+		c.JSON(http.StatusBadRequest, wph.E(400, "Endpoint Not Found", resp))
+		return
+	})
+
+	// 注册路由
+	router.GET("/test", testBlue)
+
+	// 开始监听HTTP请求
+	httpServer := &http.Server{Addr: ":8080", Handler: router}
+	go func(httpServer *http.Server) {
+		logB.Info("Start http server")
+		if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			logB.Error("http server failed to listen and serve", err.Error())
+		}
+	}(httpServer)
+	serverImpl.SignalHandler(httpServer)
+}
+
+func testBlue(ctxt *gin.Context) {
+	input, _ := ctxt.GetQuery("in")
+
+	serviceConn, err := grpc.Dial("grpc-example:50051", grpc.WithInsecure())
+	if err != nil {
+		logB.Error("did not connect", err.Error())
+	}
+	defer func() {
+		_ = serviceConn.Close()
+	}()
+	hwc := example.NewFormatDataClient(serviceConn)
+	var resp string
+	resp, err = clientImpl.Call(hwc, fmt.Sprintf("Hello %s", input))
+	if nil != err {
+		logB.Error("call grpc backend failed", err.Error())
+		ctxt.JSON(http.StatusBadRequest, wph.E(400, err.Error(), err.Error()))
+		return
+	}
+	ctxt.JSON(http.StatusOK, wph.R(resp))
+}

grpc-demo/server/http_endpoint_v1.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"../example"
+	"../serverImpl"
+	"context"
+	"fmt"
+	"git.wanpinghui.com/WPH/go_common/wph"
+	"git.wanpinghui.com/WPH/go_common/wph/logger"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/grpclog"
+	"net/http"
+	"time"
+)
+
+// Istio测试用，HTTP边界服务，同时也是后端服务的客户端
+
+var log1 *logger.Logger
+
+func main() {
+	log1 = logger.New()
+	log1.Level = logrus.WarnLevel
+
+	router := gin.New()
+
+	// 添加 GIN 请求处理中间件
+	//router.Use(
+	//	gzip.Gzip(gzip.DefaultCompression),
+	//	mw.Favicon("./static/favicon.ico"), // 请求图标时返回favicon.ico
+	//	mw.AddCrossOriginHeaders(),         // 添加跨域头
+	//	mw.HandleOptionsMethod(),           // Options和Head请求处理
+	//)
+	// 设置404、405响应
+	router.NoMethod(func(c *gin.Context) {
+		resp := wph.E(http.StatusMethodNotAllowed, "Method Not Allowed", nil)
+		resp.HttpStatus = http.StatusMethodNotAllowed
+		c.JSON(http.StatusBadRequest, wph.E(400, "Method Not Allowed", resp))
+		return
+	})
+	router.NoRoute(func(c *gin.Context) {
+		resp := wph.E(http.StatusMethodNotAllowed, "Endpoint Not Found", nil)
+		resp.HttpStatus = http.StatusMethodNotAllowed
+		c.JSON(http.StatusBadRequest, wph.E(400, "Endpoint Not Found", resp))
+		return
+	})
+
+	// 注册路由
+	router.GET("/test", testV1)
+
+	// 开始监听HTTP请求
+	httpServer := &http.Server{Addr: ":8080", Handler: router}
+	go func(httpServer *http.Server) {
+		log1.Info("Start http server")
+		if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log1.Error("http server failed to listen and serve", err.Error())
+		}
+	}(httpServer)
+	serverImpl.SignalHandler(httpServer)
+}
+
+func testV1(ctxt *gin.Context) {
+	input, _ := ctxt.GetQuery("in")
+	conn, err := grpc.Dial("grpc-example:50051", grpc.WithInsecure())
+	if err != nil {
+		log1.Error("Did not connect", err.Error())
+	}
+	defer func() {
+		_ = conn.Close()
+	}()
+	grpclog.SetLoggerV2(log1)
+	hwc := example.NewFormatDataClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	var resp *example.Data
+	resp, err = hwc.DoFormat(ctx, &example.Data{Text: fmt.Sprintf("Hello %s", input)})
+	if err != nil {
+		log1.Error("call grpc backend failed", err.Error())
+		ctxt.JSON(http.StatusBadRequest, wph.E(400, err.Error(), err.Error()))
+		return
+	}
+	ctxt.JSON(http.StatusOK, wph.R(resp.Text))
+}

grpc-demo/server/http_endpoint_v2.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"../serverImpl"
+	"fmt"
+	"git.wanpinghui.com/WPH/go_common/wph"
+	"git.wanpinghui.com/WPH/go_common/wph/logger"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"net/http"
+	"net/url"
+)
+
+// Istio测试用，HTTP边界服务，同时也是后端服务的客户端
+
+var log2 *logger.Logger
+
+func main() {
+	log2 = logger.New()
+	log2.Level = logrus.WarnLevel
+
+	router := gin.New()
+
+	// 添加 GIN 请求处理中间件
+	//router.Use(
+	//	gzip.Gzip(gzip.DefaultCompression),
+	//	mw.Favicon("./static/favicon.ico"), // 请求图标时返回favicon.ico
+	//	mw.AddCrossOriginHeaders(),         // 添加跨域头
+	//	mw.HandleOptionsMethod(),           // Options和Head请求处理
+	//)
+	// 设置404、405响应
+	router.NoMethod(func(c *gin.Context) {
+		resp := wph.E(http.StatusMethodNotAllowed, "Method Not Allowed", nil)
+		resp.HttpStatus = http.StatusMethodNotAllowed
+		c.JSON(http.StatusBadRequest, wph.E(400, "Method Not Allowed", resp))
+		return
+	})
+	router.NoRoute(func(c *gin.Context) {
+		resp := wph.E(http.StatusMethodNotAllowed, "Endpoint Not Found", nil)
+		resp.HttpStatus = http.StatusMethodNotAllowed
+		c.JSON(http.StatusBadRequest, wph.E(400, "Endpoint Not Found", resp))
+		return
+	})
+
+	// 注册路由
+	router.GET("/test", testV2)
+
+	// 开始监听HTTP请求
+	httpServer := &http.Server{Addr: ":8080", Handler: router}
+	go func(httpServer *http.Server) {
+		log2.Info("Start http server")
+		if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log2.Error("http server failed to listen and serve", err.Error())
+		}
+	}(httpServer)
+	serverImpl.SignalHandler(httpServer)
+}
+
+func testV2(ctxt *gin.Context) {
+	input, _ := ctxt.GetQuery("in")
+	uri := fmt.Sprintf("http://http-backend:8080/backend?in=%s", url.QueryEscape(fmt.Sprintf("Hello %s", input)))
+	resp, err := wph.JSONRequest().Call(http.MethodGet, uri, nil)
+	if nil != err {
+		log2.Error("call http backend failed", err.Error())
+		ctxt.JSON(http.StatusBadRequest, wph.E(400, err.Error(), err.Error()))
+		return
+	}
+	ctxt.String(http.StatusOK, string(resp))
+}

grpc-demo/server/load_balancing1.go

@@ -0,0 +1,10 @@
+package main
+
+import "../serverImpl"
+
+// gRPC官方库负载均衡实现示例，服务端，参考 https://grpc-up-and-running.github.io/
+// 也用于Istio测试用，gRPC后端服务
+
+func main() {
+	serverImpl.StartServer(`:50051`)
+}

grpc-demo/server/load_balancing2.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"../serverImpl"
+)
+
+// gRPC官方库负载均衡实现示例，服务端，参考 https://grpc-up-and-running.github.io/
+
+func main() {
+	serverImpl.StartServer(`:50052`)
+}

grpc-demo/server/load_balancing3.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"../serverImpl"
+)
+
+// gRPC官方库负载均衡实现示例，服务端，参考 https://grpc-up-and-running.github.io/
+
+func main() {
+	serverImpl.StartServer(`:50053`)
+}

grpc-demo/serverImpl/example_server.go

@@ -0,0 +1,72 @@
+package serverImpl
+
+import (
+	"../example"
+	"context"
+	"fmt"
+	"git.wanpinghui.com/WPH/go_common/wph"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/reflection"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+)
+
+// 服务端方法实现，没有什么不一样，正常实现即可。此示例只是为了看得更清楚，实例化的时候存了一下端口号，返回值中再把它带上
+
+type ExampleServer struct {
+	addr string
+}
+
+func (s *ExampleServer) DoFormat(ctx context.Context, req *example.Data) (out *example.Data, err error) {
+	var ipStr string
+	ip, err := wph.PrivateIPv4()
+	if nil == err && nil != ip {
+		ipStr = ip.String()
+	}
+	return &example.Data{Text: fmt.Sprintf("%s (from %s%s)", req.Text, ipStr, s.addr)}, nil
+}
+
+func StartServer(addr string) {
+	lis, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("failed to listen: %v", err)
+	}
+	s := grpc.NewServer()
+	example.RegisterFormatDataServer(s, &ExampleServer{addr: addr})
+	// 注册到reflection，才能用grpcurl客户端测试
+	reflection.Register(s)
+	log.Printf("serving on %s\n", addr)
+	if err := s.Serve(lis); err != nil {
+		log.Fatalf("failed to serve: %v", err)
+	}
+}
+
+func SignalHandler(httpServer *http.Server) {
+	var (
+		c chan os.Signal
+		s os.Signal
+	)
+	c = make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	for {
+		s = <-c
+		switch s {
+		case syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT:
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			println("Shutdown http server")
+			if err := httpServer.Shutdown(ctx); err != nil {
+				println("http server failed to shutdown", err.Error())
+			}
+			cancel()
+			println("Server exited")
+			return
+		default:
+			return
+		}
+	}
+}

grpc-demo/tls/deploy_ex_tls.sh

@@ -0,0 +1 @@
+kubectl create secret tls ex-wph-secret --key ex_tls.key --cert ex_tls.crt

grpc-demo/tls/deploy_gw_tls.sh

@@ -0,0 +1 @@
+kubectl create secret tls ex-wph-secret --key gw_tls.key --cert gw_tls.crt

grpc-demo/tls/ex_tls.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDVzCCAj+gAwIBAgIJALC+QRfKyje4MA0GCSqGSIb3DQEBCwUAMEIxHzAdBgNV
+BAMMFmV4YW1wbGUud2FucGluZ2h1aS5jb20xHzAdBgNVBAoMFmV4YW1wbGUud2Fu
+cGluZ2h1aS5jb20wHhcNMjAwMjI5MDI0ODU2WhcNMjEwMjI4MDI0ODU2WjBCMR8w
+HQYDVQQDDBZleGFtcGxlLndhbnBpbmdodWkuY29tMR8wHQYDVQQKDBZleGFtcGxl
+LndhbnBpbmdodWkuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+zl0mOMncZ7+B5OFeFEVvuHLPdRsA4pAe51uihlNRTgZVfeLvRbeVL2DGwIMBMbsX
+NkIb177GoB0n7WQJE0PjiVr6HUFBWHWtjf2FXY/13Z7VvBVj/4JTK4x2GP6uUcIt
+px3xilKrhvv9vSoSlLVQju5o9d8vpinHYqSxaaswA4r3jvPEraXcOwVeJp+zgdyx
+TVW4/1Dv1VhM68CFUNEJJF5Z7ca3tzF7jdAmwpqFn1ZAf4JMy1QClhY3sX7xUL5Y
+NSD8eTCpqTavHkEh7SM0F+Gn4oGjPFcpi6tTY0CWuPJSQsEvTuqUa4aZHId6IHdO
+wYT8HQtM0GZuKDKcPPur+wIDAQABo1AwTjAdBgNVHQ4EFgQUkI5qmNuDaVJlWyFR
+8NRcul/78CIwHwYDVR0jBBgwFoAUkI5qmNuDaVJlWyFR8NRcul/78CIwDAYDVR0T
+BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEACIX6QNUcCb//Wl1u+KgAQ7xWkCWE
+xxG5jTtjOmOph6y2OnDIjZMYnzdAp6DUfbOhopyPLJsAe+Ua3Tfct5jrspxKHqv2
+jl+OvrE5IuQZ3dLhPu4+DlIxJtlCdvQPx0lU7cpvKoc+yaYrDiis4PxE/zWgZWP8
+4f8de3oKD4bd8WLiHOJdU5tOp0GYZyqCL60EOdcz1PtmkmMOInghtcvBP/hEiWf6
+Wg/jRA/6KyHvab9+UZVXXiqr5lGOnJP/Ksy9a1BgFgZjSqcdxSqmpF7kqO5sQPda
++G9sc5J+gvurM0hBa7lfNND7NKBSAkbhhDAFALsXVbb4sEBGmH5O6rNwXQ==
+-----END CERTIFICATE-----

grpc-demo/tls/ex_tls.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOXSY4ydxnv4Hk
+4V4URW+4cs91GwDikB7nW6KGU1FOBlV94u9Ft5UvYMbAgwExuxc2QhvXvsagHSft
+ZAkTQ+OJWvodQUFYda2N/YVdj/XdntW8FWP/glMrjHYY/q5Rwi2nHfGKUquG+/29
+KhKUtVCO7mj13y+mKcdipLFpqzADiveO88Stpdw7BV4mn7OB3LFNVbj/UO/VWEzr
+wIVQ0QkkXlntxre3MXuN0CbCmoWfVkB/gkzLVAKWFjexfvFQvlg1IPx5MKmpNq8e
+QSHtIzQX4afigaM8VymLq1NjQJa48lJCwS9O6pRrhpkch3ogd07BhPwdC0zQZm4o
+Mpw8+6v7AgMBAAECggEAX818xhZonT2Z9wDtZSnLMO0UBMCxi3zu6+dtr0CU2U01
+eUjWXh5ZQvKBsrPe5RB/OTA9rs4hdgmzFJFBaVE2G/lBbj1yMbgIB4gv1vzTXryz
+ayrfDqqZI/mEhhwOH2SvPws2rEmu2e64oI4crC70pDZoQeUIQPLaXqAowO65Tprq
+zgX3qKFDHi9HA5+ql5BB7x0AmqrEx0+C6FSJTPkF5DjQe3CPS5Bd65RqkpKBQb0Z
+/qwSuThK/L5kDiMX4jH0XYOfEccg5f0H6qXLNTkBGXopZvqbHFz7/g0xYi9wFW8L
+n+h7HA4hT+qDOniGMHltOC7ey2oXTKRu2ezGvBjeAQKBgQDsYoIdTLmfjGURdM2r
+L+2F1xlTnk7JnlZIbN+j6mF+AbnVPS1EUSvO704isMEzy/pX/OJSuGCcMsU3SenO
+SUX/D5zFU+0N+iCwQ+QfkfvCnytb6bRjdaW92ORcI9WkMzVWCtDjQb0QWM8/BgLY
+eYfbhQidWVvSqJiXr9COBvXK+wKBgQDffOkE9sd4KpJJ04Notd7ENXs1u4QeX0rU
+5fB95hNM7XWahbqjarynAeSNcobSC+pyJopBs3m4r7A1jWCqhyfr1ah1Zfvjh/6U
+bOGMwo/eS76pU1RP4YzXanB5KKRnu3yTFZU2xILmQdqJ4WBi389JFbInS8j3Su9w
+jiUmAdvTAQKBgQC46/NIAo6wHv+t4UdH5U2WR4h7nruaBHERZYKq05+DGMyM4bf4
+2ts/nnvsrYPnBmqIzL8s4SpK53ja4Mq0YyQL/eWxhQ9MwOEzo1jhza07OpOL6s8A
+g7euRe+XmTz3oEvYTMvPDM1f7WnrrsMpICHE/FSZ657mtJfzCkpH4XAFYwKBgGJL
+1U2aoScoSklwX3b5ry5Le5KplDWrogU70Wf+fvSx6Kz7LpjbxHgMLBNOkR/nn8lv
+TBYTs3DYzT7wnNjEM3nV8o8YOQ7dbMqpjRbVKGQUmpptDUNJKCJDLY8MoisB2ovy
+zTtWeWB/DHjiiPtUBx7VDROdkJm7+s5dptctsBsBAoGBANA9zN68kU6VVWgdBFOJ
+xhAAqgcstiACBkLgk3hO1iuVBrJXptsERy2ookycgK+5swEckXCkaF4yIMbjRD/R
+d/BUxqu1wfggV5B8vg530xkXJx1b+Fi220x98jrcGmyF+kbPfUYwh+NgqC5HcZq4
+4aeVlrqezCHTRVxI2lzPi1MW
+-----END PRIVATE KEY-----

grpc-demo/tls/gw_tls.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDQzCCAiugAwIBAgIJAJpNMKnNLQEGMA0GCSqGSIb3DQEBCwUAMDgxGjAYBgNV
+BAMMEWd3LndhbnBpbmdodWkuY29tMRowGAYDVQQKDBFndy53YW5waW5naHVpLmNv
+bTAeFw0yMDAyMjkwMjI3MTJaFw0yMTAyMjgwMjI3MTJaMDgxGjAYBgNVBAMMEWd3
+LndhbnBpbmdodWkuY29tMRowGAYDVQQKDBFndy53YW5waW5naHVpLmNvbTCCASIw
+DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANw5BYoUghln3FTbtxHnoBYfeo9U
+3NI1Fq/xsLavfsVz04k6b5iyDF5lvZ8MwJqbgzqkRMd0KWWYpOyR4vbVqwe6naJN
+8TBMM7y9NmGmBRzttS8X/E9XDxPYAAo2wxioktKQqf4pabip2E7FlcnqDyoJMPVL
+9c8iQfS/iMJpX/5KqtNpLWzbvkxKMDPkW7zyr8Ikl9gymWrUzmyp6mh38RkVbmUp
+V/FjQkH0BuExuNQ7i9u9pxcqqZjZEZ9+iTg9RJ+GOw/4GUrby/wfG7sGabG13Hvt
+b8ErjB8n6hSkkwWWm3wR9CRUGyJKEF+OOUk9pqWyxcR6vJXVX1D7Vrg3EHsCAwEA
+AaNQME4wHQYDVR0OBBYEFAAkXQYCO0A6jhNVs54STJRbPFJYMB8GA1UdIwQYMBaA
+FAAkXQYCO0A6jhNVs54STJRbPFJYMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL
+BQADggEBAGSD1XNv0Ax2K5OHKA99po9Ka9isu9qRvdZV/KVJWbGSaCCjs4R4AQxy
+v1XzSlUilRKQcn65KXZxglNfV9VOe8NA/tX1RDV2KD/0MgRdXH2YmfCojj+Q49g+
+AhUIkQF97J5FH41KhiDZiIow6ByRJ9IQo+fq/N08qzJlrCBsbe7yXcAFToiS6wSs
+JxtvWs+uGnGegnPfbneg0pHnTJoVpmpJYB8ggJ+mF5XnxQyoPMwiceCzusR4l3I6
+74lDY39ytc6t1VkQYGPLPEMTYMVPWRzkYn+zvPbIL6SxJmyOFfV2t7IZ3upONQiX
+lBt460uxRODRnG5KxBmsJkyXXEYobPY=
+-----END CERTIFICATE-----

grpc-demo/tls/gw_tls.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDcOQWKFIIZZ9xU
+27cR56AWH3qPVNzSNRav8bC2r37Fc9OJOm+YsgxeZb2fDMCam4M6pETHdCllmKTs
+keL21asHup2iTfEwTDO8vTZhpgUc7bUvF/xPVw8T2AAKNsMYqJLSkKn+KWm4qdhO
+xZXJ6g8qCTD1S/XPIkH0v4jCaV/+SqrTaS1s275MSjAz5Fu88q/CJJfYMplq1M5s
+qepod/EZFW5lKVfxY0JB9AbhMbjUO4vbvacXKqmY2RGffok4PUSfhjsP+BlK28v8
+Hxu7Bmmxtdx77W/BK4wfJ+oUpJMFlpt8EfQkVBsiShBfjjlJPaalssXEeryV1V9Q
++1a4NxB7AgMBAAECggEAALSmEeUy7YXprNPvaD1HxAphyCfLdVj6M6IgoNU7IMLY
+QK5RQToA1CSbUPEnhNdOuldgaQs0PsEYdjGZepUAsUulyalaRn9YAxjQHcv8ZDVW
+yUnYYyCySav9hArc4S1ZGxefk9LqJc9VhBsZ+cWoOshAweh2Zy2C7JfYhlb0Gcp9
+7zlTBnG5IAMgW+he0ywGEbYBqEvrnv6o6HPsJF4QrU0QaocJ+Gx9WLU4ZW/o+/20
+stOJi+dQxRqe4Pnm1z4+X7ghgn06uFmnwBXiARSQj27PUglcnq9AfjAg4voNyxNk
+7YdODjuwarQgd0L+2RHV0T65ljW1EkRu8Px1+CyRIQKBgQD90ciDCJAX9fZWhCTx
+mUy3a5mn3ubdkSN/ApgWIf/oQb5czlajKOd9F2YAwSAwdJIDlPyLKS7qbLAM5f/f
+SpyCClNIaFfHMpc8Ht+ufyKFJbbVmT1WF4Cy3/ZUAkeoAa9qrvKxmbj4u+qfoP4B
+2Rfsbt1vLLQ1ko9hHFP2+a6Q4wKBgQDeHVmoPF+t+V+WmOpoQk8fq8io5k5FtMdf
+H+oYMEX/3Igs5/HLYYXNdXHb4E34pYaO8SItrYrF+bQCxyv93a1q20nUpEmDXoPU
+cmsz+85jVLAW65vLllGkSqBttwHFERWCklL3hgp+kZiHbsNIbOBhIDq5ak/v4b5s
+oKilrNANiQKBgGCtANR39HpL4vH9Dx0+qKvMPhBkpEaMO5rq4Aa+GHzcXad3qCQp
+NlK9EMsExD/whpT6sF5I52e+0id3i0f/YTFewW344EYtDNGHaOdL4VgZj1Q4M9aQ
+ySJYj7hbzfM+ARR/Yk0nIf8LhEgcx+LoT2Zsq8bmnhUxp47zfT+6xjTFAoGAXq0G
+DXcGlaKpQwkVSbGSBn7zvkLE9GN1ZC+3We4huEkpZnLLlxSPsx8vlDiNI8Jyk7Vn
+Cy9g6rhgKZr6PnDiPZ0RYGlSyINeEA1hcrwPojl0AaEWPHZrgGsq7PlH8NcyYwC6
+CrYA/ShUas4kwVdfYScawPP29No1kBn/KPd7i3kCgYEAib22pkvdhQXkc1bOdd1E
++WD7Wtgm1dYgFt9P4E6k9YieBAyaPbfEtnTHH85gycyKTRiqBfUy+7orQDpQAuIf
+ECDq3sBGl4wACea0Xut4wYs//oUVRhVd0P7pRVyAGZgkLzcWmZw7zMx12Jk99CsS
+7/Rul6+w7QUfkyAteHlhEGc=
+-----END PRIVATE KEY-----

html2image.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"git.aionnect.com/aionnect/go-common/utils"
+	"github.com/cbroglie/mustache"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+)
+
+const tplStr = `<html>
+<body style="border: 0; padding: 0; margin: 0">
+<div style="width: 768px; height: 1024px; text-align: center; background-color: bisque">
+    <h3 style="padding-top:20px">{{title}}</h3>
+    <p>{{description}}</p>
+    <img src="{{url}}" width="430" height="430" />
+</div>
+</body>
+</html>`
+
+const (
+	W = 768
+	H = 1024
+)
+
+func main() {
+	var buf bytes.Buffer
+	tpl, _ := mustache.ParseString(tplStr)
+	_ = tpl.FRender(&buf, map[string]string{
+		"title":       "Hello world",
+		"description": "A test for convert html to jpg",
+		"url":         "https://mp.weixin.qq.com/cgi-bin/showqrcode?ticket=gQFe8TwAAAAAAAAAAS5odHRwOi8vd2VpeGluLnFxLmNvbS9xLzAyUFlBM1FKcDlldWgxMDAwMDAwNzAAAgTryl1eAwQAAAAA",
+	})
+	tempId := utils.NextId()
+	htmlFileName := fmt.Sprintf("temp_%d.html", tempId)
+	imgFileName := fmt.Sprintf("temp_%d.jpg", tempId)
+	_ = ioutil.WriteFile(htmlFileName, buf.Bytes(), os.ModeAppend)
+	cmd := exec.Command(`wkhtmltoimage`, `--height`, strconv.Itoa(H), `--width`, strconv.Itoa(W), htmlFileName, imgFileName)
+	_ = cmd.Run()
+	// output, err := cmd.CombinedOutput()
+}

httpbin.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"github.com/gin-gonic/gin"
+	"net/http"
+	"runtime"
+)
+
+func main() {
+	runtime.GOMAXPROCS(-1)
+
+	router := gin.New()
+	router.POST("", reply)
+	router.GET("", reply)
+
+	_ = router.Run(":9090")
+}
+
+func reply(ctxt *gin.Context) {
+	println("url", ctxt.Request.URL.String())
+	sign := ctxt.GetHeader("X-Gogs-Signature")
+	println("X-Gogs-Signature", sign)
+	buf := make([]byte, 10240)
+	n, _ := ctxt.Request.Body.Read(buf)
+	println(string(buf[0:n]))
+	ctxt.JSON(http.StatusOK, "OK")
+}

jwt.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+	"github.com/dgrijalva/jwt-go"
+	"strconv"
+)
+
+type TokenClaims struct {
+	ID float64 `json:"id"`
+	jwt.StandardClaims
+}
+
+func main() {
+	c, err := parseToken("eyJhbGciOiJIUzI1NiIsImlhdCI6MTUzNzg0Njc4MywiZXhwIjoxNTM5MDU2MzgzfQ.eyJpZCI6MzI0NTEzNH0.RqRoad8H5oAGY6L3qMLcxCUYE5Fl7-MXtpSyfSU4aqU")
+	if nil != err {
+		println(err)
+	}
+	fmt.Printf("%v\n", c)
+	fmt.Printf("%v", strconv.FormatFloat(c.ID, 'f', 0, 64))
+}
+
+func parseToken(tokenString string) (*TokenClaims, error) {
+	token, err := jwt.ParseWithClaims(tokenString, &TokenClaims{}, func(token *jwt.Token) (interface{}, error) {
+		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
+			return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
+		}
+
+		return []byte("hard to guess string"), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if claims, ok := token.Claims.(*TokenClaims); ok && token.Valid {
+		return claims, nil
+	}
+	return nil, nil
+}

kafka/kpub.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"git.wanpinghui.com/WPH/go_common/wph/logger"
+	"github.com/Shopify/sarama"
+	"io/ioutil"
+	"os"
+)
+
+var (
+	pubLogger = logger.New()
+)
+
+func main() {
+	//brokers := []string{"127.0.0.1:9092"}
+	brokers := []string{"47.94.239.212:9093", "47.94.91.146:9093", "47.94.154.247:9093"}
+	sarama.Logger = pubLogger
+
+	config := sarama.NewConfig()
+	config.Producer.RequiredAcks = sarama.WaitForAll
+	config.Producer.Partitioner = sarama.NewRandomPartitioner
+	config.Producer.Return.Successes = true
+
+	config.Net.SASL.Enable = true
+	config.Net.SASL.User = "wangyangming"
+	config.Net.SASL.Password = "WangMy1920"
+	config.Net.SASL.Handshake = true
+	//config.Version=sarama.V0_10_2_1
+
+	config.Net.TLS.Enable = true
+	certPath := "./alicloud_kafka/ca-cert"
+	certBytes, err := ioutil.ReadFile(certPath)
+	clientCertPool := x509.NewCertPool()
+	ok := clientCertPool.AppendCertsFromPEM(certBytes)
+	if !ok {
+		panic("kafka producer failed to parse root certificate")
+	}
+	config.Net.TLS.Config = &tls.Config{
+		//Certificates:       []tls.Certificate{},
+		RootCAs:            clientCertPool,
+		InsecureSkipVerify: true,
+	}
+
+	plusMsg := &sarama.ProducerMessage{}
+	plusMsg.Topic = "chatRoom"
+	plusMsg.Partition = int32(-1)
+	plusMsg.Key = sarama.StringEncoder("key")
+	plusMsg.Value = sarama.ByteEncoder("你好, 世界++!")
+
+	//reduceMsg := &sarama.ProducerMessage{}
+	//reduceMsg.Topic = "ka-alloc-reduce-job"
+	//reduceMsg.Partition = int32(-1)
+	//reduceMsg.Key = sarama.StringEncoder("key")
+	//reduceMsg.Value = sarama.ByteEncoder("你好, 世界--!")
+
+	producer, err := sarama.NewSyncProducer(brokers, config)
+
+	if err != nil {
+		pubLogger.Errorf("Failed to produce message: %s", err.Error())
+		os.Exit(500)
+	}
+	defer func() {
+		_ = producer.Close()
+	}()
+
+	for i := 0; i < 10; i++ {
+		partition, offset, err := producer.SendMessage(plusMsg)
+		if err != nil {
+			pubLogger.Error("Failed to produce message: ", err)
+		}
+		pubLogger.Infof("partition=%d, offset=%d\n", partition, offset)
+
+		//partition, offset, err = producer.SendMessage(reduceMsg)
+		//if err != nil {
+		//	pubLogger.Error("Failed to produce message: ", err)
+		//}
+		//pubLogger.Infof("partition=%d, offset=%d\n", partition, offset)
+	}
+}

kafka/ksub.go

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"git.wanpinghui.com/WPH/go_common/wph/logger"
+	"github.com/Shopify/sarama"
+	"github.com/bsm/sarama-cluster"
+	"io/ioutil"
+	"sync"
+)
+
+var (
+	wg        sync.WaitGroup
+	subLogger = logger.New()
+)
+
+func main() {
+	//brokers := []string{"127.0.0.1:9092"}
+	brokers := []string{"47.94.239.212:9093", "47.94.91.146:9093", "47.94.154.247:9093"}
+	sarama.Logger = subLogger
+
+	config := cluster.NewConfig()
+	config.Consumer.Offsets.Initial = sarama.OffsetOldest
+	config.Consumer.Return.Errors = true
+	config.Group.Return.Notifications = false
+
+	config.Net.SASL.Enable = true
+	config.Net.SASL.User = "wangyangming"
+	config.Net.SASL.Password = "WangMy1920"
+	config.Net.SASL.Handshake = true
+	config.Version = sarama.V0_10_2_1
+
+	config.Net.TLS.Enable = true
+	certPath := "./alicloud_kafka/ca-cert"
+	certBytes, err := ioutil.ReadFile(certPath)
+	clientCertPool := x509.NewCertPool()
+	ok := clientCertPool.AppendCertsFromPEM(certBytes)
+	if !ok {
+		panic("kafka consumer failed to parse root certificate")
+	}
+	config.Net.TLS.Config = &tls.Config{
+		//Certificates:       []tls.Certificate{},
+		RootCAs:            clientCertPool,
+		InsecureSkipVerify: true,
+	}
+
+	// 初始化增加定时任务消息消费者，相同group名单播消费消息
+	plusTopics := []string{"chatRoom"}
+	plusConsumer, err := cluster.NewConsumer(brokers, "chat-room-group", plusTopics, config)
+	if err != nil {
+		panic(err)
+	}
+	defer func() {
+		_ = plusConsumer.Close()
+	}()
+	go func() {
+		for err := range plusConsumer.Errors() {
+			subLogger.Error("Error: %s\n", err.Error())
+		}
+	}()
+
+	//// 初始化减少定时任务消息消费者，不同group名广播消费消息
+	//reduceTopics := []string{"ka-alloc-reduce-job"}
+	//// wph.GetPrivateIPv4Id() 这个函数会根据当前机器内网ip的末尾两段运算出一个id，即测试和生产环境不同Pod的ip不同这个id也会不同
+	//// 正式编码请用 wph.GetPrivateIPv4Id() 代替 wph.NextId()
+	//machineId := wph.NextId()
+	//reduceConsumer, err := cluster.NewConsumer(brokers, fmt.Sprintf("ka-alloc-reduce-job-group-%d", machineId), reduceTopics, config)
+	//if err != nil {
+	//	panic(err)
+	//}
+	//defer func() {
+	//	_ = reduceConsumer.Close()
+	//}()
+	//go func() {
+	//	for err := range reduceConsumer.Errors() {
+	//		subLogger.Error("Error: %s\n", err.Error())
+	//	}
+	//}()
+
+	// 消费消息
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for msg := range plusConsumer.Messages() {
+			subLogger.Infof("%s/%d/%d\t%s\t%s\n", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
+			plusConsumer.MarkOffset(msg, "") // mark message as processed
+		}
+	}()
+	//wg.Add(1)
+	//go func() {
+	//	defer wg.Done()
+	//	for msg := range reduceConsumer.Messages() {
+	//		subLogger.Infof( "%s/%d/%d\t%s\t%s\n", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
+	//		reduceConsumer.MarkOffset(msg, "") // mark message as processed
+	//	}
+	//}()
+
+	//for {
+	//	select {
+	//	case msg, ok := <-consumer.Messages():
+	//		if ok {
+	//			fmt.Fprintf(os.Stdout, "%s/%d/%d\t%s\t%s\n", msg.Topic, msg.Partition, msg.Offset, msg.Key, msg.Value)
+	//			consumer.MarkOffset(msg, "")	// mark message as processed
+	//		}
+	//	case <-signals:
+	//		return
+	//	}
+	//}
+
+	//consumer, err := sarama.NewConsumer([]string {"127.0.0.1:9092"}, nil)
+	//if err != nil {
+	//	subLogger.Println("Failed to start consumer: %s", err)
+	//}
+	//defer consumer.Close()
+	//
+	//partitionList, err := consumer.Partitions("shuaige")
+	//if err != nil {
+	//	subLogger.Println("Failed to get the list of partitions: ", err)
+	//}
+	//
+	//for partition := range partitionList {
+	//	pc, err := consumer.ConsumePartition("shuaige", int32(partition), sarama.OffsetNewest)
+	//	if err != nil {
+	//		subLogger.Printf("Failed to start consumer for partition %d: %s\n", partition, err)
+	//	}
+	//	defer pc.AsyncClose()
+	//
+	//	wg.Add(1)
+	//
+	//	go func(sarama.PartitionConsumer) {
+	//		defer wg.Done()
+	//		for msg := range pc.Messages() {
+	//			fmt.Printf("Partition:%d, Offset:%d, Key:%s, Value:%s", msg.Partition, msg.Offset, string(msg.Key), string(msg.Value))
+	//			fmt.Println()
+	//		}
+	//	}(pc)
+	//}
+
+	wg.Wait()
+
+	subLogger.Println("Done consuming topic")
+}

md5.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+)
+
+func main() {
+	str := "wph_esp_177F64a9CF5c_4187724128Yeafw93#d392207127"
+	md5Ctx := md5.New()
+	md5Ctx.Write([]byte(str))
+	encryptPwd := md5Ctx.Sum(nil)
+	pwdmd5 := hex.EncodeToString(encryptPwd[:])
+	println(pwdmd5)
+}

mqtt.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	//import the Paho Go mqtt library
+	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"os"
+	"time"
+)
+
+//define a function for the default message handler
+var f mqtt.MessageHandler = func(client mqtt.Client, msg mqtt.Message) {
+	fmt.Printf("TOPIC: %s\n", msg.Topic())
+	fmt.Printf("MSG: %s\n", msg.Payload())
+}
+
+var lck chan bool
+
+func main() {
+	lck = make(chan bool)
+
+	println(1)
+	opts := mqtt.NewClientOptions().AddBroker("tcp://192.168.1.93:1883")
+	opts.SetClientID("go-simple")
+	opts.SetDefaultPublishHandler(f)
+
+	println(2)
+	c := mqtt.NewClient(opts)
+	if token := c.Connect(); token.Wait() && token.Error() != nil {
+		panic(token.Error())
+	}
+
+	println(3)
+	msgReceived := func(client mqtt.Client, message mqtt.Message) {
+		fmt.Printf("Received message on topic: %s Message: %s\n", message.Topic(), message.Payload())
+	}
+
+	println(4)
+	if token := c.Subscribe("/wph_esp/pub", 0, msgReceived); token.Wait() && token.Error() != nil {
+		fmt.Println(token.Error())
+		os.Exit(1)
+	}
+
+	println(5)
+	for i := 0; i < 5; i++ {
+		text := fmt.Sprintf("this is msg #%d!", i)
+		token := c.Publish("/wph_esp/sub", 0, false, text)
+		token.Wait()
+	}
+
+	<-lck
+
+	time.Sleep(3 * time.Second)
+
+	println(6)
+	if token := c.Unsubscribe("/wph_esp/pub"); token.Wait() && token.Error() != nil {
+		fmt.Println(token.Error())
+		os.Exit(1)
+	}
+
+	c.Disconnect(250)
+}

rabbitmqtest.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/streadway/amqp"
+	"log"
+	"time"
+)
+
+var conn *amqp.Connection
+var channel *amqp.Channel
+var count = 0
+
+const (
+	queueName = "test.topic.hello"
+	exchange  = ""
+	vHost     = "amqp://wph:wph@10.10.10.27:5672/"
+	//vHost     = "amqp://wph:wph@127.0.0.1:5672/"
+)
+
+func main() {
+	var err error
+	conn, err = amqp.Dial(vHost)
+	failOnError(err, "failed to connect tp rabbitmq")
+	defer conn.Close()
+
+	channel, err = conn.Channel()
+	failOnError(err, "failed to open a channel")
+	defer channel.Close()
+
+	_, err = channel.QueueDeclare(
+		queueName, // name
+		false,     // durable
+		false,     // delete when unused
+		false,     // exclusive
+		false,     // no-wait
+		nil,       // arguments
+	)
+	failOnError(err, "Failed to declare a queue")
+	go func() {
+		for {
+			for i := 0; i < 200; i++ {
+				push()
+			}
+			time.Sleep(1 * time.Second)
+		}
+	}()
+	receive()
+	fmt.Println("end")
+}
+
+func failOnError(err error, msg string) {
+	if err != nil {
+		log.Fatalf("%s:%s", msg, err)
+		panic(fmt.Sprintf("%s:%s", msg, err))
+	}
+}
+
+func push() {
+	msgContent := "hello world!"
+	channel.Publish(exchange, queueName, false, false, amqp.Publishing{
+		ContentType: "text/plain",
+		Body:        []byte(msgContent),
+	})
+}
+
+func receive() {
+	messages, err := channel.Consume(queueName, "", true, false, false, false, nil)
+	failOnError(err, "")
+
+	forever := make(chan bool)
+
+	go func() {
+		for d := range messages {
+			s := BytesToString(&(d.Body))
+			count++
+			fmt.Printf("receve msg is: %s -- %d\n", *s, count)
+		}
+	}()
+
+	fmt.Printf(" [*] Waiting for messages. To exit press CTRL+C\n")
+	<-forever
+}
+
+func BytesToString(b *[]byte) *string {
+	s := bytes.NewBuffer(*b)
+	r := s.String()
+	return &r
+}

redis.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"fmt"
+	rd "git.aionnect.com/aionnect/go-common/utils/redis"
+	"github.com/gomodule/redigo/redis"
+	"github.com/spf13/viper"
+	"time"
+)
+
+func main() {
+	// 配置
+	viper.SetDefault("redis.host", "127.0.0.1:6379")
+	//viper.SetDefault("redis.host","192.168.101.161:6379")
+	//viper.SetDefault("redis.nodes", []string{"192.168.101.161:6379", "192.168.101.125:6379", "192.168.101.170:6379"})
+
+	// 初始化
+	q := rd.NewRedisQueue("myList", "backList", 3000, 5*time.Second)
+	q.Clean()
+	q.Recycle()
+
+	// 消费
+	for i := 0; i < 4; i++ {
+		go func(idx int, q *rd.Queue) {
+			q.Pop(func(reply interface{}) {
+				content, _ := redis.String(reply, nil)
+				fmt.Printf("Receiver %d get:%+v\n", idx, content)
+			})
+		}(i, q)
+	}
+	// 生产
+	go func(q *rd.Queue) {
+		for i := 0; i < 20; i++ {
+			q.Push(fmt.Sprintf("Message %d", i+1))
+			time.Sleep(1 * time.Second)
+		}
+	}(q)
+
+	quit := make(chan bool)
+	<-quit
+}

regex.go

@@ -0,0 +1,12 @@
+package main
+
+import "regexp"
+
+func main() {
+	println(1 << 20)
+	str := "dgvdagdbd\tkkageg:feg\""
+	println(str)
+	regex2, _ := regexp.Compile("[\"\\/]")
+	str2 := regex2.ReplaceAllString(str, "\\$0")
+	println(str2)
+}

rpcx/client/main.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"../service"
+	"context"
+	client2 "github.com/smallnest/rpcx/client"
+	"log"
+)
+
+const ADDRESS string = "tcp@localhost:10001"
+
+func main() {
+	d := client2.NewPeer2PeerDiscovery(ADDRESS, "")
+	c := client2.NewXClient("Format", client2.Failtry, client2.RandomSelect, d, client2.DefaultOption)
+	defer func(c client2.XClient) {
+		_ = c.Close()
+	}(c)
+
+	replay := &service.Replay{}
+	err := c.Call(context.Background(), "DoFormat", &service.Args{Text: "Hello world!"}, replay)
+	if err != nil {
+		log.Fatalf("failed to call: %v", err)
+	}
+	log.Println(replay.Text)
+}

rpcx/server/main.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"../service"
+	"context"
+	"github.com/smallnest/rpcx/server"
+	"log"
+	"strings"
+)
+
+const ADDRESS string = "localhost:10001"
+
+type FormatData struct{}
+
+func (fd *FormatData) DoFormat(ctx context.Context, args *service.Args, reply *service.Replay) error {
+	str := args.Text
+	reply.Text = strings.ToUpper(str)
+	return nil
+}
+
+func main() {
+	s := server.NewServer()
+	err := s.RegisterName("Format", new(FormatData), "")
+	if err != nil {
+		log.Fatalln("faile register service: Format")
+	}
+	if err = s.Serve("tcp", ADDRESS); err != nil {
+		log.Fatalln("faile serve at: " + ADDRESS)
+	}
+}

rpcx/service/sample.go

@@ -0,0 +1,9 @@
+package service
+
+type Args struct {
+	Text string
+}
+
+type Replay struct {
+	Text string
+}

rpcx/xrpc_client.go

@@ -0,0 +1,61 @@
+package rpcx
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"git.wanpinghui.com/WPH/go_common/wph"
+	"git.wanpinghui.com/WPH/go_common/wph/date"
+	"github.com/smallnest/rpcx/client"
+)
+
+type DemoRQ struct {
+	ID wph.Long `json:"id"`
+}
+
+type DemoRES struct {
+	ID        wph.Long  `json:"id"`
+	StartTime date.Time `json:"startTime"`
+}
+
+func main() {
+	ctx := context.Background()
+
+	rpcRQ := &DemoRQ{ID: wph.NextId()}
+	rpcRES := &DemoRES{}
+
+	c := client.NewClient(client.DefaultOption)
+	err := c.Connect("tcp", "localhost:10000")
+	//err := c.Connect("tcp", "api.88ba.com:10007")
+	if err != nil {
+		fmt.Printf("failed to connect: %v", err)
+		return
+	}
+	defer c.Close()
+
+	//token := "rpc bearer cdEger5i9943rFqewg"
+	//metadata := ctx.Value(share.ReqMetaDataKey)
+	//if metadata == nil {
+	//	metadata = map[string]string{}
+	//	ctx = context.WithValue(ctx, share.ReqMetaDataKey, metadata)
+	//}
+	//m := metadata.(map[string]string)
+	//m[share.AuthKey] = token
+
+	println(1)
+	err = c.Call(ctx, "DemoService", "TestFn", rpcRQ, rpcRES)
+	println(2)
+
+	//d := client.NewPeer2PeerDiscovery("tcp@wph-micro-auth:10007", "")
+	//c := client.NewXClient("GovAuth", client.Failtry, client.RandomSelect, d, client.DefaultOption)
+	//c.Auth(token)
+	//err := c.Call(context.Background(), "CheckUser", rpcRQ, rpcRES)
+
+	if err != nil {
+		fmt.Printf("rpc client call err=%s\n", err.Error())
+		err = nil
+	} else {
+		b, _ := json.Marshal(rpcRES)
+		println("result is ", string(b))
+	}
+}

snowflake.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"github.com/sony/sonyflake"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func main() {
+	startTime := time.Date(2018, 9, 15, 7, 59, 59, 0, time.UTC)
+	keyGen := sonyflake.NewSonyflake(sonyflake.Settings{
+		StartTime: startTime,
+		MachineID: func() (uint16, error) {
+			return 1, nil
+		},
+	})
+
+	n := 3
+	for i := 0; i < 500; i++ {
+		newId, _ := keyGen.NextID()
+
+		idStr := strconv.FormatUint(newId, 10)
+		sLen := len(idStr)
+		subLen := sLen / n
+		if sLen%n != 0 {
+			subLen++
+		}
+
+		var partArr []string
+		var codeArr []string
+		for j := 0; j < n; j++ {
+			start := subLen * j
+			end := subLen * (j + 1)
+			if end >= sLen {
+				end = sLen
+			}
+			part := idStr[start:end]
+			partArr = append(partArr, part)
+
+			partNum, _ := strconv.ParseInt(part, 10, 32)
+			code := decimalToAny(int(partNum), 35)
+			codeArr = append(codeArr, code)
+		}
+
+		println(newId, " : ", strings.Join(partArr, "-"), " : ", strings.Join(codeArr, "-"))
+	}
+}
+
+var tenToAny = map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5", 6: "6", 7: "7", 8: "8", 9: "9", 10: "a", 11: "b", 12: "c", 13: "d", 14: "e", 15: "f", 16: "g", 17: "h", 18: "i", 19: "j", 20: "k", 21: "l", 22: "m", 23: "n", 24: "o", 25: "p", 26: "q", 27: "r", 28: "s", 29: "t", 30: "u", 31: "v", 32: "w", 33: "x", 34: "y", 35: "z", 36: ":", 37: ";", 38: "<", 39: "=", 40: ">", 41: "?", 42: "@", 43: "[", 44: "]", 45: "^", 46: "_", 47: "{", 48: "|", 49: "}", 50: "A", 51: "B", 52: "C", 53: "D", 54: "E", 55: "F", 56: "G", 57: "H", 58: "I", 59: "J", 60: "K", 61: "L", 62: "M", 63: "N", 64: "O", 65: "P", 66: "Q", 67: "R", 68: "S", 69: "T", 70: "U", 71: "V", 72: "W", 73: "X", 74: "Y", 75: "Z"}
+
+func decimalToAny(num int, n int) string {
+	new_num_str := ""
+	var remainder int
+	var remainder_string string
+	for num != 0 {
+		remainder = num % n
+		if 76 > remainder && remainder > 9 {
+			remainder_string = tenToAny[remainder]
+		} else {
+			remainder_string = strconv.Itoa(remainder)
+		}
+		new_num_str = remainder_string + new_num_str
+		num = num / n
+	}
+	return new_num_str
+}

spider/common/common.go

@@ -0,0 +1,242 @@
+package common
+
+import (
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"io"
+	"math/rand"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+)
+
+/* -----------------------------------------------------全局常量定义-------------------------------------------------- */
+var (
+	PCUserAgentList = []string{
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
+		"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
+		"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
+		"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
+		"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
+		"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
+		"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
+		"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
+		"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36",
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
+		"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
+		"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0",
+		"Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0",
+		"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE",
+		"Mozilla/5.0 (Windows NT 6.1; Win64; x64; Trident/7.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E; rv:11.0) like Gecko",
+		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 OPR/60.0.3255.84",
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 OPR/60.0.3255.83",
+		"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4098.3 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3676.400 QQBrowser/10.4.3473.400",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.7.2500",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.7 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36 QIHU 360EE",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
+		"Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko",
+		"Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:56.0)",
+		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 Request Payload",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/604.3.5 (KHTML, like Gecko) Version/11.0.1 Safari/604.3.5",
+		"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Mobile Safari/537.36",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:56.0) Gecko/20100101 Firefox/56.0",
+		"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; rv:11.0) like Gecko",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.74 Safari/537.36 Edg/79.0.309.43",
+		"Mozilla/5.0 (compatible; WOW64; MSIE 10.0; Windows NT 6.2)",
+		"Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv 11.0) like Gecko",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36Edge/13.10586",
+		"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36",
+		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
+		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/11.10 Chromium/27.0.1453.93 Chrome/27.0.1453.93 Safari/537.36",
+		"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
+		"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
+	}
+	CellUserAgentList = []string{
+		"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
+		"Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
+		"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
+		"Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
+		"Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn; HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+		"Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+		"MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+		"Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
+		"Mozilla/5.0 (Linux; U; Android 8.1.0; zh-cn; BLA-AL00 Build/HUAWEIBLA-AL00) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/8.9 Mobile Safari/537.36",
+		"Mozilla/5.0 (Linux; Android 8.1; PAR-AL00 Build/HUAWEIPAR-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/6.2 TBS/044304 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070333) NetType/WIFI Language/zh_CN Process/tools",
+		"Mozilla/5.0 (Linux; Android 6.0.1; OPPO A57 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.83 Mobile Safari/537.36 T7/10.13 baiduboxapp/10.13.0.10 (Baidu; P1 6.0.1)",
+		"Mozilla/5.0 (Linux; Android 8.1; EML-AL00 Build/HUAWEIEML-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/53.0.2785.143 Crosswalk/24.53.595.0 XWEB/358 MMWEBSDK/23 Mobile Safari/537.36 MicroMessenger/6.7.2.1340(0x2607023A) NetType/4G Language/zh_CN",
+		"Mozilla/5.0 (Linux; Android 8.0; DUK-AL20 Build/HUAWEIDUK-AL20; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/6.2 TBS/044353 Mobile Safari/537.36 MicroMessenger/6.7.3.1360(0x26070333) NetType/WIFI Language/zh_CN Process/tools",
+		"Mozilla/5.0 (Linux; U; Android 8.1.0; zh-CN; EML-AL00 Build/HUAWEIEML-AL00) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/11.9.4.974 UWS/2.13.1.48 Mobile Safari/537.36 AliApp(DingTalk/4.5.11) com.alibaba.android.rimet/10487439 Channel/227200 language/zh-CN",
+		"Mozilla/5.0 (Linux; U; Android 5.1.1; zh-CN; SM-J3109 Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/11.8.0.960 UWS/2.12.1.18 Mobile Safari/537.36 AliApp(TB/7.5.4) UCBS/2.11.1.1 WindVane/8.3.0 720X1280",
+		"Mozilla/5.0 (iPhone 6s; CPU iPhone OS 11_4_1 like Mac OS X) AppleWebKit/604.3.5 (KHTML, like Gecko) Version/11.0 MQQBrowser/8.3.0 Mobile/15B87 Safari/604.1 MttCustomUA/2 QBWebViewType/1 WKType/1",
+		"Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.3.5 (KHTML, like Gecko) Version/11.0 MQQBrowser/8.8.2 Mobile/15B87 Safari/604.1 MttCustomUA/2 QBWebViewType/1 WKType/1",
+		"Mozilla/5.0 (iPhone; CPU iPhone OS 11_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15G77 wxwork/2.5.8 MicroMessenger/6.3.22 Language/zh",
+		"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0_2 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Mobile/15A421 wxwork/2.5.8 MicroMessenger/6.3.22 Language/zh",
+		"Mozilla/5.0 (iPhone; CPU iPhone OS 6_1_4 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) CriOS/27.0.1453.10 Mobile/10B350 Safari/8536.25",
+		"Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19",
+		"Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3",
+		"Mozilla/5.0 (iPhone; CPU iPhone OS 8_0_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A366 Safari/600.1.4",
+	}
+)
+
+/* -----------------------------------------------------全局变量定义-------------------------------------------------- */
+var (
+	gbkPattern, _ = regexp.Compile(`meta\s+http-equiv="Content-Type"\s+content="text/html;\s*charset=gbk"`)
+	gbkDecoder    = simplifiedchinese.GBK.NewDecoder()
+)
+
+/* -----------------------------------------------------线程安全字典-------------------------------------------------- */
+type ConcurrentMap struct {
+	data   map[string]interface{}
+	locker sync.RWMutex
+}
+
+func NewConcurrentMap() *ConcurrentMap {
+	return &ConcurrentMap{
+		data: make(map[string]interface{}),
+	}
+}
+
+func (m *ConcurrentMap) Remove(k string) {
+	k = strings.ToLower(strings.TrimSpace(k))
+	if k == "" {
+		return
+	}
+
+	m.locker.Lock()
+	defer m.locker.Unlock()
+
+	delete(m.data, k)
+}
+
+func (m *ConcurrentMap) Set(k string, v interface{}) {
+	k = strings.ToLower(strings.TrimSpace(k))
+	if k == "" {
+		return
+	}
+
+	m.locker.Lock()
+	defer m.locker.Unlock()
+
+	m.data[k] = v
+}
+
+func (m *ConcurrentMap) Append(input map[string]interface{}) {
+	if nil == input || len(input) == 0 {
+		return
+	}
+
+	m.locker.Lock()
+	defer m.locker.Unlock()
+
+	for k, v := range input {
+		k = strings.ToLower(strings.TrimSpace(k))
+		m.data[k] = v
+	}
+}
+
+func (m *ConcurrentMap) Len() int {
+	m.locker.RLock()
+	defer m.locker.RUnlock()
+
+	if nil == m.data {
+		return 0
+	}
+	return len(m.data)
+}
+
+func (m *ConcurrentMap) Keys() []string {
+	m.locker.RLock()
+	defer m.locker.RUnlock()
+
+	if nil == m.data || len(m.data) == 0 {
+		return nil
+	}
+
+	var keys []string
+	for k := range m.data {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+func (m *ConcurrentMap) Values() []interface{} {
+	m.locker.RLock()
+	defer m.locker.RUnlock()
+
+	if nil == m.data || len(m.data) == 0 {
+		return nil
+	}
+
+	var list []interface{}
+	for _, v := range m.data {
+		list = append(list, v)
+	}
+	return list
+}
+
+/* -----------------------------------------------------请求目标消息-------------------------------------------------- */
+type Target struct {
+	Key     string      // 消息发送到的队列名
+	Method  string      // 请求http method
+	URL     string      // 请求目标url
+	Referer string      // 请求目标referer url
+	IsCell  bool        // 是否移动端
+	Data    io.Reader   // 请求参数
+	Item    interface{} // 携带的待输出数据对象（请传递对象引用）
+}
+
+/* -------------------------------------------------------工具函数---------------------------------------------------- */
+// 字符编码转换
+func EncodeTrans(input []byte) []byte {
+	if nil == input || len(input) == 0 {
+		return input
+	}
+
+	if gbkPattern.Match(input) {
+		output, _ := gbkDecoder.Bytes(input)
+		return output
+	} else {
+		return input
+	}
+}
+
+// 随机秒数
+func RandSeconds() time.Duration {
+	rand.Seed(time.Now().UnixNano())
+	return time.Duration(rand.Intn(6)) * time.Second
+}
+
+// 随机获取User-Agent
+func RandUserAgent(isCell bool) string {
+	var list []string
+	if isCell {
+		list = CellUserAgentList
+	} else {
+		list = PCUserAgentList
+	}
+
+	rand.Seed(time.Now().UnixNano())
+	idx := rand.Intn(len(list))
+	return list[idx]
+}

spider/common/proxy_info.go

@@ -0,0 +1,101 @@
+package common
+
+import (
+	"encoding/xml"
+	"git.aionnect.com/aionnect/go-common/utils"
+	"git.aionnect.com/aionnect/go-common/utils/date"
+	"git.aionnect.com/aionnect/go-common/utils/jsonutil"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// 类型
+const (
+	HTTP  = "HTTP"
+	HTTPS = "HTTPS"
+	SOCKS = "SOCKS"
+)
+
+// 匿名度
+const (
+	High   string = "高匿名"
+	Normal string = "透明"
+)
+
+type ProxyInfo struct {
+	IP         string        `json:"ip" xorm:"ip"`
+	Port       int           `json:"port" xorm:"port"`
+	Anonymity  string        `json:"anonymity" xorm:"anonymity"`
+	Type       string        `json:"type" xorm:"type"`
+	Location   string        `json:"location" xorm:"location"`
+	Speed      time.Duration `json:"speed" xorm:"speed"`
+	UpdateTime date.Datetime `json:"updateTime" xorm:"update_time"`
+
+	ID        utils.Long    `xorm:"'id' not null pk BIGINT(20)" json:"id"`
+	UpdatedAt date.Datetime `xorm:"'update_at' updated not null TIMESTAMP" json:"updatedAt,omitempty"`
+	CreatedAt date.Datetime `xorm:"'create_at' not null DATETIME" json:"createdAt,omitempty"`
+}
+
+func ConvertToIP(text string) string {
+	ip := net.ParseIP(strings.TrimSpace(text))
+	if ip.IsLoopback() || ip.IsUnspecified() {
+		return ""
+	}
+	return ip.String()
+}
+
+func ConvertToSpeed(text string) time.Duration {
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return 0
+	}
+
+	text = strings.ReplaceAll(text, "秒", "")
+	n, err := strconv.ParseFloat(text, 10)
+	if nil != err {
+		return 0
+	}
+
+	return time.Duration(n * float64(time.Second))
+}
+
+func (ProxyInfo) TableName() string {
+	return "proxy_info"
+}
+
+func (that *ProxyInfo) UnmarshalJSON(value []byte) error { // 反序列化Json的时候，如果没有ID，自动给一个
+	type Alias ProxyInfo
+	alias := &struct {
+		*Alias
+	}{Alias: (*Alias)(that)}
+	if err := jsonutil.Unmarshal(value, &alias); err != nil {
+		return err
+	}
+	if nil != alias && alias.ID == 0 {
+		alias.ID = utils.NextId()
+	}
+	return nil
+}
+
+func (that *ProxyInfo) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { // 反序列化XML的时候，如果没有ID，自动给一个
+	type Alias ProxyInfo
+	alias := &struct {
+		*Alias
+	}{Alias: (*Alias)(that)}
+	if err := d.DecodeElement(&alias, &start); err != nil {
+		return err
+	}
+	if nil != alias && alias.ID == 0 {
+		alias.ID = utils.NextId()
+	}
+	return nil
+}
+
+// 分页参数
+type ProxyPagingParams struct {
+	PagingNo int // 页码
+	Limit    int // 一次最多爬多少条（因为按分页爬，可能会稍微超出一点点）
+	Count    int // 本次已爬有效数据条数计数
+}

spider/dao/db_conn.go

@@ -0,0 +1,155 @@
+package dao
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/spf13/viper"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"time"
+	"xorm.io/xorm"
+)
+
+var (
+	dbConnMap = make(map[string]*DBInfo)
+	dbOnce    sync.Once
+)
+
+type DBInfo struct {
+	Name string
+	DB   *xorm.Engine
+}
+
+func init() {
+	viper.SetDefault("db.global.max_idle", 4000)
+	viper.SetDefault("db.global.max_open", 8000)
+	viper.SetDefault("db.global.max_lifetime", 8000000000)
+	viper.SetDefault("db.spider.host", "root:123456@tcp(localhost:3306)")
+	viper.SetDefault("db.spider.name", "spider")
+}
+
+// 获取特定数据库访问对象的实例
+func DB(name string) *xorm.Engine {
+	if len(strings.TrimSpace(name)) == 0 {
+		return nil
+	}
+
+	openDB()
+
+	if info, ok := dbConnMap[name]; ok {
+		return info.DB
+	} else {
+		println("DB "+name+" disconnected")
+		return nil
+	}
+}
+
+// 打开所有数据库
+func openDB() {
+	dbOnce.Do(func() { // 一次程序运行仅执行一次，以达到“单例”的效果
+		readDBConfigAndOpen()
+		go func() {
+			defer closeDB()
+			quit := make(chan os.Signal)
+			signal.Notify(quit, os.Interrupt)
+			<-quit
+		}()
+	})
+}
+
+// 关闭所有数据库
+func closeDB() {
+	if nil == dbConnMap || len(dbConnMap) == 0 {
+		return
+	}
+
+	for _, dbInfo := range dbConnMap {
+		if nil != dbInfo && nil != dbInfo.DB {
+			err := dbInfo.DB.Close()
+			if nil != err {
+				println("Close db "+dbInfo.Name+" failed:", err.Error())
+				continue
+			}
+			println("Close db " + dbInfo.Name)
+		}
+	}
+}
+
+// 读取配置文件并进行数据库连接，方法名首字母小写即为包私有方法，否则若大写则为公共方法
+func readDBConfigAndOpen() {
+	dbs := viper.GetStringMap("db")
+	if nil == dbs || len(dbs) == 0 {
+		println("not found any DB config")
+		return
+	}
+
+	globalMaxIdle := viper.GetInt("db.global.max_idle")
+	globalMaxOpen := viper.GetInt("db.global.max_open")
+	globalMaxLifetime := viper.GetDuration("db.global.max_lifetime")
+	for name := range dbs {
+		key := name
+		if key == "global" {
+			continue
+		}
+		cnf := dbs[key].(map[string]interface{})
+		if nil == cnf {
+			println("wrong DB config")
+			return
+		}
+		var dbHost, dbName, maxIdle, maxOpen, maxLifetime interface{}
+		var ok bool
+		if dbHost, ok = cnf["host"]; !ok {
+			println("wrong DB host config")
+			return
+		}
+		if dbName, ok = cnf["name"]; !ok {
+			println("wrong DB name config")
+			return
+		}
+		if maxIdle, ok = cnf["max_idle"]; !ok {
+			maxIdle = globalMaxIdle
+		}
+		if maxOpen, ok = cnf["max_open"]; !ok {
+			maxOpen = globalMaxOpen
+		}
+		if maxLifetime, ok = cnf["max_lifetime"]; !ok {
+			maxLifetime = globalMaxLifetime
+		}
+
+		newDB(name, dbHost.(string), dbName.(string),
+			maxIdle.(int), maxOpen.(int), maxLifetime.(time.Duration))
+	}
+}
+
+func newDB(name string, dbHost string, dbName string,
+	maxIdle int, maxOpen int, maxLifetime time.Duration) {
+	db, err := xorm.NewEngine("mysql", dbHost+"/"+dbName+"?charset=utf8&parseTime=True&loc=Local")
+	if err != nil {
+		println("Open DB conn failed", err.Error())
+		return
+	}
+
+	// 设置连接池的空闲数大小
+	if maxIdle > 0 {
+		db.SetMaxIdleConns(maxIdle)
+	}
+	// 设置最大打开连接数
+	if maxOpen > 0 {
+		db.SetMaxOpenConns(maxOpen)
+	}
+	// 设置最大连接超时时间
+	if maxLifetime > 0 {
+		db.SetConnMaxLifetime(maxLifetime)
+	}
+
+	// 连接测试
+	if err = db.DB().Ping(); err != nil {
+		println("Can not conn to DB", err.Error())
+		return
+	}
+
+	println("Open db " + dbName)
+
+	dbConnMap[name] = &DBInfo{Name: dbName, DB: db}
+}

spider/dao/proxy_dao.go

@@ -0,0 +1,101 @@
+package dao
+
+import (
+	"fmt"
+	"git.aionnect.com/aionnect/go-common/utils"
+	"git.aionnect.com/aionnect/go-common/utils/date"
+	"git.aionnect.com/hello-go/spider/common"
+	"math/rand"
+	"strings"
+	"sync"
+	"time"
+	"xorm.io/xorm"
+)
+
+var proxyOnce sync.Once
+var proxyDao *ProxyDao
+
+// 代理信息数据访问对象
+type ProxyDao struct {
+	db    *xorm.Engine          // 数据库访问对象
+	cache *common.ConcurrentMap // 本地缓存
+}
+
+// 返回代理信息数据库访问对象
+func NewProxyDao() *ProxyDao {
+	proxyOnce.Do(func() {
+		proxyDao = &ProxyDao{
+			db:    DB("spider"),
+			cache: common.NewConcurrentMap(),
+		}
+	})
+	return proxyDao
+}
+
+// 保存代理信息到数据库
+func (d *ProxyDao) Save(proxies []*common.ProxyInfo) {
+	if nil == proxies || len(proxies) == 0 {
+		return
+	}
+
+	for i := 0; i < len(proxies); i++ {
+		if proxies[i].ID == 0 {
+			proxies[i].ID = utils.NextId()
+		}
+		proxies[i].CreatedAt = date.Now()
+	}
+
+	err := utils.Insert(d.db, &proxies)
+	if nil != err {
+		fmt.Printf("save proxies to db failed %s", err.Error())
+	}
+}
+
+const ReadClause = `select distinct addr from (
+  select concat(ip, ':', port) as 'addr'
+  from proxy_info
+  where (type = 'HTTP' or type = 'HTTPS')
+    and anonymity = '高匿名'
+  order by update_time desc, speed
+  limit 100) T`
+
+// 查询代理信息
+func (d *ProxyDao) Get() string {
+	// 当缓存中数据量过少时，从数据库中读取最新的100条
+	if d.cache.Len() < 10 {
+		var addrs []string
+		err := d.db.SQL(ReadClause).Find(&addrs)
+		if nil != err {
+			fmt.Printf("read proxies to db failed %s", err.Error())
+		} else if nil != addrs && len(addrs) > 0 {
+			m := make(map[string]interface{})
+			for i := 0; i < len(addrs); i++ {
+				m[addrs[i]] = true
+			}
+			d.cache.Append(m)
+		}
+	}
+	// 从缓存中返回随机一个
+	keys := d.cache.Keys()
+	rand.Seed(time.Now().UnixNano())
+	idx := rand.Intn(len(keys))
+	return keys[idx]
+}
+
+// 删除代理信息
+func (d *ProxyDao) Remove(key string) {
+	key = strings.TrimSpace(key)
+	if key == "" {
+		return
+	}
+	// 缓存中移除
+	d.cache.Remove(key)
+	// 数据库中移除
+	idx := strings.LastIndex(key, ":")
+	ip := key[:idx]
+	port := strings.TrimLeft(key[idx:], ":")
+	_, err := d.db.Where("ip=? and port=?", ip, port).Delete(&common.ProxyInfo{})
+	if nil != err {
+		fmt.Printf("remove proxy to db failed %s", err.Error())
+	}
+}

spider/schema.sql

@@ -0,0 +1,40 @@
+create schema spider;
+
+create table if not exists proxy_info
+(
+	id bigint not null comment 'ID'
+		primary key,
+	ip varchar(20) null comment 'IP',
+	port int null comment '端口',
+	anonymity varchar(10) null comment '匿名度',
+	type varchar(10) null comment '类型',
+	location varchar(100) null comment '位置',
+	speed bigint null comment '响应速度',
+	update_time datetime null comment '最后验证时间',
+	update_at timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP comment '最后更新时间',
+	create_at datetime not null comment '创建时间'
+)
+comment '代理信息' charset=utf8mb4;
+
+create table if not exists qian_li_ma_content
+(
+	id bigint not null comment 'ID'
+		primary key,
+	content_id int not null comment '项目id',
+	prog_name varchar(20) null comment '项目类型名称',
+	update_time date null comment '项目更新时间',
+	url varchar(500) null comment '项目详情页链接',
+	area_id int null comment '区域id',
+	area_name varchar(50) null comment '区域名称',
+	title varchar(500) null comment '项目标题',
+	prog_id int null comment '项目类型编号',
+	cat_id int null comment '项目类别编号',
+	is_concern int null comment '是否影响',
+	province varchar(50) null comment '省份',
+	city varchar(50) null comment '城市',
+	status varchar(20) null comment '项目状态',
+	text longtext null comment '项目正文',
+	update_at timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP comment '最后更新时间',
+	create_at datetime not null comment '创建时间'
+)
+comment '千里马项目内容' charset=utf8mb4;

spider/spiders/base_spider.go

@@ -0,0 +1,166 @@
+package spiders
+
+import (
+	"fmt"
+	"git.aionnect.com/aionnect/go-common/utils"
+	"git.aionnect.com/hello-go/spider/common"
+	"git.aionnect.com/hello-go/spider/dao"
+	"net/http"
+	"strings"
+	"sync"
+)
+
+// 爬虫调度上下文
+type spidersContext struct {
+	counter   sync.WaitGroup     // 同步器
+	spiderMap map[string]*Spider // 多级爬虫价值对
+}
+
+// 开始爬虫调度
+func Run(startingTarget *common.Target, spiders ...ISpider) {
+	if nil == startingTarget || nil == spiders || len(spiders) == 0 {
+		return
+	}
+
+	// 爬虫调度上下文初始化
+	ctx := &spidersContext{
+		spiderMap: make(map[string]*Spider),
+	}
+	for i := 0; i < len(spiders); i++ {
+		spider := newSpider(spiders[i], ctx) // 包装上爬虫执行类
+		name := strings.TrimSpace(spider.Name())
+		if _, ok := ctx.spiderMap[name]; !ok {
+			ctx.spiderMap[name] = spider
+		}
+	}
+
+	// 从起始页开始处理
+	ctx.router(startingTarget)
+
+	// 等待爬虫执行完毕
+	ctx.counter.Wait()
+
+	// 爬虫执行完毕后处理
+	for _, spider := range ctx.spiderMap {
+		if after, ok := spider.ISpider.(IAfterSpiderExit); ok {
+			after.AfterExit()
+		}
+		close(spider.queue)
+	}
+}
+
+// 爬虫路由处理
+func (ctx *spidersContext) router(target *common.Target) {
+	if nil == target || nil == ctx.spiderMap || len(ctx.spiderMap) == 0 {
+		fmt.Println("spidersContext.router() invalid params")
+		return
+	}
+	target.Key = strings.TrimSpace(target.Key)
+	if target.Key == "" {
+		fmt.Println("spidersContext.router() target key can not be empty")
+		return
+	}
+
+	if spider, ok := ctx.spiderMap[target.Key]; ok {
+		ctx.counter.Add(1)
+		spider.Send(target)
+	}
+}
+
+// 爬虫主接口抽象
+type ISpider interface {
+	Name() string                                                  // 爬虫命名
+	Parse(item interface{}, body []byte, headers http.Header) ([]*common.Target, error) // 响应解析
+}
+
+// 自定义爬虫请求接口抽象
+type IPromiseDefiner interface {
+	GetPromise() *utils.RequestPromise
+}
+
+// 爬虫执行结束后需进行的处理接口抽象
+type IAfterSpiderExit interface {
+	AfterExit()
+}
+
+// 爬虫执行类（模拟抽象类及模版方法模式）
+type Spider struct {
+	ISpider
+	queue    chan *common.Target   // 待爬队列
+	promise  *utils.RequestPromise // 请求对象
+	context  *spidersContext       // 爬虫调度上下文的引用
+	proxyDao *dao.ProxyDao         // 代理信息数据访问对象
+}
+
+// 返回新的爬虫调度类对象实例（模版方法）
+func newSpider(instance ISpider, ctx *spidersContext) *Spider {
+	spider := &Spider{
+		ISpider:  instance,
+		queue:    make(chan *common.Target, 5),
+		context:  ctx,
+		proxyDao: dao.NewProxyDao(),
+	}
+	if definer, ok := instance.(IPromiseDefiner); ok {
+		spider.promise = definer.GetPromise()
+	} else {
+		spider.promise = utils.NewRequest()
+	}
+	go spider.handle() // 启动待爬队列异步处理协程
+	return spider
+}
+
+// 新待爬请求入队
+func (s *Spider) Send(target *common.Target) {
+	if nil == target {
+		return
+	}
+
+	// 入队不启动新协程，起到bufferSize满时阻塞等待的效果
+	s.queue <- target
+}
+
+// 待爬队列处理
+func (s *Spider) handle() {
+	for {
+		select {
+		case target, ok := <-s.queue:
+			if ok {
+				s.invoke(target)
+				s.context.counter.Done() // 同步器扣减
+			}
+		}
+	}
+}
+
+// 请求并处理（模版方法）
+func (s *Spider) invoke(target *common.Target) {
+	target.Referer = strings.TrimSpace(target.Referer)
+	if target.Referer != "" {
+		s.promise = s.promise.SetHeader("Referer", target.Referer)
+	}
+	s.promise.SetHeader("User-Agent", common.RandUserAgent(target.IsCell)) // 随机User-Agent
+	s.promise.SetHttpProxy(s.proxyDao.Get())                               // 随机代理
+	body, err := s.promise.Call(target.Method, target.URL, target.Data)
+	if nil != err {
+		fmt.Printf("http request [%s] %s failed. %s\n", target.Method, target.URL, err.Error())
+		return
+	}
+	if nil == body || len(body) <= 0 {
+		fmt.Printf("empty http response [%s] %s", target.Method, target.URL)
+		return
+	}
+	body = common.EncodeTrans(body)
+	var nextTargets []*common.Target
+	nextTargets, err = s.Parse(target.Item, body, nil)
+	if nil != err {
+		fmt.Println(err.Error())
+		return
+	}
+	if nil == nextTargets || len(nextTargets) == 0 {
+		return
+	}
+	for i := 0; i < len(nextTargets); i++ {
+		next := nextTargets[i]
+		s.context.router(next)
+	}
+}

spider/spiders/kuaidaili/free_paging.go

@@ -0,0 +1,113 @@
+package kuaidaili
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"git.aionnect.com/aionnect/go-common/utils/date"
+	"git.aionnect.com/hello-go/spider/common"
+	"git.aionnect.com/hello-go/spider/dao"
+	"git.aionnect.com/hello-go/spider/spiders"
+	"github.com/PuerkitoBio/goquery"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+/* -------------------------------------------------------常量定义---------------------------------------------------- */
+const TargetUrl = "https://www.kuaidaili.com/free/inha/%d/"
+const KuaiDaiLi = "KuaiDaiLi"
+
+/* -------------------------------------------------------爬虫实现---------------------------------------------------- */
+
+// 千里马项目网访客详情页爬虫类
+type FreePagingSpider struct {
+	proxyDao *dao.ProxyDao // 代理信息数据访问对象
+}
+
+// 返回当前爬虫类相应爬虫调度类对象实例
+func NewFreePagingSpider() spiders.ISpider {
+	spider := &FreePagingSpider{
+		proxyDao: dao.NewProxyDao(),
+	}
+	return spider
+}
+
+// 爬虫命名
+func (s *FreePagingSpider) Name() string {
+	return KuaiDaiLi
+}
+
+// 响应解析
+func (s *FreePagingSpider) Parse(item interface{}, body []byte, headers http.Header) ([]*common.Target, error) {
+	params := item.(*common.ProxyPagingParams)
+	if params.PagingNo == 2 { // 只爬一页，仅测试时使用
+		return nil, nil
+	}
+
+	// 数据解析
+	dom, _ := goquery.NewDocumentFromReader(bytes.NewReader(body))
+	pageNo := dom.Find("#listnav li a").Last().Text()
+	totalPage, err := strconv.Atoi(pageNo)
+	if nil != err { // 获取不到尾页信息，异常终止
+		return nil, errors.New(fmt.Sprintf("wrong last page bumber %s", err.Error()))
+	}
+	flag := true
+	var proxies []*common.ProxyInfo
+	dom.Find(".table-bordered tbody tr").Each(func(i int, row *goquery.Selection) {
+		flag = false // 有数据
+		proxy := &common.ProxyInfo{}
+		row.Find("td").Each(func(j int, cell *goquery.Selection) {
+			switch j {
+			case 0:
+				proxy.IP = common.ConvertToIP(cell.Text())
+			case 1:
+				proxy.Port, _ = strconv.Atoi(strings.TrimSpace(cell.Text()))
+			case 2:
+				proxy.Anonymity = strings.TrimSpace(cell.Text())
+			case 3:
+				proxy.Type = strings.ToUpper(strings.TrimSpace(cell.Text()))
+			case 4:
+				proxy.Location = strings.TrimSpace(cell.Text())
+			case 5:
+				proxy.Speed = common.ConvertToSpeed(cell.Text())
+			case 6:
+				proxy.UpdateTime = date.ParseDatetime(cell.Text())
+			}
+		})
+		if proxy.IP != "" && proxy.Port >= 0 && proxy.Anonymity == common.High && (proxy.Type == common.HTTP || proxy.Type == common.HTTPS) {
+			if proxy.Port == 0 {
+				if proxy.Type == common.HTTP {
+					proxy.Port = 80
+				} else {
+					proxy.Port = 443
+				}
+			}
+			proxies = append(proxies, proxy)
+		}
+	})
+	// 保存
+	s.proxyDao.Save(proxies)
+
+	// 判断是否终止分页轮询
+	if flag { // 当前分页无数据，异常终止
+		return nil, errors.New(fmt.Sprintf("paging has no rows %d", params.PagingNo))
+	}
+	params.Count += len(proxies)
+	if params.Count > params.Limit { // 超限，终止
+		return nil, nil
+	}
+	if totalPage <= params.PagingNo { // 尾页，终止
+		return nil, nil
+	}
+	// 轮询列表页下一页（暂为顺序轮询）
+	params.PagingNo++
+	url := fmt.Sprintf(TargetUrl, params.PagingNo)
+	target := &common.Target{
+		Key:    KuaiDaiLi,
+		Method: http.MethodGet,
+		URL:    url,
+		Item:   params,
+	}
+	return []*common.Target{target}, nil
+}

spider/spiders/kuaidaili/kuaidaiki_test.go

@@ -0,0 +1,24 @@
+package kuaidaili
+
+import (
+	"fmt"
+	"git.aionnect.com/hello-go/spider/common"
+	"git.aionnect.com/hello-go/spider/spiders"
+	"net/http"
+	"testing"
+)
+
+func TestFreePagingSpider(t *testing.T) {
+	// 从起始页开始执行爬虫
+	params := &common.ProxyPagingParams{PagingNo: 1, Limit: 1000}
+	startingUrl := fmt.Sprintf(TargetUrl, params.PagingNo)
+	target := &common.Target{
+		Key:     KuaiDaiLi,
+		Method:  http.MethodGet,
+		URL:     startingUrl,
+		Item:    params,
+	}
+	spiders.Run(target,
+		NewFreePagingSpider(),
+	)
+}

spider/spiders/qianlima/items/qianlima.go

@@ -0,0 +1,81 @@
+package items
+
+import (
+	"encoding/xml"
+	"git.aionnect.com/aionnect/go-common/utils"
+	"git.aionnect.com/aionnect/go-common/utils/date"
+	"git.aionnect.com/aionnect/go-common/utils/jsonutil"
+)
+
+type Content struct {
+	ContentId  int       `json:"contentid" xorm:"content_id"`
+	ProgName   string    `json:"progName" xorm:"prog_name"`
+	UpdateTime date.Date `json:"updateTime" xorm:"update_time"`
+	URL        string    `json:"url" xorm:"url"`
+	AreaId     int       `json:"areaId" xorm:"area_id"`
+	AreaName   string    `json:"areaName" xorm:"area_name"`
+	PopTitle   string    `json:"popTitle" xorm:"title"`
+	ProgId     int       `json:"progId" xorm:"prog_id"`
+	CatId      int       `json:"catid" xorm:"cat_id"`
+	IsConcern  int       `json:"isConcern" xorm:"is_concern"`
+	//ShowTitle  string    `json:"showTitle"`
+	Province string `json:"province" xorm:"province"`
+	City     string `json:"city" xorm:"city"`
+	Status   string `json:"status" xorm:"status"`
+	Text     string `json:"text" xorm:"text"`
+
+	ID        utils.Long    `xorm:"'id' not null pk BIGINT(20)" json:"id"`
+	UpdatedAt date.Datetime `xorm:"'update_at' updated not null TIMESTAMP" json:"updatedAt,omitempty"`
+	CreatedAt date.Datetime `xorm:"'create_at' not null DATETIME" json:"createdAt,omitempty"`
+}
+
+type Paging struct {
+	RowCount   int        `json:"rowCount"`
+	PagesCount int        `json:"pagesCount"`
+	Data       []*Content `json:"data"`
+	IsLaunch   int        `json:"isLaunch"`
+}
+
+type Result struct {
+	Status int     `json:"status"`
+	Msg    string  `json:"msg"`
+	Data   *Paging `json:"data"`
+}
+
+func (Content) TableName() string {
+	return "qian_li_ma_content"
+}
+
+func (that *Content) UnmarshalJSON(value []byte) error { // 反序列化Json的时候，如果没有ID，自动给一个
+	type Alias Content
+	alias := &struct {
+		*Alias
+	}{Alias: (*Alias)(that)}
+	if err := jsonutil.Unmarshal(value, &alias); err != nil {
+		return err
+	}
+	if nil != alias && alias.ID == 0 {
+		alias.ID = utils.NextId()
+	}
+	return nil
+}
+
+func (that *Content) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { // 反序列化XML的时候，如果没有ID，自动给一个
+	type Alias Content
+	alias := &struct {
+		*Alias
+	}{Alias: (*Alias)(that)}
+	if err := d.DecodeElement(&alias, &start); err != nil {
+		return err
+	}
+	if nil != alias && alias.ID == 0 {
+		alias.ID = utils.NextId()
+	}
+	return nil
+}
+
+// 分页参数
+type PagingParams struct {
+	PagingNo  int // 页码
+	DaysLimit int // 最多只爬到几天前的数据
+}

spider/spiders/qianlima/visitor_detail.go

@@ -0,0 +1,126 @@
+package qianlima
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"git.aionnect.com/aionnect/go-common/utils"
+	"git.aionnect.com/aionnect/go-common/utils/date"
+	"git.aionnect.com/hello-go/spider/common"
+	"git.aionnect.com/hello-go/spider/dao"
+	"git.aionnect.com/hello-go/spider/spiders"
+	"git.aionnect.com/hello-go/spider/spiders/qianlima/items"
+	"github.com/PuerkitoBio/goquery"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+	"xorm.io/xorm"
+)
+
+/* -------------------------------------------------------常量定义---------------------------------------------------- */
+const VisitorDetail = "VisitorDetail"
+
+/* -------------------------------------------------------全局变量---------------------------------------------------- */
+var (
+	visitorDetailPromise = utils.NewRequest(). // 详情页请求
+		SetHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9").
+		SetHeader("Accept-Encoding", "gzip, deflate").
+		SetHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8").
+		SetHeader("Cache-Control", "max-age=0").
+		SetHeader("Connection", "keep-alive").
+		SetHeader("Host", "www.qianlima.com").
+		SetHeader("Upgrade-Insecure-Requests", "1").
+		SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+)
+
+/* -------------------------------------------------------爬虫实现---------------------------------------------------- */
+
+// 千里马项目网访客详情页爬虫类
+type VisitorDetailSpider struct {
+	db        *xorm.Engine          // 数据库访问对象
+	resultMap *common.ConcurrentMap // 数据暂存价值对
+	ticker    *time.Ticker          // 显示进度的定时器
+}
+
+// 返回当前爬虫类相应爬虫调度类对象实例
+func NewVisitorDetailSpider() spiders.ISpider {
+	spider := &VisitorDetailSpider{
+		db:        dao.DB("spider"),
+		resultMap: common.NewConcurrentMap(),
+		ticker:    time.NewTicker(5 * time.Second),
+	}
+
+	// 进度打印（非核心代码）
+	go func(ticker *time.Ticker) {
+		for {
+			<-ticker.C
+			println("---------------", spider.resultMap.Len())
+		}
+	}(spider.ticker)
+
+	return spider
+}
+
+// 获取请求对象
+func (s *VisitorDetailSpider) GetPromise() *utils.RequestPromise {
+	return visitorDetailPromise
+}
+
+// 爬虫命名
+func (s *VisitorDetailSpider) Name() string {
+	return VisitorDetail
+}
+
+// 响应解析
+func (s *VisitorDetailSpider) Parse(item interface{}, body []byte, headers http.Header) ([]*common.Target, error) {
+	content, ok := item.(*items.Content)
+	if !ok {
+		return nil, errors.New(fmt.Sprintf("invaild item %+v", item))
+	}
+
+	// 数据解析
+	dom, _ := goquery.NewDocumentFromReader(bytes.NewReader(body))
+	status := dom.Find(".wenshang .zhuangtai").First().Text()
+	content.Status = strings.TrimLeft(strings.TrimSpace(status), "状态：")
+	dom.Find(".wenshang .site a").Each(func(i int, selection *goquery.Selection) {
+		text := strings.TrimSpace(strings.Trim(strings.TrimSpace(selection.Text()), "-"))
+		if i == 0 {
+			content.Province = text
+		} else if i == 1 {
+			content.City = text
+		}
+	})
+	wen, _ := dom.Find("#wen").First().Html()
+	wen = strings.Replace(wen, "<input type=\"hidden\" id=\"zbunit\" value=\"\"/>", "", -1)
+	wen = strings.Replace(wen, "<input type=\"hidden\" id=\"zburl\" value=\"\"/>", "", -1)
+	wen = spacePattern.ReplaceAllString(wen, "")
+	content.Text = wen
+	// 数据解析后处理逻辑
+	s.resultMap.Set(strconv.Itoa(content.ContentId), nil) // 暂存结果到字典中（暂时无用，仅做进度打印，非核心代码）
+	s.exportToDB(content)                                 // 逐条储存结果到数据库
+	return nil, nil
+}
+
+// 爬虫执行结束后需进行的处理
+func (s *VisitorDetailSpider) AfterExit() {
+	s.ticker.Stop()
+	println("@---------------", s.resultMap.Len()) // 详情页爬虫执行结束时再打印一下进度（非核心代码）
+}
+
+/* -------------------------------------------------------数据保存---------------------------------------------------- */
+func (s *VisitorDetailSpider) exportToDB(content *items.Content) {
+	if nil == content {
+		return
+	}
+
+	if content.ID == 0 {
+		content.ID = utils.NextId()
+	}
+	content.CreatedAt = date.Now()
+
+	_, err := s.db.InsertOne(content)
+	if nil != err {
+		fmt.Printf("export to db failed %s %+v\n", err.Error(), content)
+	}
+}

spider/spiders/qianlima/visitor_paging.go

@@ -0,0 +1,117 @@
+package qianlima
+
+import (
+	"errors"
+	"fmt"
+	"git.aionnect.com/aionnect/go-common/utils"
+	"git.aionnect.com/aionnect/go-common/utils/date"
+	"git.aionnect.com/aionnect/go-common/utils/jsonutil"
+	"git.aionnect.com/hello-go/spider/common"
+	"git.aionnect.com/hello-go/spider/spiders"
+	"git.aionnect.com/hello-go/spider/spiders/qianlima/items"
+	"net/http"
+	"regexp"
+	"time"
+)
+
+/* -------------------------------------------------------常量定义---------------------------------------------------- */
+//const TargetUrl = "http://search.qianlima.com/api/v1/website/search?filtermode=1&timeType=101&areas=&types=-1&searchMode=0&keywords=led&beginTime=&endTime=&isfirst=true&currentPage=1&numPerPage=20"
+const TargetUrl = "http://search.qianlima.com/api/v1/website/search?keywords=led&currentPage=%d&numPerPage=50"
+const RefererUrl = "http://search.qianlima.com/?q=led"
+const VisitorPaging = "VisitorPaging"
+
+/* -------------------------------------------------------全局变量---------------------------------------------------- */
+var (
+	visitorPagingPromise = utils.NewRequest(). // 请求对象
+		SetHeader("Accept", "*/*").
+		SetHeader("Accept-Encoding", "gzip, deflate").
+		SetHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8").
+		SetHeader("Connection", "keep-alive").
+		SetHeader("Content-Length", "0").
+		SetHeader("Content-Type", "application/x-www-form-unlencoded").
+		SetHeader("Host", "search.qianlima.com").
+		SetHeader("Origin", "http://search.qianlima.com").
+		SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+	spacePattern, _ = regexp.Compile(`\n\s*`) // 去空格正则
+)
+
+/* -------------------------------------------------------爬虫实现---------------------------------------------------- */
+
+// 千里马项目网访客列表页爬虫类
+type VisitorPagingSpider struct{}
+
+// 返回当前爬虫类相应爬虫调度类对象实例
+func NewVisitorPagingSpider() spiders.ISpider {
+	spider := &VisitorPagingSpider{}
+	return spider
+}
+
+// 获取请求对象
+func (s *VisitorPagingSpider) GetPromise() *utils.RequestPromise {
+	return visitorPagingPromise
+}
+
+// 爬虫命名
+func (s *VisitorPagingSpider) Name() string {
+	return VisitorPaging
+}
+
+// 响应解析
+func (s *VisitorPagingSpider) Parse(item interface{}, body []byte, headers http.Header) ([]*common.Target, error) {
+	params := item.(*items.PagingParams)
+	//if params.PagingNo == 2 { // 只爬一页，仅测试时使用
+	//	return nil, nil
+	//}
+
+	// 数据解析
+	var res items.Result
+	err := jsonutil.Unmarshal(body, &res)
+	if nil != err { // 无法解析的响应，异常终止
+		return nil, errors.New(fmt.Sprint("decode paging response failed", err.Error()))
+	}
+	if nil == &res || nil == res.Data || res.Status != 200 || // 错误响应，异常终止
+		res.Data.RowCount <= 0 || res.Data.PagesCount <= 0 ||
+		nil == res.Data.Data || len(res.Data.Data) <= 0 {
+		return nil, errors.New(fmt.Sprint("invalid paging response", string(body)))
+	}
+	// 当前分页列表，将项目循环加入详情页待爬队列（暂为顺序轮询）
+	var targets []*common.Target
+	limitDate := date.Today().AddDays(params.DaysLimit) // 千里马免费项目列表七天前存在大量重复无效数据
+	flag := false
+	for i := 0; i < len(res.Data.Data); i++ {
+		item := res.Data.Data[i]
+		if item.UpdateTime.After(limitDate) { // 爬到七天前的数据即置flag为否，不再爬后续分页，也可按照实际业务要求更改此逻辑
+			flag = true
+			break
+		}
+
+		target := &common.Target{
+			Key:    VisitorDetail,
+			Method: http.MethodGet,
+			URL:    item.URL,
+			Item:   item, // 这个对象会传递给下一层的Parse方法
+		}
+		targets = append(targets, target)
+	}
+
+	// 判断是否终止分页轮询
+	if flag { // 超限，终止
+		return targets, nil
+	}
+	if res.Data.PagesCount <= params.PagingNo { // 尾页，终止
+		return targets, nil
+	}
+	time.Sleep(common.RandSeconds()) // 随机暂停几秒
+	// 轮询列表页下一页（暂为顺序轮询）
+	params.PagingNo++
+	url := fmt.Sprintf(TargetUrl, params.PagingNo)
+	target := &common.Target{
+		Key:     VisitorPaging,
+		Method:  http.MethodPost,
+		URL:     url,
+		Referer: RefererUrl,
+		Item:    params,
+	}
+	targets = append(targets, target)
+	return targets, nil
+}

spider/spiders/qianlima/visitor_test.go

@@ -0,0 +1,28 @@
+package qianlima
+
+import (
+	"fmt"
+	"git.aionnect.com/hello-go/spider/common"
+	"git.aionnect.com/hello-go/spider/spiders"
+	"git.aionnect.com/hello-go/spider/spiders/qianlima/items"
+	"net/http"
+	"testing"
+)
+
+func TestVisitorSpider(t *testing.T) {
+	// 从起始页开始执行爬虫
+	params := &items.PagingParams{PagingNo: 1, DaysLimit: 7}
+	startingUrl := fmt.Sprintf(TargetUrl, params.PagingNo)
+	target := &common.Target{
+		Key:     VisitorPaging,
+		Method:  http.MethodPost,
+		URL:     startingUrl,
+		Referer: RefererUrl,
+		Item:    params,
+	}
+	spiders.Run(target,
+		NewVisitorPagingSpider(),
+		NewVisitorDetailSpider(),
+	)
+	fmt.Println("Done!")
+}

temp.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"strings"
+)
+
+func main() {
+	s := "234.32.12.34:4222"
+	idx := strings.LastIndex(s, ":")
+	ip := s[:idx]
+	port := strings.TrimLeft(s[idx:], ":")
+	println(ip, port)
+}